diff --git a/degesch.c b/degesch.c
index 1cdc12c..e90d393 100644
--- a/degesch.c
+++ b/degesch.c
@@ -2767,6 +2767,21 @@ irc_get_or_make_user_buffer (struct server *s, const char *nickname)
 	return buffer;
 }
 
+// Note that this eats the user reference
+static void
+irc_channel_link_user (struct channel *channel, struct user *user,
+	const char *prefixes)
+{
+	struct user_channel *user_channel = user_channel_new ();
+	user_channel->channel = channel;
+	LIST_PREPEND (user->channels, user_channel);
+
+	struct channel_user *channel_user = channel_user_new ();
+	channel_user->user = user;
+	str_append (&channel_user->prefixes, prefixes);
+	LIST_PREPEND (channel->users, channel_user);
+}
+
 static void
 irc_channel_unlink_user
 	(struct channel *channel, struct channel_user *channel_user)
@@ -3870,25 +3885,10 @@ irc_handle_join (struct server *s, const struct irc_message *msg)
 	if (!channel)
 		return;
 
-	// Get or make a user object
+	// Add the user to the channel
 	char *nickname = irc_cut_nickname (msg->prefix);
-	struct user *user = str_map_find (&s->irc_users, nickname);
-	if (!user)
-		user = irc_make_user (s, nickname);
-	else
-	{
-		user = user_ref (user);
-		free (nickname);
-	}
-
-	// Link the user with the channel
-	struct user_channel *user_channel = user_channel_new ();
-	user_channel->channel = channel;
-	LIST_PREPEND (user->channels, user_channel);
-
-	struct channel_user *channel_user = channel_user_new ();
-	channel_user->user = user;
-	LIST_PREPEND (channel->users, channel_user);
+	irc_channel_link_user (channel, irc_get_or_make_user (s, nickname), "");
+	free (nickname);
 
 	// Finally log the message
 	if (buffer)
@@ -4705,12 +4705,11 @@ irc_process_names (struct server *s, struct channel *channel)
 			irc_channel_get_user (channel, user);
 		if (!channel_user)
 		{
-			channel_user = channel_user_new ();
-			channel_user->user = user;
-			LIST_PREPEND (channel->users, channel_user);
+			irc_channel_link_user (channel, user, prefixes);
+			continue;
 		}
-		else
-			user_unref (user);
+
+		user_unref (user);
 
 		// If our idea of the user's modes disagrees with what the server's
 		// sent us (the most powerful modes differ), use the latter one