diff --git a/degesch.c b/degesch.c
index 0346867..1722e40 100644
--- a/degesch.c
+++ b/degesch.c
@@ -409,6 +409,8 @@ struct app_context
 	//   maybe also broadcast all buffers about the disconnection event
 	// TODO: when getting connected again, rejoin all current channels
 
+	// FIXME: these should only have weak references, so that we don't have
+	//   to watch all places where they might get unref-ed
 	struct str_map irc_users;           ///< IRC user data
 	struct str_map irc_channels;        ///< IRC channel data
 	struct str_map irc_buffer_map;      ///< Maps IRC identifiers to buffers
@@ -1055,6 +1057,19 @@ buffer_remove (struct app_context *ctx, struct buffer *buffer)
 	}
 #endif // RL_READLINE_VERSION
 
+	// Get rid of the channel, as well as the users,
+	// if the buffer was the last thing keeping them alive
+	if (buffer->channel && buffer->channel->ref_count == 2)
+		str_map_set (&ctx->irc_channels, buffer->channel->name, NULL);
+	if (buffer->user && buffer->user->ref_count == 2)
+		str_map_set (&ctx->irc_users, buffer->user->nickname, NULL);
+
+	// And make sure to unlink the buffer from "irc_buffer_map"
+	if (buffer->channel)
+		str_map_set (&ctx->irc_buffer_map, buffer->channel->name, NULL);
+	if (buffer->user)
+		str_map_set (&ctx->irc_buffer_map, buffer->user->nickname, NULL);
+
 	str_map_set (&ctx->buffers_by_name, buffer->name, NULL);
 	LIST_UNLINK_WITH_TAIL (ctx->buffers, ctx->buffers_tail, buffer);
 	buffer_destroy (buffer);
@@ -1907,6 +1922,9 @@ irc_handle_nick (struct app_context *ctx, const struct irc_message *msg)
 		str_map_find (&ctx->irc_buffer_map, user->nickname);
 	if (pm_buffer)
 	{
+		str_map_set (&ctx->irc_buffer_map, new_nickname, user_ref (user));
+		str_map_set (&ctx->irc_buffer_map, user->nickname, NULL);
+
 		buffer_send (ctx, pm_buffer, BUFFER_LINE_NICK, 0,
 			msg->prefix, NULL, "%s", new_nickname);
 		// TODO: use a full weechat-style buffer name here