diff --git a/matrix/config.py b/matrix/config.py
index d4d745b..efb6ddc 100644
--- a/matrix/config.py
+++ b/matrix/config.py
@@ -45,7 +45,8 @@ def matrix_config_change_cb(data, option):
         OPTIONS.look_server_buf = ServerBufferType(
             W.config_integer(option))
         for server in SERVERS.values():
-            server_buffer_merge(server.server_buffer)
+            if server.server_buffer:
+                server_buffer_merge(server.server_buffer)
 
     elif option_name == "max_initial_sync_events":
         OPTIONS.sync_limit = W.config_integer(option)
diff --git a/matrix/utils.py b/matrix/utils.py
index f057a5d..b43d9ae 100644
--- a/matrix/utils.py
+++ b/matrix/utils.py
@@ -84,10 +84,15 @@ def server_buffer_merge(buffer):
         W.buffer_merge(buffer, W.buffer_search_main())
     elif OPTIONS.look_server_buf == ServerBufferType.MERGE:
         if SERVERS:
-            first = list(SERVERS.values())[0].server_buffer
-            num = W.buffer_get_integer(W.buffer_search_main(), "number")
-            W.buffer_unmerge(buffer, num + 1)
-            W.buffer_merge(buffer, first)
+            first = None
+            for server in SERVERS.values():
+                if server.server_buffer and buffer is not server.server_buffer:
+                    first = server.server_buffer
+                    break
+            if first:
+                num = W.buffer_get_integer(W.buffer_search_main(), "number")
+                W.buffer_unmerge(buffer, num + 1)
+                W.buffer_merge(buffer, first)
     else:
         num = W.buffer_get_integer(W.buffer_search_main(), "number")
         W.buffer_unmerge(buffer, num + 1)