diff --git a/main.py b/main.py
index 3f5fb69..2c8be57 100644
--- a/main.py
+++ b/main.py
@@ -461,6 +461,35 @@ class WeechatHandler(StreamHandler):
         W.prnt(buf, item)
 
 
+def lazy_fetch_members_signal(_, _signal, buffer_ptr):
+    """ Fetch room members on a buffer switch signal.
+    This function is called every time we switch a buffer. The pointer of
+    the new buffer is given to us by weechat. If it is one of our room buffers
+    we check if the members for the room aren't fetched and fetch them now if
+    they aren't.
+    """
+    for server in SERVERS.values():
+        if buffer_ptr == server.server_buffer:
+            return W.WEECHAT_RC_OK
+
+        if buffer_ptr not in server.buffers.values():
+            continue
+
+        room_buffer = server.find_room_from_ptr(buffer_ptr)
+        if not room_buffer:
+            continue
+
+        if room_buffer.members_fetched:
+            return W.WEECHAT_RC_OK
+
+        room_id = room_buffer.room.room_id
+        server.get_joined_members(room_id)
+
+        break
+
+    return W.WEECHAT_RC_OK
+
+
 if __name__ == "__main__":
     if W.register(WEECHAT_SCRIPT_NAME, WEECHAT_SCRIPT_AUTHOR,
                   WEECHAT_SCRIPT_VERSION, WEECHAT_SCRIPT_LICENSE,
@@ -483,6 +512,8 @@ if __name__ == "__main__":
         init_bar_items()
         init_completion()
 
+        hook = W.hook_signal("buffer_switch", "lazy_fetch_members_signal", "")
+
         if not SERVERS:
             create_default_server(G.CONFIG)
 
diff --git a/matrix/buffer.py b/matrix/buffer.py
index b26c97f..83fb8b8 100644
--- a/matrix/buffer.py
+++ b/matrix/buffer.py
@@ -805,6 +805,7 @@ class RoomBuffer(object):
         self.prev_batch = prev_batch
         self.joined = True
         self.leave_event_id = None  # type: Optional[str]
+        self.members_fetched = False
         self.unhandled_users = []   # type: List[str]
 
         buffer_name = "{}.{}".format(server_name, room.room_id)
diff --git a/matrix/config.py b/matrix/config.py
index cc04dc7..570bab9 100644
--- a/matrix/config.py
+++ b/matrix/config.py
@@ -385,6 +385,17 @@ class MatrixConfig(WeechatConfig):
                 "off",
                 ("Use a separate buffer for debug logs."),
             ),
+            Option(
+                "lazy_load_room_users",
+                "boolean",
+                "",
+                0,
+                0,
+                "off",
+                ("If on, room users wont be loaded background proactively "
+                 "they will be loaded when the buffer becomes active. This "
+                 "only affects non encrypted rooms."),
+            ),
         ]
 
         color_options = [
diff --git a/matrix/server.py b/matrix/server.py
index a42afac..bbb8d52 100644
--- a/matrix/server.py
+++ b/matrix/server.py
@@ -252,6 +252,7 @@ class MatrixServer(object):
             # type: DefaultDict[str, Deque[EncrytpionQueueItem]]
         self.backlog_queue = dict()      # type: Dict[str, str]
 
+        self.member_request_list = []         # type: List[str]
         self.rooms_with_missing_members = []  # type: List[str]
         self.lazy_load_hook = None       # type: Optional[str]
         self.partial_sync_hook = None    # type: Optional[str]
@@ -483,6 +484,7 @@ class MatrixServer(object):
 
         self.send_buffer = b""
         self.transport_type = None
+        self.member_request_list.clear()
 
         if self.client:
             try:
@@ -733,6 +735,10 @@ class MatrixServer(object):
         self.send_or_queue(request)
 
     def get_joined_members(self, room_id):
+        if room_id in self.member_request_list:
+            return
+
+        self.member_request_list.append(room_id)
         _, request = self.client.joined_members(room_id)
         self.send(request)
 
@@ -928,10 +934,14 @@ class MatrixServer(object):
                 self.keys_query()
 
             for room_buffer in self.room_buffers.values():
+                # It's our initial sync, we need to fetch room members, so add
+                # the room to the missing members queue.
                 if not self.next_batch:
-                    self.rooms_with_missing_members.append(
-                        room_buffer.room.room_id
-                    )
+                    if (not G.CONFIG.network.lazy_load_room_users
+                            or room_buffer.room.encrypted):
+                        self.rooms_with_missing_members.append(
+                            room_buffer.room.room_id
+                        )
                 if room_buffer.unhandled_users:
                     self._hook_lazy_user_adding()
                     break
@@ -1023,12 +1033,14 @@ class MatrixServer(object):
             self.keys_queried = False
 
         elif isinstance(response, JoinedMembersResponse):
+            self.member_request_list.remove(response.room_id)
             room_buffer = self.room_buffers[response.room_id]
             users = [user.user_id for user in response.members]
 
             # Don't add the users directly use the lazy load hook.
             room_buffer.unhandled_users += users
             self._hook_lazy_user_adding()
+            room_buffer.members_fetched = True
 
             # Fetch the users for the next room.
             if self.rooms_with_missing_members: