diff --git a/matrix/server.py b/matrix/server.py
index 1057ff7..94922a5 100644
--- a/matrix/server.py
+++ b/matrix/server.py
@@ -459,10 +459,14 @@ class MatrixServer(object):
         if self.socket:
             try:
                 self.socket.shutdown(socket.SHUT_RDWR)
-                self.socket.close()
             except socket.error:
                 pass
 
+            try:
+                self.socket.close()
+            except OSError:
+                pass
+
     def disconnect(self, reconnect=True):
         # type: (bool) -> None
         if self.fd_hook: