From 45e6f742ece597231417d03e7d46a034401a667c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sun, 4 Mar 2018 17:19:10 +0100 Subject: [PATCH 001/269] Fix device id loading. --- matrix/server.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index ab67015..668f797 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -94,7 +94,6 @@ class MatrixServer: self._create_options(config_file) self._create_session_dir() - self._load_devide_id() # yapf: enable def _create_session_dir(self): @@ -104,13 +103,13 @@ class MatrixServer: "directory").format(prefix=W.prefix("error")) W.prnt("", message) - def _get_session_path(self): + def get_session_path(self): home_dir = W.info_get('weechat_dir', '') return os.path.join(home_dir, "matrix", self.name) - def _load_devide_id(self): - file_name = "{}{}".format(self.name, ".device_id") - path = os.path.join(self._get_session_path(), file_name) + def _load_device_id(self): + file_name = "{}{}".format(self.user, ".device_id") + path = os.path.join(self.get_session_path(), file_name) if not os.path.isfile(path): return @@ -121,8 +120,8 @@ class MatrixServer: self.device_id = device_id def save_device_id(self): - file_name = "{}{}".format(self.name, ".device_id") - path = os.path.join(self._get_session_path(), file_name) + file_name = "{}{}".format(self.user, ".device_id") + path = os.path.join(self.get_session_path(), file_name) with open(path, 'w') as f: f.write(self.device_id) @@ -198,6 +197,9 @@ class MatrixServer: value = W.config_string(option) self.user = value self.access_token = "" + + self._load_device_id() + elif option_name == "password": value = W.config_string(option) self.password = W.string_eval_expression(value, {}, {}, {}) From c52b7866c81b6500577440a8cd0a8b92984360f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Mon, 5 Mar 2018 19:34:51 +0100 Subject: [PATCH 002/269] Add olm account loading/creation and a initial olm command. --- main.py | 1 + matrix/commands.py | 3 + matrix/encryption.py | 176 +++++++++++++++++++++++++++++++++++++++++++ matrix/globals.py | 1 + matrix/server.py | 29 +++++++ 5 files changed, 210 insertions(+) create mode 100644 matrix/encryption.py diff --git a/main.py b/main.py index d14ed1b..eab1e41 100644 --- a/main.py +++ b/main.py @@ -36,6 +36,7 @@ from matrix.colors import Formatted from matrix.utf import utf8_decode from matrix.http import HttpResponse from matrix.api import MatrixSendMessage +from matrix.encryption import matrix_olm_command_cb # Weechat searches for the registered callbacks in the scope of the main script # file, import the callbacks here so weechat can find them. diff --git a/matrix/commands.py b/matrix/commands.py index dd7a440..958f180 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -32,6 +32,7 @@ from matrix.utils import key_from_value, tags_from_line_data from matrix.plugin_options import DebugType from matrix.server import MatrixServer from matrix.colors import Formatted +from matrix.encryption import matrix_hook_olm_command def hook_commands(): @@ -98,6 +99,8 @@ def hook_commands(): "matrix_me_command_cb", "") + matrix_hook_olm_command() + W.hook_command_run('/topic', 'matrix_command_topic_cb', '') W.hook_command_run('/buffer clear', 'matrix_command_buf_clear_cb', '') W.hook_command_run('/join', 'matrix_command_join_cb', '') diff --git a/matrix/encryption.py b/matrix/encryption.py new file mode 100644 index 0000000..9291b5f --- /dev/null +++ b/matrix/encryption.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- + +# Weechat Matrix Protocol Script +# Copyright © 2018 Damir Jelić +# +# Permission to use, copy, modify, and/or distribute this software for +# any purpose with or without fee is hereby granted, provided that the +# above copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import unicode_literals + +import os + +# pylint: disable=redefined-builtin +from builtins import str + +from functools import wraps +from future.moves.itertools import zip_longest + +import matrix.globals + +try: + from olm.account import Account, OlmAccountError +except ImportError: + matrix.globals.ENCRYPTION = False + +from matrix.globals import W, SERVERS +from matrix.utf import utf8_decode + + +def own_buffer(f): + + @wraps(f) + def wrapper(data, buffer, *args, **kwargs): + + for server in SERVERS.values(): + if buffer in server.buffers.values(): + return f(server.name, buffer, *args, **kwargs) + elif buffer == server.server_buffer: + return f(server.name, buffer, *args, **kwargs) + + return W.WEECHAT_RC_OK + + return wrapper + + +def encrypt_enabled(f): + + @wraps(f) + def wrapper(*args, **kwds): + if matrix.globals.ENCRYPTION: + return f(*args, **kwds) + return None + + return wrapper + + +@encrypt_enabled +def matrix_hook_olm_command(): + W.hook_command( + # Command name and short description + "olm", + "Matrix olm encryption command", + # Synopsis + ("info all|blacklisted|private|unverified|verified ||" + "blacklist ||" + "unverify ||" + "verify "), + # Description + (" info: show info about known devices and their keys\n" + "blacklist: blacklist a device\n" + " unverify: unverify a device\n" + " verify: verify a device\n\n" + "Examples:\n"), + # Completions + ('info all|blacklisted|private|unverified|verified ||' + 'blacklist %(device_ids) ||' + 'unverify %(device_ids) ||' + 'verify %(device_ids)'), + # Function name + 'matrix_olm_command_cb', + '') + + +def olm_cmd_parse_args(args): + split_args = args.split() + + command = split_args.pop(0) if split_args else "info" + + rest_args = split_args if split_args else [] + + return command, rest_args + + +def grouper(iterable, n, fillvalue=None): + "Collect data into fixed-length chunks or blocks" + # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" + args = [iter(iterable)] * n + return zip_longest(*args, fillvalue=fillvalue) + + +def partition_key(key): + groups = grouper(key, 4, " ") + return ' '.join(''.join(g) for g in groups) + + +@own_buffer +@utf8_decode +def matrix_olm_command_cb(server_name, buffer, args): + server = SERVERS[server_name] + command, args = olm_cmd_parse_args(args) + + if not command or command == "info": + olm = server.olm + device_msg = (" - Device ID: {}\n".format(server.device_id) + if server.device_id else "") + id_key = partition_key(olm.account.identity_keys()["curve25519"]) + fp_key = partition_key(olm.account.identity_keys()["ed25519"]) + message = ("{prefix}matrix: Identity keys:\n" + " - User: {user}\n" + "{device_msg}" + " - Identity key: {id_key}\n" + " - Fingerprint key: {fp_key}\n").format( + prefix=W.prefix("network"), + user=server.user, + device_msg=device_msg, + id_key=id_key, + fp_key=fp_key) + W.prnt(server.server_buffer, message) + else: + message = ("{prefix}matrix: Command not implemented.".format( + prefix=W.prefix("error"))) + W.prnt(server.server_buffer, message) + + return W.WEECHAT_RC_OK + + +class EncryptionError(Exception): + pass + + +class Olm(): + + @encrypt_enabled + def __init__(self, server, account=None): + # type: (Server, Account) -> None + self.server = server + if account: + self.account = account + else: + self.account = Account() + + @classmethod + @encrypt_enabled + def from_session_dir(cls, server): + # type: (Server) -> Olm + account_file_name = "{}_{}.account".format(server.user, + server.device_id) + session_path = server.get_session_path() + path = os.path.join(session_path, account_file_name) + + try: + with open(path, "rb") as f: + pickle = f.read() + account = Account.from_pickle(pickle) + return cls(server, account) + except OlmAccountError as error: + raise EncryptionError(error) diff --git a/matrix/globals.py b/matrix/globals.py index 9c125b7..e7f7a29 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -31,3 +31,4 @@ except ImportError: OPTIONS = PluginOptions() # type: PluginOptions SERVERS = dict() # type: Dict[str, MatrixServer] CONFIG = None # type: weechat.config +ENCRYPTION = True # type: bool diff --git a/matrix/server.py b/matrix/server.py index 668f797..4902ab5 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -35,6 +35,13 @@ from matrix.globals import W, SERVERS, OPTIONS import matrix.api as API from matrix.api import MatrixClient, MatrixSyncMessage, MatrixLoginMessage +from matrix.encryption import Olm, EncryptionError + +try: + FileNotFoundError +except NameError: + FileNotFoundError = IOError + class MatrixServer: # pylint: disable=too-many-instance-attributes @@ -48,6 +55,7 @@ class MatrixServer: self.options = dict() # type: Dict[str, weechat.config] self.device_name = "Weechat Matrix" # type: str self.device_id = "" # type: str + self.olm = None # type: Olm self.user = "" # type: str self.password = "" # type: str @@ -126,6 +134,24 @@ class MatrixServer: with open(path, 'w') as f: f.write(self.device_id) + def _load_olm(self): + try: + self.olm = Olm.from_session_dir(self) + except FileNotFoundError: + message = ("{prefix}matrix: Creating new Olm identity for {user}" + " on {server} for device {device}.").format( + prefix=W.prefix("network"), + user=self.user, + server=self.name, + device=self.device_id) + W.prnt("", message) + self.olm = Olm(self) + except EncryptionError as error: + message = ("{prefix}matrix: Error loading Olm" + "account: {error}.").format( + prefix=W.prefix("error"), error=error) + W.prnt("", message) + def _create_options(self, config_file): options = [ Option('autoconnect', 'boolean', '', 0, 0, 'off', @@ -200,6 +226,9 @@ class MatrixServer: self._load_device_id() + if self.device_id: + self._load_olm() + elif option_name == "password": value = W.config_string(option) self.password = W.string_eval_expression(value, {}, {}, {}) From b309be8a98c30e122f99d372480700983e0af8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Tue, 6 Mar 2018 13:47:51 +0100 Subject: [PATCH 003/269] olm: Don't store the server in the olm class. --- matrix/encryption.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 9291b5f..d180d27 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -150,9 +150,8 @@ class EncryptionError(Exception): class Olm(): @encrypt_enabled - def __init__(self, server, account=None): + def __init__(self, account=None): # type: (Server, Account) -> None - self.server = server if account: self.account = account else: From 357c76edccaca8e0d5f9287add621042ba60eac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 22 Mar 2018 11:32:54 +0100 Subject: [PATCH 004/269] olm: Create a new olm account only after login. That way we can be sure to have a valid device ID. --- matrix/events.py | 4 ++++ matrix/server.py | 24 ++++++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/matrix/events.py b/matrix/events.py index b70141d..8f5df81 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -86,6 +86,10 @@ class MatrixLoginEvent(MatrixEvent): prefix=W.prefix("network"), user=self.user_id) W.prnt(self.server.server_buffer, message) + + if not self.server.olm: + self.server.create_olm() + self.server.sync() @classmethod diff --git a/matrix/server.py b/matrix/server.py index 4902ab5..0976e8c 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -138,20 +138,28 @@ class MatrixServer: try: self.olm = Olm.from_session_dir(self) except FileNotFoundError: - message = ("{prefix}matrix: Creating new Olm identity for {user}" - " on {server} for device {device}.").format( - prefix=W.prefix("network"), - user=self.user, - server=self.name, - device=self.device_id) - W.prnt("", message) - self.olm = Olm(self) + pass except EncryptionError as error: message = ("{prefix}matrix: Error loading Olm" "account: {error}.").format( prefix=W.prefix("error"), error=error) W.prnt("", message) + def create_olm(self): + message = ("{prefix}matrix: Creating new Olm identity for " + "{self_color}{user}{ncolor}" + " on {server_color}{server}{ncolor} for device " + "{device}.").format( + prefix=W.prefix("network"), + self_color=W.color("chat_nick_self"), + ncolor=W.color("reset"), + user=self.user_id, + server_color=W.color("chat_server"), + server=self.name, + device=self.device_id) + W.prnt(self.server_buffer, message) + self.olm = Olm(self) + def _create_options(self, config_file): options = [ Option('autoconnect', 'boolean', '', 0, 0, 'off', From 823cc1bf2838c39daf4360cda8e2e153479f5db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 22 Mar 2018 12:07:22 +0100 Subject: [PATCH 005/269] olm: Save and restore the olm account keys. --- matrix/encryption.py | 17 ++++++++++++++++- matrix/events.py | 1 + matrix/server.py | 5 ++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index d180d27..9f1fecb 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -170,6 +170,21 @@ class Olm(): with open(path, "rb") as f: pickle = f.read() account = Account.from_pickle(pickle) - return cls(server, account) + return cls(account) + except OlmAccountError as error: + raise EncryptionError(error) + + @encrypt_enabled + def to_session_dir(self, server): + # type: (Server) -> None + account_file_name = "{}_{}.account".format(server.user, + server.device_id) + session_path = server.get_session_path() + path = os.path.join(session_path, account_file_name) + + try: + with open(path, "wb") as f: + pickle = self.account.pickle() + f.write(pickle) except OlmAccountError as error: raise EncryptionError(error) diff --git a/matrix/events.py b/matrix/events.py index 8f5df81..b87436b 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -89,6 +89,7 @@ class MatrixLoginEvent(MatrixEvent): if not self.server.olm: self.server.create_olm() + self.server.store_olm() self.server.sync() diff --git a/matrix/server.py b/matrix/server.py index 0976e8c..fcac056 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -158,7 +158,10 @@ class MatrixServer: server=self.name, device=self.device_id) W.prnt(self.server_buffer, message) - self.olm = Olm(self) + self.olm = Olm() + + def store_olm(self): + self.olm.to_session_dir(self) def _create_options(self, config_file): options = [ From 27b376e1f5a806ce6ca76f81059d5022eea65632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 22 Mar 2018 21:02:31 +0100 Subject: [PATCH 006/269] olm: Upload the identity keys. --- matrix/api.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ matrix/events.py | 21 ++++++++++++++++ matrix/server.py | 16 +++++++++++- 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/matrix/api.py b/matrix/api.py index f9e4a83..221d6b5 100644 --- a/matrix/api.py +++ b/matrix/api.py @@ -224,6 +224,49 @@ class MatrixClient: h = HttpRequest(RequestType.POST, self.host, path, content) return h + def keys_upload(self, user_id, device_id, account, keys=None, + one_time_keys=None): + query_parameters = {"access_token": self.access_token} + + path = ("{api}/keys/upload?" + "{query_parameters}").format( + api=MATRIX_API_PATH, + query_parameters=urlencode(query_parameters)) + + content = {} + + # TODO one time keys + if keys: + device_keys = { + "algorithms": [ + "m.olm.v1.curve25519-aes-sha2", + "m.megolm.v1.aes-sha2" + ], + "device_id": device_id, + "user_id": user_id, + "keys": { + "curve25519:" + device_id: keys["curve25519"], + "ed25519:" + device_id: keys["ed25519"] + } + } + + signature = account.sign(json.dumps( + device_keys, + ensure_ascii=False, + separators=(',', ':'), + sort_keys=True, + )) + + device_keys["signatures"] = { + user_id: { + "ed25519:" + device_id: signature + } + } + + content["device_keys"] = device_keys + + return HttpRequest(RequestType.POST, self.host, path, content) + def mxc_to_http(self, mxc): # type: (str) -> str url = urlparse(mxc) @@ -510,3 +553,24 @@ class MatrixKickMessage(MatrixMessage): self.reason) return self._decode(server, object_hook) + + +class MatrixKeyUploadMessage(MatrixMessage): + + def __init__(self, client, user_id, device_id, account, keys=None, + one_time_keys=None): + data = { + "device_id": device_id, + "user_id": user_id, + "account": account, + "keys": keys, + "one_time_keys": one_time_keys + } + + MatrixMessage.__init__(self, client.keys_upload, data) + + def decode_body(self, server): + object_hook = partial(MatrixEvents.MatrixKeyUploadEvent.from_dict, + server) + + return self._decode(server, object_hook) diff --git a/matrix/events.py b/matrix/events.py index b87436b..7d5f495 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -67,6 +67,26 @@ class MatrixErrorEvent(MatrixEvent): fatal=fatal) +class MatrixKeyUploadEvent(MatrixEvent): + + def __init__(self, server): + MatrixEvent.__init__(self, server) + + def execute(self): + message = "{prefix}matrix: Uploaded olm device keys.".format( + prefix=W.prefix("network")) + + W.prnt(self.server.server_buffer, message) + + @classmethod + def from_dict(cls, server, parsed_dict): + try: + return cls(server) + except (KeyError, TypeError, ValueError): + return MatrixErrorEvent.from_dict(server, "Error uploading device" + "keys", False, parsed_dict) + + class MatrixLoginEvent(MatrixEvent): def __init__(self, server, user_id, device_id, access_token): @@ -90,6 +110,7 @@ class MatrixLoginEvent(MatrixEvent): if not self.server.olm: self.server.create_olm() self.server.store_olm() + self.server.upload_keys(device_keys=True, one_time_keys=False) self.server.sync() diff --git a/matrix/server.py b/matrix/server.py index fcac056..e9f0cc5 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -33,7 +33,12 @@ from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, from matrix.utf import utf8_decode from matrix.globals import W, SERVERS, OPTIONS import matrix.api as API -from matrix.api import MatrixClient, MatrixSyncMessage, MatrixLoginMessage +from matrix.api import ( + MatrixClient, + MatrixSyncMessage, + MatrixLoginMessage, + MatrixKeyUploadMessage +) from matrix.encryption import Olm, EncryptionError @@ -456,6 +461,15 @@ class MatrixServer: message = MatrixSyncMessage(self.client, self.next_batch, limit) self.send_queue.append(message) + def upload_keys(self, device_keys=False, one_time_keys=False): + keys = self.olm.account.identity_keys() if device_keys else None + + # TODO generate one time keys and upload them as well + message = MatrixKeyUploadMessage(self.client, self.user_id, + self.device_id, self.olm.account, + keys, None) + self.send_queue.append(message) + def login(self): # type: (MatrixServer) -> None message = MatrixLoginMessage(self.client, self.user, self.password, From 36cd12225b05016423b2b51e3a5b00e86b681597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 23 Mar 2018 18:43:56 +0100 Subject: [PATCH 007/269] olm: One time key creation/uploading. --- matrix/api.py | 36 +++++++++++++++++++++++++----------- matrix/encryption.py | 11 +++++++++++ matrix/events.py | 28 ++++++++++++++++++++++------ matrix/server.py | 22 ++++++++++++++++++---- 4 files changed, 76 insertions(+), 21 deletions(-) diff --git a/matrix/api.py b/matrix/api.py index 221d6b5..ae4539f 100644 --- a/matrix/api.py +++ b/matrix/api.py @@ -224,7 +224,7 @@ class MatrixClient: h = HttpRequest(RequestType.POST, self.host, path, content) return h - def keys_upload(self, user_id, device_id, account, keys=None, + def keys_upload(self, user_id, device_id, olm, keys=None, one_time_keys=None): query_parameters = {"access_token": self.access_token} @@ -235,7 +235,6 @@ class MatrixClient: content = {} - # TODO one time keys if keys: device_keys = { "algorithms": [ @@ -250,12 +249,7 @@ class MatrixClient: } } - signature = account.sign(json.dumps( - device_keys, - ensure_ascii=False, - separators=(',', ':'), - sort_keys=True, - )) + signature = olm.sign_json(device_keys) device_keys["signatures"] = { user_id: { @@ -265,6 +259,24 @@ class MatrixClient: content["device_keys"] = device_keys + if one_time_keys: + one_time_key_dict = {} + + for key_id, key in one_time_keys.items(): + key_dict = {"key": key} + signature = olm.sign_json(key_dict) + + one_time_key_dict["signed_curve25519:" + key_id] = { + "key": key_dict.pop("key"), + "signatures": { + user_id: { + "ed25519:" + device_id: signature + } + } + } + + content["one_time_keys"] = one_time_key_dict + return HttpRequest(RequestType.POST, self.host, path, content) def mxc_to_http(self, mxc): @@ -557,20 +569,22 @@ class MatrixKickMessage(MatrixMessage): class MatrixKeyUploadMessage(MatrixMessage): - def __init__(self, client, user_id, device_id, account, keys=None, + def __init__(self, client, user_id, device_id, olm, keys=None, one_time_keys=None): data = { "device_id": device_id, "user_id": user_id, - "account": account, + "olm": olm, "keys": keys, "one_time_keys": one_time_keys } + self.device_keys = True if keys else False + MatrixMessage.__init__(self, client.keys_upload, data) def decode_body(self, server): object_hook = partial(MatrixEvents.MatrixKeyUploadEvent.from_dict, - server) + server, self.device_keys) return self._decode(server, object_hook) diff --git a/matrix/encryption.py b/matrix/encryption.py index 9f1fecb..aced6ac 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -18,6 +18,7 @@ from __future__ import unicode_literals import os +import json # pylint: disable=redefined-builtin from builtins import str @@ -188,3 +189,13 @@ class Olm(): f.write(pickle) except OlmAccountError as error: raise EncryptionError(error) + + def sign_json(self, json_dict): + signature = self.account.sign(json.dumps( + json_dict, + ensure_ascii=False, + separators=(',', ':'), + sort_keys=True, + )) + + return signature diff --git a/matrix/events.py b/matrix/events.py index 7d5f495..fb6932c 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -69,19 +69,23 @@ class MatrixErrorEvent(MatrixEvent): class MatrixKeyUploadEvent(MatrixEvent): - def __init__(self, server): + def __init__(self, server, device_keys): + self.device_keys = device_keys MatrixEvent.__init__(self, server) def execute(self): - message = "{prefix}matrix: Uploaded olm device keys.".format( + if not self.device_keys: + return + + message = "{prefix}matrix: Uploaded Olm device keys.".format( prefix=W.prefix("network")) W.prnt(self.server.server_buffer, message) @classmethod - def from_dict(cls, server, parsed_dict): + def from_dict(cls, server, device_keys, parsed_dict): try: - return cls(server) + return cls(server, device_keys) except (KeyError, TypeError, ValueError): return MatrixErrorEvent.from_dict(server, "Error uploading device" "keys", False, parsed_dict) @@ -415,10 +419,12 @@ class MatrixBacklogEvent(MatrixEvent): class MatrixSyncEvent(MatrixEvent): - def __init__(self, server, next_batch, room_infos, invited_infos): + def __init__(self, server, next_batch, room_infos, invited_infos, + one_time_key_count): self.next_batch = next_batch self.joined_room_infos = room_infos self.invited_room_infos = invited_infos + self.one_time_key_count = one_time_key_count MatrixEvent.__init__(self, server) @@ -439,12 +445,21 @@ class MatrixSyncEvent(MatrixEvent): def from_dict(cls, server, parsed_dict): try: next_batch = sanitize_id(parsed_dict["next_batch"]) + one_time_key_count = 0 + + if "device_one_time_keys_count" in parsed_dict: + if ("signed_curve25519" in + parsed_dict["device_one_time_keys_count"]): + one_time_key_count = ( + parsed_dict["device_one_time_keys_count"]["signed_curve25519"]) + room_info_dict = parsed_dict["rooms"] join_infos, invite_infos = MatrixSyncEvent._infos_from_dict( room_info_dict) - return cls(server, next_batch, join_infos, invite_infos) + return cls(server, next_batch, join_infos, invite_infos, + one_time_key_count) except (KeyError, ValueError, TypeError): return MatrixErrorEvent.from_dict(server, "Error syncing", False, parsed_dict) @@ -475,5 +490,6 @@ class MatrixSyncEvent(MatrixEvent): self._queue_joined_info() server.next_batch = self.next_batch + server.check_one_time_keys(self.one_time_key_count) server.handle_events() diff --git a/matrix/server.py b/matrix/server.py index e9f0cc5..4d03480 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -40,7 +40,7 @@ from matrix.api import ( MatrixKeyUploadMessage ) -from matrix.encryption import Olm, EncryptionError +from matrix.encryption import Olm, EncryptionError, encrypt_enabled try: FileNotFoundError @@ -464,12 +464,26 @@ class MatrixServer: def upload_keys(self, device_keys=False, one_time_keys=False): keys = self.olm.account.identity_keys() if device_keys else None - # TODO generate one time keys and upload them as well + one_time_keys = (self.olm.account.one_time_keys()["curve25519"] if + one_time_keys else None) + message = MatrixKeyUploadMessage(self.client, self.user_id, - self.device_id, self.olm.account, - keys, None) + self.device_id, self.olm, + keys, one_time_keys) self.send_queue.append(message) + @encrypt_enabled + def check_one_time_keys(self, key_count): + max_keys = self.olm.account.max_one_time_keys() + + key_count = (max_keys / 2) - key_count + + if key_count <= 0: + return + + self.olm.account.generate_one_time_keys(key_count) + self.upload_keys(device_keys=False, one_time_keys=True) + def login(self): # type: (MatrixServer) -> None message = MatrixLoginMessage(self.client, self.user, self.password, From d28e6eb598849cd4baec799166d08ef0f732a58b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sat, 24 Mar 2018 20:14:46 +0100 Subject: [PATCH 008/269] encryption: Add encrypt_enabled decorator to some functions. --- matrix/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/matrix/server.py b/matrix/server.py index 4d03480..eadf9b5 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -150,6 +150,7 @@ class MatrixServer: prefix=W.prefix("error"), error=error) W.prnt("", message) + @encrypt_enabled def create_olm(self): message = ("{prefix}matrix: Creating new Olm identity for " "{self_color}{user}{ncolor}" @@ -165,6 +166,7 @@ class MatrixServer: W.prnt(self.server_buffer, message) self.olm = Olm() + @encrypt_enabled def store_olm(self): self.olm.to_session_dir(self) @@ -461,6 +463,7 @@ class MatrixServer: message = MatrixSyncMessage(self.client, self.next_batch, limit) self.send_queue.append(message) + @encrypt_enabled def upload_keys(self, device_keys=False, one_time_keys=False): keys = self.olm.account.identity_keys() if device_keys else None From f8c1f564ae484b3a994b389188622bbe5e22861e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sun, 25 Mar 2018 22:01:34 +0200 Subject: [PATCH 009/269] olm: Mark one time keys as published and store the account. --- main.py | 4 ++++ matrix/encryption.py | 4 ++++ matrix/events.py | 3 +++ 3 files changed, 11 insertions(+) diff --git a/main.py b/main.py index eab1e41..4e18f7b 100644 --- a/main.py +++ b/main.py @@ -432,6 +432,10 @@ def room_close_cb(data, buffer): @utf8_decode def matrix_unload_cb(): matrix_config_free(matrix.globals.CONFIG) + + for server in SERVERS.values(): + server.store_olm() + W.prnt("", "unloading") return W.WEECHAT_RC_OK diff --git a/matrix/encryption.py b/matrix/encryption.py index aced6ac..23e4156 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -199,3 +199,7 @@ class Olm(): )) return signature + + @encrypt_enabled + def mark_keys_as_published(self): + self.account.mark_keys_as_published() diff --git a/matrix/events.py b/matrix/events.py index fb6932c..8cfdf06 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -74,6 +74,9 @@ class MatrixKeyUploadEvent(MatrixEvent): MatrixEvent.__init__(self, server) def execute(self): + self.server.olm.mark_keys_as_published() + self.server.store_olm() + if not self.device_keys: return From ec2995fe52e05feeb622c5aab716516a70f9c68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 29 Mar 2018 11:41:01 +0200 Subject: [PATCH 010/269] encryption: Add inbound session creation. --- matrix/encryption.py | 50 ++++++++++++++++++++++++++-- matrix/events.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 23e4156..6af199b 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -23,6 +23,7 @@ import json # pylint: disable=redefined-builtin from builtins import str +from collections import defaultdict from functools import wraps from future.moves.itertools import zip_longest @@ -30,6 +31,9 @@ import matrix.globals try: from olm.account import Account, OlmAccountError + from olm.session import (InboundSession, OlmSessionError, OlmMessage, + OlmPreKeyMessage) + from olm.group_session import InboundGroupSession, OlmGroupSessionError except ImportError: matrix.globals.ENCRYPTION = False @@ -151,13 +155,55 @@ class EncryptionError(Exception): class Olm(): @encrypt_enabled - def __init__(self, account=None): - # type: (Server, Account) -> None + def __init__( + self, + account=None, + sessions=defaultdict(list), + group_sessions=defaultdict(dict) + ): + # type: (Account, Dict[str, List[Session]) -> None if account: self.account = account else: self.account = Account() + self.sessions = sessions + self.group_sessions = group_sessions + + def _create_session(self, sender, sender_key, message): + session = InboundSession(self.account, message, sender_key) + self.sessions[sender].append(session) + self.account.remove_one_time_keys(session) + + return session + + def create_group_session(self, room_id, session_id, session_key): + session = InboundGroupSession(session_key) + self.group_sessions[room_id][session_id] = session + + @encrypt_enabled + def decrypt(self, sender, sender_key, message): + plaintext = None + + for session in self.sessions[sender]: + try: + if isinstance(message, OlmPreKeyMessage): + if not session.matches(message): + continue + + plaintext = session.decrypt(message) + break + except OlmSessionError: + pass + + session = self._create_session(sender, sender_key, message) + + try: + plaintext = session.decrypt(message) + return plaintext + except OlmSessionError: + return None + @classmethod @encrypt_enabled def from_session_dir(cls, server): diff --git a/matrix/events.py b/matrix/events.py index 8cfdf06..10fb86e 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -28,6 +28,11 @@ from matrix.rooms import (matrix_create_room_buffer, RoomInfo, RoomMessageText, RoomMessageEvent, RoomRedactedMessageEvent, RoomMessageEmote) +try: + from olm.session import OlmMessage, OlmPreKeyMessage +except ImportError: + pass + class MatrixEvent(): @@ -444,6 +449,75 @@ class MatrixSyncEvent(MatrixEvent): return (join_infos, invite_infos) + @staticmethod + def _get_olm_device_event(server, parsed_dict): + device_key = server.olm.account.device_keys["curve25519"] + + if device_key not in parsed_dict["content"]["ciphertext"]: + return None + + ciphertext = parsed_dict["content"]["ciphertext"].pop(device_key) + sender = sanitize_id(parsed_dict["sender"]) + sender_key = sanitize_id(parsed_dict["sender_key"]) + + message = None + + if ciphertext["type"] == 0: + message = OlmPreKeyMessage(parsed_dict["body"]) + elif ciphertext["type"] == 1: + message = OlmMessage(parsed_dict["body"]) + else: + raise ValueError("Invalid Olm message type") + + olm = server.olm + plaintext = olm.decrypt(sender, sender_key, message) + + # TODO check sender key + decrypted_sender = sanitize_id(plaintext["sender"]) + decrypted_recepient = sanitize_id(plaintext["recipient"]) + decrypted_recepient_key = sanitize_id( + plaintext["recipient_keys"]["ed25519"]) + + if (sender != decrypted_sender or + server.user_id != decrypted_recepient or + device_key != decrypted_recepient_key): + error_message = ("{prefix}matrix: Mismatch in decrypted Olm" + "message").format(W.prefix("error")) + W.prnt("", error_message) + return None + + if plaintext["type"] != "m.room.key": + return None + + MatrixSyncEvent._handle_key_event(server, sender_key, plaintext) + + @staticmethod + def _handle_key_event(server, sender_key, parsed_dict): + # type: (MatrixServer, str, Dict[Any, Any] -> None + olm = server.olm + content = parsed_dict.pop("content") + + if content["type"] != "m.megolm.v1.aes-sha2": + return + + room_id = sanitize_id(content["room_id"]) + session_id = sanitize_id(content["session_id"]) + session_key = sanitize_id(content["session_key"]) + + if session_id in olm.group_sessions[room_id]: + return + + olm.create_group_session(room_id, session_id, session_key) + + @staticmethod + def _get_to_device_events(server, parsed_dict): + # type: (MatrixServer, Dict[Any, Any]) -> None + for event in parsed_dict["events"]: + if event["type"] == "m.room.encrypted": + if (event["content"]["algorithm"] == + 'm.olm.v1.curve25519-aes-sha2'): + MatrixSyncEvent._get_olm_device_event(server, parsed_dict) + @classmethod def from_dict(cls, server, parsed_dict): try: @@ -456,6 +530,9 @@ class MatrixSyncEvent(MatrixEvent): one_time_key_count = ( parsed_dict["device_one_time_keys_count"]["signed_curve25519"]) + MatrixSyncEvent._get_to_device_events( + server.olm, parsed_dict.pop("to_device")) + room_info_dict = parsed_dict["rooms"] join_infos, invite_infos = MatrixSyncEvent._infos_from_dict( From 1fd5bd637d14d81414bdc016d295e01ddc25139f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Tue, 3 Apr 2018 21:51:03 +0200 Subject: [PATCH 011/269] encryption: Decrypt incomming messages. --- matrix/encryption.py | 20 +++++++++++++++++- matrix/events.py | 50 ++++++++++++++++++++++++-------------------- matrix/rooms.py | 33 +++++++++++++++++++++++------ 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 6af199b..dd077fa 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -171,15 +171,20 @@ class Olm(): self.group_sessions = group_sessions def _create_session(self, sender, sender_key, message): + W.prnt("", "matrix: Creating session for {}".format(sender)) session = InboundSession(self.account, message, sender_key) + W.prnt("", "matrix: Created session for {}".format(sender)) self.sessions[sender].append(session) - self.account.remove_one_time_keys(session) + # self.account.remove_one_time_keys(session) + # TODO store account here return session def create_group_session(self, room_id, session_id, session_key): + W.prnt("", "matrix: Creating group session for {}".format(room_id)) session = InboundGroupSession(session_key) self.group_sessions[room_id][session_id] = session + # TODO store account here @encrypt_enabled def decrypt(self, sender, sender_key, message): @@ -204,6 +209,19 @@ class Olm(): except OlmSessionError: return None + @encrypt_enabled + def group_decrypt(self, room_id, session_id, ciphertext): + if session_id not in self.group_sessions[room_id]: + return None + + session = self.group_sessions[room_id][session_id] + try: + plaintext = session.decrypt(ciphertext) + except OlmGroupSessionError: + return None + + return plaintext + @classmethod @encrypt_enabled def from_session_dir(cls, server): diff --git a/matrix/events.py b/matrix/events.py index 10fb86e..ad576a7 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -17,6 +17,8 @@ from __future__ import unicode_literals from builtins import str +import json + from collections import deque from functools import partial from operator import itemgetter @@ -437,7 +439,7 @@ class MatrixSyncEvent(MatrixEvent): MatrixEvent.__init__(self, server) @staticmethod - def _infos_from_dict(parsed_dict): + def _infos_from_dict(olm, parsed_dict): join_infos = [] invite_infos = [] @@ -445,27 +447,28 @@ class MatrixSyncEvent(MatrixEvent): if not room_id: continue - join_infos.append(RoomInfo.from_dict(room_id, room_dict)) + join_infos.append(RoomInfo.from_dict(olm, room_id, room_dict)) return (join_infos, invite_infos) @staticmethod def _get_olm_device_event(server, parsed_dict): - device_key = server.olm.account.device_keys["curve25519"] + device_key = server.olm.account.identity_keys()["curve25519"] if device_key not in parsed_dict["content"]["ciphertext"]: return None - ciphertext = parsed_dict["content"]["ciphertext"].pop(device_key) sender = sanitize_id(parsed_dict["sender"]) - sender_key = sanitize_id(parsed_dict["sender_key"]) + sender_key = sanitize_id(parsed_dict["content"]["sender_key"]) + + ciphertext = parsed_dict["content"]["ciphertext"].pop(device_key) message = None if ciphertext["type"] == 0: - message = OlmPreKeyMessage(parsed_dict["body"]) + message = OlmPreKeyMessage(ciphertext["body"]) elif ciphertext["type"] == 1: - message = OlmMessage(parsed_dict["body"]) + message = OlmMessage(ciphertext["body"]) else: raise ValueError("Invalid Olm message type") @@ -473,23 +476,24 @@ class MatrixSyncEvent(MatrixEvent): plaintext = olm.decrypt(sender, sender_key, message) # TODO check sender key - decrypted_sender = sanitize_id(plaintext["sender"]) - decrypted_recepient = sanitize_id(plaintext["recipient"]) - decrypted_recepient_key = sanitize_id( - plaintext["recipient_keys"]["ed25519"]) + parsed_plaintext = json.loads(plaintext, encoding='utf-8') + decrypted_sender = parsed_plaintext["sender"] + decrypted_recepient = parsed_plaintext["recipient"] + decrypted_recepient_key = parsed_plaintext["recipient_keys"]["ed25519"] if (sender != decrypted_sender or server.user_id != decrypted_recepient or - device_key != decrypted_recepient_key): - error_message = ("{prefix}matrix: Mismatch in decrypted Olm" - "message").format(W.prefix("error")) + olm.account.identity_keys()["ed25519"] != + decrypted_recepient_key): + error_message = ("{prefix}matrix: Mismatch in decrypted Olm " + "message").format(prefix=W.prefix("error")) W.prnt("", error_message) return None - if plaintext["type"] != "m.room.key": + if parsed_plaintext["type"] != "m.room_key": return None - MatrixSyncEvent._handle_key_event(server, sender_key, plaintext) + MatrixSyncEvent._handle_key_event(server, sender_key, parsed_plaintext) @staticmethod def _handle_key_event(server, sender_key, parsed_dict): @@ -497,12 +501,12 @@ class MatrixSyncEvent(MatrixEvent): olm = server.olm content = parsed_dict.pop("content") - if content["type"] != "m.megolm.v1.aes-sha2": + if content["algorithm"] != "m.megolm.v1.aes-sha2": return - room_id = sanitize_id(content["room_id"]) - session_id = sanitize_id(content["session_id"]) - session_key = sanitize_id(content["session_key"]) + room_id = content["room_id"] + session_id = content["session_id"] + session_key = content["session_key"] if session_id in olm.group_sessions[room_id]: return @@ -516,7 +520,7 @@ class MatrixSyncEvent(MatrixEvent): if event["type"] == "m.room.encrypted": if (event["content"]["algorithm"] == 'm.olm.v1.curve25519-aes-sha2'): - MatrixSyncEvent._get_olm_device_event(server, parsed_dict) + MatrixSyncEvent._get_olm_device_event(server, event) @classmethod def from_dict(cls, server, parsed_dict): @@ -531,12 +535,12 @@ class MatrixSyncEvent(MatrixEvent): parsed_dict["device_one_time_keys_count"]["signed_curve25519"]) MatrixSyncEvent._get_to_device_events( - server.olm, parsed_dict.pop("to_device")) + server, parsed_dict.pop("to_device")) room_info_dict = parsed_dict["rooms"] join_infos, invite_infos = MatrixSyncEvent._infos_from_dict( - room_info_dict) + server.olm, room_info_dict) return cls(server, next_batch, join_infos, invite_infos, one_time_key_count) diff --git a/matrix/rooms.py b/matrix/rooms.py index b0dc06f..ff7f7c0 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -17,6 +17,8 @@ from __future__ import unicode_literals from builtins import str +import json + from pprint import pformat from collections import namedtuple, deque @@ -243,7 +245,7 @@ class RoomInfo(): return None, None @staticmethod - def parse_event(event_dict): + def parse_event(olm, room_id, event_dict): # type: (Dict[Any, Any]) -> (RoomEvent, RoomEvent) state_event = None message_event = None @@ -270,11 +272,30 @@ class RoomInfo(): state_event = RoomAliasEvent.from_dict(event_dict) elif event_dict["type"] == "m.room.encryption": state_event = RoomEncryptionEvent.from_dict(event_dict) + elif event_dict["type"] == "m.room.encrypted": + state_event, message_event = RoomInfo._decrypt_event(olm, room_id, + event_dict) return state_event, message_event @staticmethod - def _parse_events(parsed_dict, messages=True, state=True): + def _decrypt_event(olm, room_id, event_dict): + session_id = event_dict["content"]["session_id"] + ciphertext = event_dict["content"]["ciphertext"] + plaintext = olm.group_decrypt(room_id, session_id, ciphertext) + + if not plaintext: + return None, None + + parsed_plaintext = json.loads(plaintext, encoding="utf-8") + + event_dict["content"] = parsed_plaintext["content"] + event_dict["type"] = parsed_plaintext["type"] + + return RoomInfo.parse_event(olm, room_id, event_dict) + + @staticmethod + def _parse_events(olm, room_id, parsed_dict, messages=True, state=True): state_events = [] message_events = [] @@ -283,7 +304,7 @@ class RoomInfo(): try: for event in parsed_dict: - m_event, s_event = RoomInfo.parse_event(event) + m_event, s_event = RoomInfo.parse_event(olm, room_id, event) state_events.append(m_event) message_events.append(s_event) except (ValueError, TypeError, KeyError) as error: @@ -306,14 +327,14 @@ class RoomInfo(): return events @classmethod - def from_dict(cls, room_id, parsed_dict): + def from_dict(cls, olm, room_id, parsed_dict): prev_batch = sanitize_id(parsed_dict['timeline']['prev_batch']) state_dict = parsed_dict['state']['events'] timeline_dict = parsed_dict['timeline']['events'] - state_events = RoomInfo._parse_events(state_dict, messages=False) - timeline_events = RoomInfo._parse_events(timeline_dict) + state_events = RoomInfo._parse_events(olm, room_id, state_dict, messages=False) + timeline_events = RoomInfo._parse_events(olm, room_id, timeline_dict) events = state_events + timeline_events From 8a162a7a80a06facd7ea937827af4d54b84c91c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sat, 7 Apr 2018 11:30:36 +0200 Subject: [PATCH 012/269] encryption: Don't pass the server to the Olm class. --- matrix/encryption.py | 25 ++++++++++++++----------- matrix/server.py | 10 +++++++--- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index dd077fa..aa1429a 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -157,11 +157,17 @@ class Olm(): @encrypt_enabled def __init__( self, + user, + device_id, + session_path, account=None, sessions=defaultdict(list), group_sessions=defaultdict(dict) ): - # type: (Account, Dict[str, List[Session]) -> None + # type: (str, str, str, Account, Dict[str, List[Session]) -> None + self.user = user + self.device_id = device_id + self.session_path = session_path if account: self.account = account else: @@ -224,28 +230,25 @@ class Olm(): @classmethod @encrypt_enabled - def from_session_dir(cls, server): + def from_session_dir(cls, user, device_id, session_path): # type: (Server) -> Olm - account_file_name = "{}_{}.account".format(server.user, - server.device_id) - session_path = server.get_session_path() + account_file_name = "{}_{}.account".format(user, device_id) path = os.path.join(session_path, account_file_name) try: with open(path, "rb") as f: pickle = f.read() account = Account.from_pickle(pickle) - return cls(account) + return cls(user, device_id, session_path, account) except OlmAccountError as error: raise EncryptionError(error) @encrypt_enabled - def to_session_dir(self, server): + def to_session_dir(self): # type: (Server) -> None - account_file_name = "{}_{}.account".format(server.user, - server.device_id) - session_path = server.get_session_path() - path = os.path.join(session_path, account_file_name) + account_file_name = "{}_{}.account".format(self.user, + self.device_id) + path = os.path.join(self.session_path, account_file_name) try: with open(path, "wb") as f: diff --git a/matrix/server.py b/matrix/server.py index eadf9b5..deb1f80 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -141,7 +141,11 @@ class MatrixServer: def _load_olm(self): try: - self.olm = Olm.from_session_dir(self) + self.olm = Olm.from_session_dir( + self.user, + self.device_id, + self.get_session_path() + ) except FileNotFoundError: pass except EncryptionError as error: @@ -164,11 +168,11 @@ class MatrixServer: server=self.name, device=self.device_id) W.prnt(self.server_buffer, message) - self.olm = Olm() + self.olm = Olm(self.user, self.device_id, self.get_session_path()) @encrypt_enabled def store_olm(self): - self.olm.to_session_dir(self) + self.olm.to_session_dir() def _create_options(self, config_file): options = [ From 0bd20cc3336985c158e1a19f0c78fe9e42590160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Wed, 11 Apr 2018 14:00:37 +0200 Subject: [PATCH 013/269] encryption: Store the account and sessions in a sqlite db. --- matrix/encryption.py | 175 ++++++++++++++++++++++++++++++++++++------- matrix/events.py | 2 +- matrix/server.py | 6 ++ 3 files changed, 157 insertions(+), 26 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index aa1429a..28e684a 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -19,9 +19,10 @@ from __future__ import unicode_literals import os import json +import sqlite3 # pylint: disable=redefined-builtin -from builtins import str +from builtins import str, bytes from collections import defaultdict from functools import wraps @@ -31,7 +32,7 @@ import matrix.globals try: from olm.account import Account, OlmAccountError - from olm.session import (InboundSession, OlmSessionError, OlmMessage, + from olm.session import (Session, InboundSession, OlmSessionError, OlmPreKeyMessage) from olm.group_session import InboundGroupSession, OlmGroupSessionError except ImportError: @@ -160,37 +161,55 @@ class Olm(): user, device_id, session_path, + database=None, account=None, - sessions=defaultdict(list), - group_sessions=defaultdict(dict) + sessions=None, + inbound_group_sessions=None ): # type: (str, str, str, Account, Dict[str, List[Session]) -> None self.user = user self.device_id = device_id self.session_path = session_path + self.database = database + + if not database: + db_file = "{}_{}.db".format(user, device_id) + db_path = os.path.join(session_path, db_file) + self.database = sqlite3.connect(db_path) + Olm._check_db_tables(self.database) + if account: self.account = account + else: self.account = Account() + self._insert_acc_to_db() + + if not sessions: + sessions = defaultdict(list) + + if not inbound_group_sessions: + inbound_group_sessions = defaultdict(dict) self.sessions = sessions - self.group_sessions = group_sessions + self.inbound_group_sessions = inbound_group_sessions def _create_session(self, sender, sender_key, message): W.prnt("", "matrix: Creating session for {}".format(sender)) session = InboundSession(self.account, message, sender_key) W.prnt("", "matrix: Created session for {}".format(sender)) self.sessions[sender].append(session) - # self.account.remove_one_time_keys(session) - # TODO store account here + self._store_session(sender, session) + self.account.remove_one_time_keys(session) + self._update_acc_in_db() return session def create_group_session(self, room_id, session_id, session_key): W.prnt("", "matrix: Creating group session for {}".format(room_id)) session = InboundGroupSession(session_key) - self.group_sessions[room_id][session_id] = session - # TODO store account here + self.inbound_group_sessions[room_id][session_id] = session + self._store_inbound_group_session(room_id, session) @encrypt_enabled def decrypt(self, sender, sender_key, message): @@ -217,10 +236,10 @@ class Olm(): @encrypt_enabled def group_decrypt(self, room_id, session_id, ciphertext): - if session_id not in self.group_sessions[room_id]: + if session_id not in self.inbound_group_sessions[room_id]: return None - session = self.group_sessions[room_id][session_id] + session = self.inbound_group_sessions[room_id][session_id] try: plaintext = session.decrypt(ciphertext) except OlmGroupSessionError: @@ -232,28 +251,134 @@ class Olm(): @encrypt_enabled def from_session_dir(cls, user, device_id, session_path): # type: (Server) -> Olm - account_file_name = "{}_{}.account".format(user, device_id) - path = os.path.join(session_path, account_file_name) + db_file = "{}_{}.db".format(user, device_id) + db_path = os.path.join(session_path, db_file) + database = sqlite3.connect(db_path) + Olm._check_db_tables(database) + + cursor = database.cursor() + + cursor.execute("select pickle from olmaccount where user = ?", (user,)) + row = cursor.fetchone() + account_pickle = row[0] + + cursor.execute("select user, pickle from olmsessions") + db_sessions = cursor.fetchall() + + cursor.execute("select room_id, pickle from inbound_group_sessions") + db_inbound_group_sessions = cursor.fetchall() + + cursor.close() + + sessions = defaultdict(list) + inbound_group_sessions = defaultdict(dict) try: - with open(path, "rb") as f: - pickle = f.read() - account = Account.from_pickle(pickle) - return cls(user, device_id, session_path, account) - except OlmAccountError as error: + account = Account.from_pickle(bytes(account_pickle, "utf-8")) + + for db_session in db_sessions: + sessions[db_session[0]].append( + Session.from_pickle(bytes(db_session[1], "utf-8"))) + + for db_session in db_inbound_group_sessions: + session = InboundGroupSession.from_pickle( + bytes(db_session[1], "utf-8")) + inbound_group_sessions[db_session[0]][session.id] = session + + return cls(user, device_id, session_path, database, account, + sessions, inbound_group_sessions) + except (OlmAccountError, OlmSessionError) as error: raise EncryptionError(error) + def _update_acc_in_db(self): + cursor = self.database.cursor() + cursor.execute("update olmaccount set pickle=? where user = ?", + (self.account.pickle(), self.user)) + self.database.commit() + cursor.close() + + def _update_sessions_in_db(self): + cursor = self.database.cursor() + + for user, session_list in self.sessions.items(): + for session in session_list: + cursor.execute("""update olmsessions set pickle=? + where user = ? and session_id = ?""", + (session.pickle(), user, session.id())) + self.database.commit() + + cursor.close() + + def _update_inbound_group_sessions(self): + cursor = self.database.cursor() + + for room_id, session_dict in self.inbound_group_sessions.items(): + for session in session_dict.values(): + cursor.execute("""update inbound_group_sessions set pickle=? + where room_id = ? and session_id = ?""", + (session.pickle(), room_id, session.id())) + self.database.commit() + + cursor.close() + + def _store_session(self, user, session): + cursor = self.database.cursor() + + cursor.execute("insert into olmsessions values(?,?,?)", + (user, session.id(), session.pickle())) + + self.database.commit() + + cursor.close() + + def _store_inbound_group_session(self, room_id, session): + cursor = self.database.cursor() + + cursor.execute("insert into inbound_group_sessions values(?,?,?)", + (room_id, session.id, session.pickle())) + + self.database.commit() + + cursor.close() + + def _insert_acc_to_db(self): + cursor = self.database.cursor() + cursor.execute("insert into olmaccount values (?,?)", + (self.user, self.account.pickle())) + self.database.commit() + cursor.close() + + @staticmethod + def _check_db_tables(database): + cursor = database.cursor() + cursor.execute("""select name from sqlite_master where type='table' + and name='olmaccount'""") + if not cursor.fetchone(): + cursor.execute("create table olmaccount (user text, pickle text)") + database.commit() + + cursor.execute("""select name from sqlite_master where type='table' + and name='olmsessions'""") + if not cursor.fetchone(): + cursor.execute("""create table olmsessions (user text, + session_id text, pickle text)""") + database.commit() + + cursor.execute("""select name from sqlite_master where type='table' + and name='inbound_group_sessions'""") + if not cursor.fetchone(): + cursor.execute("""create table inbound_group_sessions + (room_id text, session_id text, pickle text)""") + database.commit() + + cursor.close() + @encrypt_enabled def to_session_dir(self): # type: (Server) -> None - account_file_name = "{}_{}.account".format(self.user, - self.device_id) - path = os.path.join(self.session_path, account_file_name) - try: - with open(path, "wb") as f: - pickle = self.account.pickle() - f.write(pickle) + self._update_acc_in_db() + self._update_sessions_in_db() except OlmAccountError as error: raise EncryptionError(error) diff --git a/matrix/events.py b/matrix/events.py index ad576a7..e3127e2 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -508,7 +508,7 @@ class MatrixSyncEvent(MatrixEvent): session_id = content["session_id"] session_key = content["session_key"] - if session_id in olm.group_sessions[room_id]: + if session_id in olm.inbound_group_sessions[room_id]: return olm.create_group_session(room_id, session_id, session_key) diff --git a/matrix/server.py b/matrix/server.py index deb1f80..f5517fa 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -146,6 +146,12 @@ class MatrixServer: self.device_id, self.get_session_path() ) + message = ("{prefix}matrix: Loaded Olm account for {user} (device:" + "{device})").format(prefix=W.prefix("network"), + user=self.user, + device=self.device_id) + W.prnt("", message) + except FileNotFoundError: pass except EncryptionError as error: From 49eb6548d1d8ec82bcba3d821f797669af39ba53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Wed, 11 Apr 2018 20:56:11 +0200 Subject: [PATCH 014/269] encryption: Fix olm decryption if we already have a session. --- matrix/encryption.py | 7 +++++-- matrix/events.py | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 28e684a..f2c826b 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -222,11 +222,14 @@ class Olm(): continue plaintext = session.decrypt(message) - break + return plaintext except OlmSessionError: pass - session = self._create_session(sender, sender_key, message) + try: + session = self._create_session(sender, sender_key, message) + except OlmSessionError: + return None try: plaintext = session.decrypt(message) diff --git a/matrix/events.py b/matrix/events.py index e3127e2..4af350a 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -475,6 +475,9 @@ class MatrixSyncEvent(MatrixEvent): olm = server.olm plaintext = olm.decrypt(sender, sender_key, message) + if not plaintext: + return None + # TODO check sender key parsed_plaintext = json.loads(plaintext, encoding='utf-8') decrypted_sender = parsed_plaintext["sender"] From 18bd62cb906f15db44ccc26e2b13c058efe9fe25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 12 Apr 2018 14:19:32 +0200 Subject: [PATCH 015/269] encryption: Add key query functionality. --- matrix/api.py | 30 +++++++++++++++++++++++++++++ matrix/encryption.py | 9 +++++++++ matrix/events.py | 45 +++++++++++++++++++++++++++++++++++++++++++- matrix/server.py | 33 +++++++++++++++++++++++++++++++- 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/matrix/api.py b/matrix/api.py index ae4539f..dfb775a 100644 --- a/matrix/api.py +++ b/matrix/api.py @@ -279,6 +279,20 @@ class MatrixClient: return HttpRequest(RequestType.POST, self.host, path, content) + def keys_query(self, users): + query_parameters = {"access_token": self.access_token} + + path = ("{api}/keys/query?" + "{query_parameters}").format( + api=MATRIX_API_PATH, + query_parameters=urlencode(query_parameters)) + + content = { + "device_keys": {user: {} for user in users} + } + + return HttpRequest(RequestType.POST, self.host, path, content) + def mxc_to_http(self, mxc): # type: (str) -> str url = urlparse(mxc) @@ -588,3 +602,19 @@ class MatrixKeyUploadMessage(MatrixMessage): server, self.device_keys) return self._decode(server, object_hook) + + +class MatrixKeyQueryMessage(MatrixMessage): + + def __init__(self, client, users): + data = { + "users": users, + } + + MatrixMessage.__init__(self, client.keys_query, data) + + def decode_body(self, server): + object_hook = partial(MatrixEvents.MatrixKeyQueryEvent.from_dict, + server) + + return self._decode(server, object_hook) diff --git a/matrix/encryption.py b/matrix/encryption.py index f2c826b..acaef9c 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -153,6 +153,14 @@ class EncryptionError(Exception): pass +class OlmDeviceKey(): + def __init__(self, user_id, device_id, key_dict): + # type: (str, str, Dict[str, str]) + self.user_id = user_id + self.device_id = device_id + self.keys = key_dict + + class Olm(): @encrypt_enabled @@ -171,6 +179,7 @@ class Olm(): self.device_id = device_id self.session_path = session_path self.database = database + self.device_keys = {} if not database: db_file = "{}_{}.db".format(user, device_id) diff --git a/matrix/events.py b/matrix/events.py index 4af350a..f79d048 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -19,7 +19,7 @@ from builtins import str import json -from collections import deque +from collections import deque, defaultdict from functools import partial from operator import itemgetter @@ -30,6 +30,8 @@ from matrix.rooms import (matrix_create_room_buffer, RoomInfo, RoomMessageText, RoomMessageEvent, RoomRedactedMessageEvent, RoomMessageEmote) +from matrix.encryption import OlmDeviceKey + try: from olm.session import OlmMessage, OlmPreKeyMessage except ImportError: @@ -307,6 +309,47 @@ class MatrixKickEvent(MatrixEvent): False, parsed_dict) +class MatrixKeyQueryEvent(MatrixEvent): + + def __init__(self, server, keys): + self.keys = keys + MatrixEvent.__init__(self, server) + + @staticmethod + def _get_keys(key_dict): + keys = {} + + for key_type, key in key_dict.items(): + key_type, _ = key_type.split(":") + keys[key_type] = key + + return keys + + @classmethod + def from_dict(cls, server, parsed_dict): + keys = defaultdict(list) + try: + for user_id, device_dict in parsed_dict["device_keys"].items(): + for device_id, key_dict in device_dict.items(): + device_keys = MatrixKeyQueryEvent._get_keys( + key_dict.pop("keys")) + keys[user_id].append(OlmDeviceKey(user_id, device_id, + device_keys)) + return cls(server, keys) + except KeyError: + return MatrixErrorEvent.from_dict(server, "Error kicking user", + False, parsed_dict) + + def execute(self): + olm = self.server.olm + + if olm.device_keys == self.keys: + return + + olm.device_keys = self.keys + # TODO invalidate megolm sessions for rooms that got new devices + + class MatrixBacklogEvent(MatrixEvent): def __init__(self, server, room_id, end_token, events): diff --git a/matrix/server.py b/matrix/server.py index f5517fa..f9cb46a 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -37,7 +37,8 @@ from matrix.api import ( MatrixClient, MatrixSyncMessage, MatrixLoginMessage, - MatrixKeyUploadMessage + MatrixKeyUploadMessage, + MatrixKeyQueryMessage ) from matrix.encryption import Olm, EncryptionError, encrypt_enabled @@ -92,6 +93,7 @@ class MatrixServer: self.send_fd_hook = None # type: weechat.hook self.send_buffer = b"" # type: bytes self.current_message = None # type: MatrixMessage + self.device_check_timestamp = None self.http_parser = HttpParser() # type: HttpParser self.http_buffer = [] # type: List[bytes] @@ -497,6 +499,21 @@ class MatrixServer: self.olm.account.generate_one_time_keys(key_count) self.upload_keys(device_keys=False, one_time_keys=True) + @encrypt_enabled + def query_keys(self): + users = [] + + for room in self.rooms.values(): + if not room.encrypted: + continue + users += list(room.users) + + if not users: + return + + message = MatrixKeyQueryMessage(self.client, users) + self.send_queue.append(message) + def login(self): # type: (MatrixServer) -> None message = MatrixLoginMessage(self.client, self.user, self.password, @@ -705,6 +722,20 @@ def matrix_timer_cb(server_name, remaining_calls): server.send_queue.appendleft(message) break + if not server.next_batch: + return W.WEECHAT_RC_OK + + # check for new devices by users in encrypted rooms periodically + if (not server.device_check_timestamp or + current_time - server.device_check_timestamp > 600): + + W.prnt(server.server_buffer, + "{prefix}matrix: Querying user devices.".format( + prefix=W.prefix("networ"))) + + server.query_keys() + server.device_check_timestamp = current_time + return W.WEECHAT_RC_OK From 9e86ccc3f734366122e533140eb4ebd11bb00316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sun, 6 May 2018 13:00:07 +0200 Subject: [PATCH 016/269] encryption: Add key printing support for queried keys. --- matrix/encryption.py | 52 +++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index acaef9c..d8e9803 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -126,21 +126,43 @@ def matrix_olm_command_cb(server_name, buffer, args): if not command or command == "info": olm = server.olm - device_msg = (" - Device ID: {}\n".format(server.device_id) - if server.device_id else "") - id_key = partition_key(olm.account.identity_keys()["curve25519"]) - fp_key = partition_key(olm.account.identity_keys()["ed25519"]) - message = ("{prefix}matrix: Identity keys:\n" - " - User: {user}\n" - "{device_msg}" - " - Identity key: {id_key}\n" - " - Fingerprint key: {fp_key}\n").format( - prefix=W.prefix("network"), - user=server.user, - device_msg=device_msg, - id_key=id_key, - fp_key=fp_key) - W.prnt(server.server_buffer, message) + + if not args or args[0] == "private": + device_msg = (" - Device ID: {}\n".format(server.device_id) + if server.device_id else "") + id_key = partition_key(olm.account.identity_keys()["curve25519"]) + fp_key = partition_key(olm.account.identity_keys()["ed25519"]) + message = ("{prefix}matrix: Identity keys:\n" + " - User: {user}\n" + "{device_msg}" + " - Identity key: {id_key}\n" + " - Fingerprint key: {fp_key}\n").format( + prefix=W.prefix("network"), + user=server.user, + device_msg=device_msg, + id_key=id_key, + fp_key=fp_key) + W.prnt(server.server_buffer, message) + elif args[0] == "all": + for user, keys in olm.device_keys.items(): + message = ("{prefix}matrix: Identity keys:\n" + " - User: {user}\n").format( + prefix=W.prefix("network"), + user=user) + W.prnt(server.server_buffer, message) + + for key in keys: + id_key = partition_key(key.keys["curve25519"]) + fp_key = partition_key(key.keys["ed25519"]) + device_msg = (" - Device ID: {}\n".format( + key.device_id) if key.device_id else "") + message = ("{device_msg}" + " - Identity key: {id_key}\n" + " - Fingerprint key: {fp_key}\n\n").format( + device_msg=device_msg, + id_key=id_key, + fp_key=fp_key) + W.prnt(server.server_buffer, message) else: message = ("{prefix}matrix: Command not implemented.".format( prefix=W.prefix("error"))) From 658ec67ff4df0df889bfae01db3c77e56c645dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sun, 6 May 2018 13:37:34 +0200 Subject: [PATCH 017/269] encryption: Add a error message for the olm command. --- matrix/encryption.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index d8e9803..ff7d1d2 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -42,7 +42,7 @@ from matrix.globals import W, SERVERS from matrix.utf import utf8_decode -def own_buffer(f): +def own_buffer_or_error(f): @wraps(f) def wrapper(data, buffer, *args, **kwargs): @@ -53,6 +53,10 @@ def own_buffer(f): elif buffer == server.server_buffer: return f(server.name, buffer, *args, **kwargs) + W.prnt("", "{prefix}matrix: command \"olm\" must be executed on a " + "matrix buffer (server or channel)".format( + prefix=W.prefix("error"))) + return W.WEECHAT_RC_OK return wrapper @@ -118,7 +122,7 @@ def partition_key(key): return ' '.join(''.join(g) for g in groups) -@own_buffer +@own_buffer_or_error @utf8_decode def matrix_olm_command_cb(server_name, buffer, args): server = SERVERS[server_name] From 8f7dac4a0d655537330839dc031df4ea98c18146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sun, 6 May 2018 15:03:50 +0200 Subject: [PATCH 018/269] encryption: Change the way olm sessions are stored. --- matrix/encryption.py | 64 ++++++++++++++++++++++++++------------------ matrix/events.py | 11 ++++---- 2 files changed, 43 insertions(+), 32 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index ff7d1d2..92088b1 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -39,6 +39,7 @@ except ImportError: matrix.globals.ENCRYPTION = False from matrix.globals import W, SERVERS +from matrix.utils import sanitize_id from matrix.utf import utf8_decode @@ -221,7 +222,7 @@ class Olm(): self._insert_acc_to_db() if not sessions: - sessions = defaultdict(list) + sessions = defaultdict(lambda: defaultdict(list)) if not inbound_group_sessions: inbound_group_sessions = defaultdict(dict) @@ -233,8 +234,6 @@ class Olm(): W.prnt("", "matrix: Creating session for {}".format(sender)) session = InboundSession(self.account, message, sender_key) W.prnt("", "matrix: Created session for {}".format(sender)) - self.sessions[sender].append(session) - self._store_session(sender, session) self.account.remove_one_time_keys(session) self._update_acc_in_db() @@ -250,16 +249,21 @@ class Olm(): def decrypt(self, sender, sender_key, message): plaintext = None - for session in self.sessions[sender]: - try: - if isinstance(message, OlmPreKeyMessage): - if not session.matches(message): - continue + for device_id, session_list in self.sessions[sender].items(): + for session in session_list: + W.prnt("", "Trying session for device {}".format(device_id)) + try: + if isinstance(message, OlmPreKeyMessage): + if not session.matches(message): + continue - plaintext = session.decrypt(message) - return plaintext - except OlmSessionError: - pass + W.prnt("", "Decrypting using existing session") + plaintext = session.decrypt(message) + parsed_plaintext = json.loads(plaintext, encoding='utf-8') + W.prnt("", "Decrypted using existing session") + return parsed_plaintext + except OlmSessionError: + pass try: session = self._create_session(sender, sender_key, message) @@ -268,7 +272,12 @@ class Olm(): try: plaintext = session.decrypt(message) - return plaintext + parsed_plaintext = json.loads(plaintext, encoding='utf-8') + + device_id = sanitize_id(parsed_plaintext["sender_device"]) + self.sessions[sender][device_id].append(session) + self._store_session(sender, device_id, session) + return parsed_plaintext except OlmSessionError: return None @@ -300,7 +309,7 @@ class Olm(): row = cursor.fetchone() account_pickle = row[0] - cursor.execute("select user, pickle from olmsessions") + cursor.execute("select user, device_id, pickle from olmsessions") db_sessions = cursor.fetchall() cursor.execute("select room_id, pickle from inbound_group_sessions") @@ -308,15 +317,15 @@ class Olm(): cursor.close() - sessions = defaultdict(list) + sessions = defaultdict(lambda: defaultdict(list)) inbound_group_sessions = defaultdict(dict) try: account = Account.from_pickle(bytes(account_pickle, "utf-8")) for db_session in db_sessions: - sessions[db_session[0]].append( - Session.from_pickle(bytes(db_session[1], "utf-8"))) + sessions[db_session[0]][db_session[1]].append( + Session.from_pickle(bytes(db_session[2], "utf-8"))) for db_session in db_inbound_group_sessions: session = InboundGroupSession.from_pickle( @@ -338,11 +347,14 @@ class Olm(): def _update_sessions_in_db(self): cursor = self.database.cursor() - for user, session_list in self.sessions.items(): - for session in session_list: - cursor.execute("""update olmsessions set pickle=? - where user = ? and session_id = ?""", - (session.pickle(), user, session.id())) + for user, session_dict in self.sessions.items(): + for device_id, session_list in session_dict.items(): + for session in session_list: + cursor.execute("""update olmsessions set pickle=? + where user = ? and session_id = ? and + device_id = ?""", + (session.pickle(), user, session.id(), + device_id)) self.database.commit() cursor.close() @@ -359,11 +371,11 @@ class Olm(): cursor.close() - def _store_session(self, user, session): + def _store_session(self, user, device_id, session): cursor = self.database.cursor() - cursor.execute("insert into olmsessions values(?,?,?)", - (user, session.id(), session.pickle())) + cursor.execute("insert into olmsessions values(?,?,?,?)", + (user, device_id, session.id(), session.pickle())) self.database.commit() @@ -399,7 +411,7 @@ class Olm(): and name='olmsessions'""") if not cursor.fetchone(): cursor.execute("""create table olmsessions (user text, - session_id text, pickle text)""") + device_id text, session_id text, pickle text)""") database.commit() cursor.execute("""select name from sqlite_master where type='table' diff --git a/matrix/events.py b/matrix/events.py index f79d048..2b96865 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -522,10 +522,9 @@ class MatrixSyncEvent(MatrixEvent): return None # TODO check sender key - parsed_plaintext = json.loads(plaintext, encoding='utf-8') - decrypted_sender = parsed_plaintext["sender"] - decrypted_recepient = parsed_plaintext["recipient"] - decrypted_recepient_key = parsed_plaintext["recipient_keys"]["ed25519"] + decrypted_sender = plaintext["sender"] + decrypted_recepient = plaintext["recipient"] + decrypted_recepient_key = plaintext["recipient_keys"]["ed25519"] if (sender != decrypted_sender or server.user_id != decrypted_recepient or @@ -536,10 +535,10 @@ class MatrixSyncEvent(MatrixEvent): W.prnt("", error_message) return None - if parsed_plaintext["type"] != "m.room_key": + if plaintext["type"] != "m.room_key": return None - MatrixSyncEvent._handle_key_event(server, sender_key, parsed_plaintext) + MatrixSyncEvent._handle_key_event(server, sender_key, plaintext) @staticmethod def _handle_key_event(server, sender_key, parsed_dict): From 320f4902737557c93300771e9db07aef52adffe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sun, 6 May 2018 15:04:19 +0200 Subject: [PATCH 019/269] encryption: Only try to store the olm info if the olm object exists. --- matrix/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index f9cb46a..72f8599 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -180,7 +180,8 @@ class MatrixServer: @encrypt_enabled def store_olm(self): - self.olm.to_session_dir() + if self.olm: + self.olm.to_session_dir() def _create_options(self, config_file): options = [ From 8a3bda97977c57ea01336647f0df70c20b59f880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Wed, 9 May 2018 14:00:32 +0200 Subject: [PATCH 020/269] encryption: Stub functions for encrypted messages. --- main.py | 7 ++++--- matrix/api.py | 30 ++++++++++++++++++++++++++++++ matrix/commands.py | 3 +++ matrix/encryption.py | 34 +++++++++++++++++++++++++++++++++- matrix/events.py | 17 +++++++++++++++++ matrix/server.py | 20 ++++++++++++++++++++ 6 files changed, 107 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 4e18f7b..5b9b02a 100644 --- a/main.py +++ b/main.py @@ -410,11 +410,12 @@ def room_input_cb(server_name, buffer, input_data): room_id = key_from_value(server.buffers, buffer) room = server.rooms[room_id] - if room.encrypted: - return W.WEECHAT_RC_OK - formatted_data = Formatted.from_input_line(input_data) + if room.encrypted: + server.send_room_message(room_id, formatted_data) + return W.WEECHAT_RC_OK + message = MatrixSendMessage( server.client, room_id=room_id, formatted_message=formatted_data) diff --git a/matrix/api.py b/matrix/api.py index dfb775a..00f1570 100644 --- a/matrix/api.py +++ b/matrix/api.py @@ -293,6 +293,20 @@ class MatrixClient: return HttpRequest(RequestType.POST, self.host, path, content) + def key_claim(self, key_dict): + query_parameters = {"access_token": self.access_token} + + path = ("{api}/keys/claim?" + "{query_parameters}").format( + api=MATRIX_API_PATH, + query_parameters=urlencode(query_parameters)) + + content = { + "one_time_keys": {key_dict} + } + + return HttpRequest(RequestType.POST, self.host, path, content) + def mxc_to_http(self, mxc): # type: (str) -> str url = urlparse(mxc) @@ -618,3 +632,19 @@ class MatrixKeyQueryMessage(MatrixMessage): server) return self._decode(server, object_hook) + + +class MatrixKeyClaimMessage(MatrixMessage): + + def __init__(self, client, key_dict): + data = { + "key_dict": key_dict, + } + + MatrixMessage.__init__(self, client.keys_query, data) + + def decode_body(self, server): + object_hook = partial(MatrixEvents.MatrixKeyClaimEvent.from_dict, + server) + + return self._decode(server, object_hook) diff --git a/matrix/commands.py b/matrix/commands.py index 958f180..0046526 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -134,6 +134,9 @@ def matrix_me_command_cb(data, buffer, args): room_id=room_id, formatted_message=formatted_data) + if server.rooms[room_id].encrypted: + return W.WEECHAT_RC_OK + server.send_or_queue(message) return W.WEECHAT_RC_OK diff --git a/matrix/encryption.py b/matrix/encryption.py index 92088b1..79ac911 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -20,6 +20,7 @@ from __future__ import unicode_literals import os import json import sqlite3 +import pprint # pylint: disable=redefined-builtin from builtins import str, bytes @@ -34,7 +35,11 @@ try: from olm.account import Account, OlmAccountError from olm.session import (Session, InboundSession, OlmSessionError, OlmPreKeyMessage) - from olm.group_session import InboundGroupSession, OlmGroupSessionError + from olm.group_session import ( + InboundGroupSession, + OutboundGroupSession, + OlmGroupSessionError + ) except ImportError: matrix.globals.ENCRYPTION = False @@ -229,6 +234,7 @@ class Olm(): self.sessions = sessions self.inbound_group_sessions = inbound_group_sessions + self.outbound_group_sessions = {} def _create_session(self, sender, sender_key, message): W.prnt("", "matrix: Creating session for {}".format(sender)) @@ -245,6 +251,32 @@ class Olm(): self.inbound_group_sessions[room_id][session_id] = session self._store_inbound_group_session(room_id, session) + def create_outbound_group_session(self, room_id): + session = OutboundGroupSession() + self.outbound_group_sessions[room_id] = session + self.create_group_session(room_id, session.id, session.session_key) + + @encrypt_enabled + def get_missing_sessions(self, users): + # type: (List[str]) -> Dict[str, Dict[str, str]] + missing = {} + + for user in users: + devices = [] + + for key in self.device_keys[user]: + # we don't need a session for our own device, skip it + if key.device_id == self.device_id: + continue + + if not self.sessions[user][key.device_id]: + devices.append(key.device_id) + + if devices: + missing[user] = {device: "ed25519" for device in devices} + + return missing + @encrypt_enabled def decrypt(self, sender, sender_key, message): plaintext = None diff --git a/matrix/events.py b/matrix/events.py index 2b96865..ee624d9 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -18,6 +18,7 @@ from __future__ import unicode_literals from builtins import str import json +import pprint from collections import deque, defaultdict from functools import partial @@ -337,10 +338,12 @@ class MatrixKeyQueryEvent(MatrixEvent): device_keys)) return cls(server, keys) except KeyError: + # TODO error message return MatrixErrorEvent.from_dict(server, "Error kicking user", False, parsed_dict) def execute(self): + # TODO move this logic into an Olm method olm = self.server.olm if olm.device_keys == self.keys: @@ -350,6 +353,20 @@ class MatrixKeyQueryEvent(MatrixEvent): # TODO invalidate megolm sessions for rooms that got new devices +class MatrixKeyClaimEvent(MatrixEvent): + + def __init__(self, server, keys): + self.keys = keys + MatrixEvent.__init__(self, server) + + @classmethod + def from_dict(cls, server, parsed_dict): + raise NotImplementedError + + def execute(self): + pass + + class MatrixBacklogEvent(MatrixEvent): def __init__(self, server, room_id, end_token, events): diff --git a/matrix/server.py b/matrix/server.py index 72f8599..d32e155 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -476,6 +476,26 @@ class MatrixServer: message = MatrixSyncMessage(self.client, self.next_batch, limit) self.send_queue.append(message) + def send_room_message(self, room_id, formatted_data): + # type: (str, Formatted) -> None + room = self.rooms[room_id] + + if not room.encrypted: + return + + missing = self.olm.get_missing_sessions(room.users.keys()) + + if missing: + W.prnt("", "{prefix}matrix: Olm session missing for room, can't" + " encrypt message.") + W.prnt("", pprint.pformat(missing)) + # message = MatrixKeyClaimMessage(self.client, missing) + # self.send_or_queue(message) + # TODO claim keys for the missing user/device combinations + return + + # self.send_queue.append(message) + @encrypt_enabled def upload_keys(self, device_keys=False, one_time_keys=False): keys = self.olm.account.identity_keys() if device_keys else None From 642e518464c527e7a4f3bef1c636cd0d31432058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 11 May 2018 13:03:42 +0200 Subject: [PATCH 021/269] encryption: Initial encrypted message sending support. --- matrix/api.py | 69 +++++++++++++++++++++++++++++++ matrix/encryption.py | 98 +++++++++++++++++++++++++++++++++++++++++++- matrix/events.py | 17 ++++++++ matrix/rooms.py | 5 +++ matrix/server.py | 42 ++++++++++++++++++- 5 files changed, 228 insertions(+), 3 deletions(-) diff --git a/matrix/api.py b/matrix/api.py index 00f1570..69b416c 100644 --- a/matrix/api.py +++ b/matrix/api.py @@ -94,6 +94,19 @@ class MatrixClient: return HttpRequest(RequestType.GET, self.host, path) + def room_encrypted_message(self, room_id, content): + # type: (str, Dict[Any, Any]) -> HttpRequest + query_parameters = {"access_token": self.access_token} + + path = ("{api}/rooms/{room}/send/m.room.encrypted/" + "{tx_id}?{query_parameters}").format( + api=MATRIX_API_PATH, + room=quote(room_id), + tx_id=quote(str(self._get_txn_id())), + query_parameters=urlencode(query_parameters)) + + return HttpRequest(RequestType.PUT, self.host, path, content) + def room_send_message(self, room_id, message_type, @@ -307,6 +320,18 @@ class MatrixClient: return HttpRequest(RequestType.POST, self.host, path, content) + def to_device(self, event_type, content): + query_parameters = {"access_token": self.access_token} + + path = ("{api}/sendToDevice/{event_type}/{tx_id}?" + "{query_parameters}").format( + api=MATRIX_API_PATH, + event_type=event_type, + tx_id=quote(str(self._get_txn_id())), + query_parameters=urlencode(query_parameters)) + + return HttpRequest(RequestType.PUT, self.host, path, content) + def mxc_to_http(self, mxc): # type: (str) -> str url = urlparse(mxc) @@ -648,3 +673,47 @@ class MatrixKeyClaimMessage(MatrixMessage): server) return self._decode(server, object_hook) + + +class MatrixToDeviceMessage(MatrixMessage): + def __init__(self, client, to_device_dict): + data = { + "content": to_device_dict, + "event_type": "m.room.encrypted" + } + + MatrixMessage.__init__(self, client.to_device, data) + + def decode_body(self, server): + object_hook = partial(MatrixEvents.MatrixToDeviceEvent.from_dict, + server) + + return self._decode(server, object_hook) + + +class MatrixEncryptedMessage(MatrixMessage): + + def __init__(self, + client, + room_id, + formatted_message, + content): + self.room_id = room_id + self.formatted_message = formatted_message + + data = { + "room_id": self.room_id, + "content": content + } + + MatrixMessage.__init__(self, client.room_encrypted_message, data) + + def decode_body(self, server): + object_hook = partial( + MatrixEvents.MatrixSendEvent.from_dict, + server, + self.room_id, + self.formatted_message, + ) + + return self._decode(server, object_hook) diff --git a/matrix/encryption.py b/matrix/encryption.py index 79ac911..39cc6c5 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -34,7 +34,7 @@ import matrix.globals try: from olm.account import Account, OlmAccountError from olm.session import (Session, InboundSession, OlmSessionError, - OlmPreKeyMessage) + OlmMessage, OlmPreKeyMessage) from olm.group_session import ( InboundGroupSession, OutboundGroupSession, @@ -313,6 +313,29 @@ class Olm(): except OlmSessionError: return None + def group_encrypt(self, room_id, plaintext_dict): + # type: (str, Dict[str, str]) -> Dict[str, str], Bool + is_new = False + plaintext_dict["room_id"] = room_id + + if not room_id in self.outbound_group_sessions: + self.create_outbound_group_session(room_id) + is_new = True + + session = self.outbound_group_sessions[room_id] + + ciphertext = session.encrypt(Olm._to_json(plaintext_dict)) + + payload_dict = { + "algorithm": "m.megolm.v1.aes-sha2", + "sender_key": self.account.identity_keys()["curve25519"], + "ciphertext": ciphertext, + "session_id": session.id, + "device_id": self.device_id + } + + return payload_dict, is_new + @encrypt_enabled def group_decrypt(self, room_id, session_id, ciphertext): if session_id not in self.inbound_group_sessions[room_id]: @@ -326,6 +349,69 @@ class Olm(): return plaintext + def share_group_session(self, room_id, own_id, users): + group_session = self.outbound_group_sessions[room_id] + + key_content = { + "algorithm": "m.megolm.v1.aes-sha2", + "room_id": room_id, + "session_id": group_session.id, + "session_key": group_session.session_key, + "chain_index": group_session.message_index + } + + payload_dict = { + "type": "m.room_key", + "content": key_content, + # TODO we don't have the user_id in the Olm class + "sender": own_id, + "sender_device": self.device_id, + "keys": { + "ed25519": self.account.identity_keys()["ed25519"] + } + } + + to_device_dict = { + "messages": {} + } + + for user in users: + + for key in self.device_keys[user]: + if key.device_id == self.device_id: + continue + + device_payload_dict = payload_dict.copy() + # TODO sort the sessions + session = self.sessions[user][key.device_id][0] + device_payload_dict["recipient"] = user + device_payload_dict["recipient_keys"] = { + "ed25519": key.keys["ed25519"] + } + + W.prnt("", pprint.pformat(device_payload_dict)) + + olm_message = session.encrypt(Olm._to_json(device_payload_dict)) + + olm_dict = { + "algorithm": "m.olm.v1.curve25519-aes-sha2", + "sender_key": self.account.identity_keys()["curve25519"], + "ciphertext": { + key.keys["curve25519"]: { + "type": (0 if isinstance(olm_message, + OlmPreKeyMessage) else 1), + "body": olm_message.ciphertext + } + } + } + + to_device_dict["messages"][user] = { + key.device_id: olm_dict + } + + return to_device_dict + # return {} + @classmethod @encrypt_enabled def from_session_dir(cls, user, device_id, session_path): @@ -474,6 +560,16 @@ class Olm(): return signature + @staticmethod + def _to_json(json_dict): + # type: (Dict[Any, Any]) -> str + return json.dumps( + json_dict, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True + ) + @encrypt_enabled def mark_keys_as_published(self): self.account.mark_keys_as_published() diff --git a/matrix/events.py b/matrix/events.py index ee624d9..36773a7 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -367,6 +367,23 @@ class MatrixKeyClaimEvent(MatrixEvent): pass +class MatrixToDeviceEvent(MatrixEvent): + + def __init__(self, server): + MatrixEvent.__init__(self, server) + + @classmethod + def from_dict(cls, server, parsed_dict): + try: + if parsed_dict == {}: + return cls(server) + + raise KeyError + except KeyError: + return MatrixErrorEvent.from_dict(server, ("Error sending to " + "device message"), + False, parsed_dict) + class MatrixBacklogEvent(MatrixEvent): def __init__(self, server, room_id, end_token, events): diff --git a/matrix/rooms.py b/matrix/rooms.py index ff7f7c0..3313f12 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -652,6 +652,11 @@ class RoomMemberJoin(RoomEvent): # calculate room display name and set it as the buffer list name room_name = room.display_name(server.user_id) + + # A user has joined an encrypted room, we need to check for new devices + if room.encrypted: + server.device_check_timestamp = None + W.buffer_set(buff, "short_name", room_name) diff --git a/matrix/server.py b/matrix/server.py index d32e155..21918e4 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -38,7 +38,9 @@ from matrix.api import ( MatrixSyncMessage, MatrixLoginMessage, MatrixKeyUploadMessage, - MatrixKeyQueryMessage + MatrixKeyQueryMessage, + MatrixToDeviceMessage, + MatrixEncryptedMessage ) from matrix.encryption import Olm, EncryptionError, encrypt_enabled @@ -483,6 +485,7 @@ class MatrixServer: if not room.encrypted: return + # TODO don't send messages unless all the devices are verified missing = self.olm.get_missing_sessions(room.users.keys()) if missing: @@ -494,7 +497,42 @@ class MatrixServer: # TODO claim keys for the missing user/device combinations return - # self.send_queue.append(message) + body = {"msgtype": "m.text", "body": formatted_data.to_plain()} + + if formatted_data.is_formatted(): + body["format"] = "org.matrix.custom.html" + body["formatted_body"] = formatted_data.to_html() + + plaintext_dict = { + "type": "m.room.message", + "content": body + } + + W.prnt("", "matrix: Encrypting message") + + payload_dict, session_is_new = self.olm.group_encrypt( + room_id, + plaintext_dict + ) + + if session_is_new: + to_device_dict = self.olm.share_group_session( + room_id, + self.user_id, + room.users.keys() + ) + message = MatrixToDeviceMessage(self.client, to_device_dict) + W.prnt("", "matrix: Megolm session missing for room.") + self.send_queue.append(message) + + message = MatrixEncryptedMessage( + self.client, + room_id, + formatted_data, + payload_dict + ) + + self.send_queue.append(message) @encrypt_enabled def upload_keys(self, device_keys=False, one_time_keys=False): From 5f3842e4fa3ed1957c1e2a5550612375c9e52ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sat, 12 May 2018 10:23:28 +0200 Subject: [PATCH 022/269] encryption: Style changes. --- matrix/encryption.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 39cc6c5..0663b2f 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -34,7 +34,7 @@ import matrix.globals try: from olm.account import Account, OlmAccountError from olm.session import (Session, InboundSession, OlmSessionError, - OlmMessage, OlmPreKeyMessage) + OlmPreKeyMessage) from olm.group_session import ( InboundGroupSession, OutboundGroupSession, @@ -318,7 +318,7 @@ class Olm(): is_new = False plaintext_dict["room_id"] = room_id - if not room_id in self.outbound_group_sessions: + if room_id not in self.outbound_group_sessions: self.create_outbound_group_session(room_id) is_new = True @@ -391,15 +391,19 @@ class Olm(): W.prnt("", pprint.pformat(device_payload_dict)) - olm_message = session.encrypt(Olm._to_json(device_payload_dict)) + olm_message = session.encrypt( + Olm._to_json(device_payload_dict) + ) olm_dict = { "algorithm": "m.olm.v1.curve25519-aes-sha2", "sender_key": self.account.identity_keys()["curve25519"], "ciphertext": { key.keys["curve25519"]: { - "type": (0 if isinstance(olm_message, - OlmPreKeyMessage) else 1), + "type": (0 if isinstance( + olm_message, + OlmPreKeyMessage + ) else 1), "body": olm_message.ciphertext } } @@ -410,7 +414,6 @@ class Olm(): } return to_device_dict - # return {} @classmethod @encrypt_enabled From 2bf9766276a3d26d41b1ff61e4e35fd541771eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Sat, 12 May 2018 11:45:31 +0200 Subject: [PATCH 023/269] encryption: Send the group session key before ratcheting it. --- matrix/encryption.py | 10 +++++----- matrix/server.py | 15 ++++++--------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 0663b2f..ea731d0 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -313,14 +313,14 @@ class Olm(): except OlmSessionError: return None - def group_encrypt(self, room_id, plaintext_dict): - # type: (str, Dict[str, str]) -> Dict[str, str], Bool - is_new = False + def group_encrypt(self, room_id, plaintext_dict, own_id, users): + # type: (str, Dict[str, str]) -> Dict[str, str], Optional[Dict[Any, Any]] plaintext_dict["room_id"] = room_id + to_device_dict = None if room_id not in self.outbound_group_sessions: self.create_outbound_group_session(room_id) - is_new = True + to_device_dict = self.share_group_session(room_id, own_id, users) session = self.outbound_group_sessions[room_id] @@ -334,7 +334,7 @@ class Olm(): "device_id": self.device_id } - return payload_dict, is_new + return payload_dict, to_device_dict @encrypt_enabled def group_decrypt(self, room_id, session_id, ciphertext): diff --git a/matrix/server.py b/matrix/server.py index 21918e4..897e89f 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -510,19 +510,16 @@ class MatrixServer: W.prnt("", "matrix: Encrypting message") - payload_dict, session_is_new = self.olm.group_encrypt( + payload_dict, to_device_dict = self.olm.group_encrypt( room_id, - plaintext_dict + plaintext_dict, + self.user_id, + room.users.keys() ) - if session_is_new: - to_device_dict = self.olm.share_group_session( - room_id, - self.user_id, - room.users.keys() - ) - message = MatrixToDeviceMessage(self.client, to_device_dict) + if to_device_dict: W.prnt("", "matrix: Megolm session missing for room.") + message = MatrixToDeviceMessage(self.client, to_device_dict) self.send_queue.append(message) message = MatrixEncryptedMessage( From 47fae308e03ed7360c897a0a8ccf0bd0fca88263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Tue, 15 May 2018 23:24:14 +0200 Subject: [PATCH 024/269] encryption: Send to device messages to every device of a user. --- matrix/encryption.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index ea731d0..d5f6e08 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -409,9 +409,10 @@ class Olm(): } } - to_device_dict["messages"][user] = { - key.device_id: olm_dict - } + if user not in to_device_dict["messages"]: + to_device_dict["messages"][user] = {} + + to_device_dict["messages"][user][key.device_id] = olm_dict return to_device_dict From c8a7b4815dabbe7701a52ee6bad579fda39ae290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Wed, 16 May 2018 11:21:59 +0200 Subject: [PATCH 025/269] encryption: Initial key claiming support. --- matrix/api.py | 12 +++++++----- matrix/encryption.py | 46 +++++++++++++++++++++++++++++++++++++++----- matrix/events.py | 33 ++++++++++++++++++++++++++----- matrix/server.py | 22 ++++++++++++++------- 4 files changed, 91 insertions(+), 22 deletions(-) diff --git a/matrix/api.py b/matrix/api.py index 69b416c..0bbf4e6 100644 --- a/matrix/api.py +++ b/matrix/api.py @@ -306,7 +306,7 @@ class MatrixClient: return HttpRequest(RequestType.POST, self.host, path, content) - def key_claim(self, key_dict): + def keys_claim(self, key_dict): query_parameters = {"access_token": self.access_token} path = ("{api}/keys/claim?" @@ -315,7 +315,8 @@ class MatrixClient: query_parameters=urlencode(query_parameters)) content = { - "one_time_keys": {key_dict} + "one_time_keys": key_dict, + "timeout": 10000 } return HttpRequest(RequestType.POST, self.host, path, content) @@ -661,16 +662,17 @@ class MatrixKeyQueryMessage(MatrixMessage): class MatrixKeyClaimMessage(MatrixMessage): - def __init__(self, client, key_dict): + def __init__(self, client, room_id, key_dict): + self.room_id = room_id data = { "key_dict": key_dict, } - MatrixMessage.__init__(self, client.keys_query, data) + MatrixMessage.__init__(self, client.keys_claim, data) def decode_body(self, server): object_hook = partial(MatrixEvents.MatrixKeyClaimEvent.from_dict, - server) + server, self.room_id) return self._decode(server, object_hook) diff --git a/matrix/encryption.py b/matrix/encryption.py index d5f6e08..70f460d 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -33,8 +33,8 @@ import matrix.globals try: from olm.account import Account, OlmAccountError - from olm.session import (Session, InboundSession, OlmSessionError, - OlmPreKeyMessage) + from olm.session import (Session, InboundSession, OutboundSession, + OlmSessionError, OlmPreKeyMessage) from olm.group_session import ( InboundGroupSession, OutboundGroupSession, @@ -193,6 +193,14 @@ class OlmDeviceKey(): self.keys = key_dict +class OneTimeKey(): + def __init__(self, user_id, device_id, key): + # type: (str, str, str) -> None + self.user_id = user_id + self.device_id = device_id + self.key = key + + class Olm(): @encrypt_enabled @@ -245,6 +253,28 @@ class Olm(): return session + def create_session(self, user_id, device_id, one_time_key): + W.prnt("", "matrix: Creating session for {}".format(user_id)) + id_key = None + + for user, keys in self.device_keys.items(): + if user != user_id: + continue + + for key in keys: + if key.device_id == device_id: + id_key = key.keys["curve25519"] + break + + if not id_key: + W.prnt("", "ERRR not found ID key") + W.prnt("", "Found id key {}".format(id_key)) + session = OutboundSession(self.account, id_key, one_time_key) + self._update_acc_in_db() + self.sessions[user_id][device_id].append(session) + self._store_session(user_id, device_id, session) + W.prnt("", "matrix: Created session for {}".format(user_id)) + def create_group_session(self, room_id, session_id, session_key): W.prnt("", "matrix: Creating group session for {}".format(room_id)) session = InboundGroupSession(session_key) @@ -270,10 +300,12 @@ class Olm(): continue if not self.sessions[user][key.device_id]: + W.prnt("", "Missing session for device {}".format(key.device_id)) devices.append(key.device_id) if devices: - missing[user] = {device: "ed25519" for device in devices} + missing[user] = {device: "signed_curve25519" for + device in devices} return missing @@ -376,11 +408,16 @@ class Olm(): } for user in users: + if user not in self.device_keys: + continue for key in self.device_keys[user]: if key.device_id == self.device_id: continue + if not self.sessions[user][key.device_id]: + continue + device_payload_dict = payload_dict.copy() # TODO sort the sessions session = self.sessions[user][key.device_id][0] @@ -389,8 +426,6 @@ class Olm(): "ed25519": key.keys["ed25519"] } - W.prnt("", pprint.pformat(device_payload_dict)) - olm_message = session.encrypt( Olm._to_json(device_payload_dict) ) @@ -414,6 +449,7 @@ class Olm(): to_device_dict["messages"][user][key.device_id] = olm_dict + # W.prnt("", pprint.pformat(to_device_dict)) return to_device_dict @classmethod diff --git a/matrix/events.py b/matrix/events.py index 36773a7..56cea83 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -31,7 +31,7 @@ from matrix.rooms import (matrix_create_room_buffer, RoomInfo, RoomMessageText, RoomMessageEvent, RoomRedactedMessageEvent, RoomMessageEmote) -from matrix.encryption import OlmDeviceKey +from matrix.encryption import OlmDeviceKey, OneTimeKey try: from olm.session import OlmMessage, OlmPreKeyMessage @@ -355,16 +355,38 @@ class MatrixKeyQueryEvent(MatrixEvent): class MatrixKeyClaimEvent(MatrixEvent): - def __init__(self, server, keys): + def __init__(self, server, room_id, keys): self.keys = keys + self.room_id = room_id MatrixEvent.__init__(self, server) @classmethod - def from_dict(cls, server, parsed_dict): - raise NotImplementedError + def from_dict(cls, server, room_id, parsed_dict): + W.prnt("", pprint.pformat(parsed_dict)) + keys = [] + try: + for user_id, user_dict in parsed_dict["one_time_keys"].items(): + for device_id, device_dict in user_dict.items(): + for key_dict in device_dict.values(): + # TODO check the signature of the key + key = OneTimeKey(user_id, device_id, key_dict["key"]) + keys.append(key) + + return cls(server, room_id, keys) + except KeyError: + return MatrixErrorEvent.from_dict( + server, ("Error claiming onetime keys."), False, parsed_dict) def execute(self): - pass + server = self.server + olm = server.olm + + for key in self.keys: + olm.create_session(key.user_id, key.device_id, key.key) + + while server.encryption_queue[self.room_id]: + formatted_message = server.encryption_queue[self.room_id].popleft() + server.send_room_message(self.room_id, formatted_message, True) class MatrixToDeviceEvent(MatrixEvent): @@ -384,6 +406,7 @@ class MatrixToDeviceEvent(MatrixEvent): "device message"), False, parsed_dict) + class MatrixBacklogEvent(MatrixEvent): def __init__(self, server, room_id, end_token, events): diff --git a/matrix/server.py b/matrix/server.py index 897e89f..b84bc0a 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -24,7 +24,7 @@ import time import datetime import pprint -from collections import deque +from collections import deque, defaultdict from http_parser.pyparser import HttpParser from matrix.plugin_options import Option, DebugType @@ -40,7 +40,8 @@ from matrix.api import ( MatrixKeyUploadMessage, MatrixKeyQueryMessage, MatrixToDeviceMessage, - MatrixEncryptedMessage + MatrixEncryptedMessage, + MatrixKeyClaimMessage ) from matrix.encryption import Olm, EncryptionError, encrypt_enabled @@ -63,7 +64,9 @@ class MatrixServer: self.options = dict() # type: Dict[str, weechat.config] self.device_name = "Weechat Matrix" # type: str self.device_id = "" # type: str + self.olm = None # type: Olm + self.encryption_queue = defaultdict(deque) self.user = "" # type: str self.password = "" # type: str @@ -478,7 +481,12 @@ class MatrixServer: message = MatrixSyncMessage(self.client, self.next_batch, limit) self.send_queue.append(message) - def send_room_message(self, room_id, formatted_data): + def send_room_message( + self, + room_id, + formatted_data, + already_claimed=False + ): # type: (str, Formatted) -> None room = self.rooms[room_id] @@ -488,13 +496,13 @@ class MatrixServer: # TODO don't send messages unless all the devices are verified missing = self.olm.get_missing_sessions(room.users.keys()) - if missing: + if missing and not already_claimed: W.prnt("", "{prefix}matrix: Olm session missing for room, can't" " encrypt message.") W.prnt("", pprint.pformat(missing)) - # message = MatrixKeyClaimMessage(self.client, missing) - # self.send_or_queue(message) - # TODO claim keys for the missing user/device combinations + self.encryption_queue[room_id].append(formatted_data) + message = MatrixKeyClaimMessage(self.client, room_id, missing) + self.send_or_queue(message) return body = {"msgtype": "m.text", "body": formatted_data.to_plain()} From dedff37a60e9433c327f2ad8a5f37fca77e16b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Wed, 16 May 2018 18:18:02 +0200 Subject: [PATCH 026/269] encryption: Add a device store class for the trust database. --- matrix/encryption.py | 158 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/matrix/encryption.py b/matrix/encryption.py index 70f460d..37111ab 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -29,6 +29,11 @@ from collections import defaultdict from functools import wraps from future.moves.itertools import zip_longest +try: + FileNotFoundError +except NameError: + FileNotFoundError = IOError + import matrix.globals try: @@ -185,6 +190,159 @@ class EncryptionError(Exception): pass +class DeviceStore(object): + def __init__(self, filename): + self._entries = [] + self._filename = filename + + self._load(filename) + + def _load(self, filename): + # type: (str) -> None + try: + with open(filename, "r") as f: + for line in f: + line = line.strip() + + if not line or line.startswith("#"): + continue + + entry = StoreEntry.from_line(line) + + if not entry: + continue + + self._entries.append(entry) + except FileNotFoundError: + pass + + def _save_store(f): + @wraps(f) + def decorated(*args, **kwargs): + self = args[0] + ret = f(*args, **kwargs) + self._save() + return ret + + return decorated + + def _save(self): + # type: (str) -> None + with open(self._filename, "w") as f: + for entry in self._entries: + line = entry.to_line() + f.write(line) + + @_save_store + def add(self, device): + # type: (OlmDeviceKey) -> None + new_entries = StoreEntry.from_olmdevice(device) + self._entries += new_entries + + # Remove duplicate entries + self._entries = list(set(self._entries)) + + self._save() + + @_save_store + def remove(self, device): + # type: (OlmDeviceKey) -> int + removed = 0 + entries = StoreEntry.from_olmdevice(device) + + for entry in entries: + if entry in self._entries: + self._entries.remove(entry) + removed += 1 + + self._save() + + return removed + + def check(self, device): + # type: (OlmDeviceKey) -> bool + entries = StoreEntry.from_olmdevice(device) + result = map(lambda entry: entry in self._entries, entries) + + if False in result: + return False + + return True + + +class StoreEntry(object): + def __init__(self, user_id, device_id, key_type, key): + # type: (str, str, str, str) -> None + self.user_id = user_id + self.device_id = device_id + self.key_type = key_type + self.key = key + + @classmethod + def from_line(cls, line): + # type: (str) -> StoreEntry + fields = line.split(' ') + + if len(fields) < 4: + return None + + user_id, device_id, key_type, key = fields[:4] + + if key_type == "matrix-ed25519": + return cls(user_id, device_id, "ed25519", key) + else: + return None + + @classmethod + def from_olmdevice(cls, device_key): + # type: (OlmDeviceKey) -> [StoreEntry] + entries = [] + + user_id = device_key.user_id + device_id = device_key.device_id + + for key_type, key in device_key.keys.items(): + if key_type == "ed25519": + entries.append(cls(user_id, device_id, "ed25519", key)) + + return entries + + def to_line(self): + # type: () -> str + key_type = "matrix-{}".format(self.key_type) + line = "{} {} {} {}".format( + self.user_id, + self.device_id, + key_type, + self.key + ) + return line + + def __hash__(self): + # type: () -> int + return hash(str(self)) + + def __str__(self): + # type: () -> str + key_type = "matrix-{}".format(self.key_type) + line = "{} {} {} {}".format( + self.user_id, + self.device_id, + key_type, + self.key + ) + return line + + def __eq__(self, value): + # type: (StoreEntry) -> bool + if (self.user_id == value.user_id + and self.device_id == value.device_id + and self.key_type == value.key_type and self.key == value.key): + return True + + return False + + class OlmDeviceKey(): def __init__(self, user_id, device_id, key_dict): # type: (str, str, Dict[str, str]) From d5b768e7e09c136f6e84c2a44045a1b7f0c015e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Wed, 16 May 2018 22:46:22 +0200 Subject: [PATCH 027/269] encryption: Port the olm command parser to argparse. --- matrix/encryption.py | 139 ++++++++++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 49 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 37111ab..7734838 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -21,6 +21,7 @@ import os import json import sqlite3 import pprint +import argparse # pylint: disable=redefined-builtin from builtins import str, bytes @@ -53,6 +54,22 @@ from matrix.utils import sanitize_id from matrix.utf import utf8_decode +class ParseError(Exception): + pass + + +class WeechatArgParse(argparse.ArgumentParser): + def print_usage(self, file): + pass + + def error(self, message): + m = ("{prefix}Error: {message} for command {command} " + "(see /help {command})").format(prefix=W.prefix("error"), + message=message, command=self.prog) + W.prnt("", m) + raise ParseError + + def own_buffer_or_error(f): @wraps(f) @@ -74,7 +91,6 @@ def own_buffer_or_error(f): def encrypt_enabled(f): - @wraps(f) def wrapper(*args, **kwds): if matrix.globals.ENCRYPTION: @@ -92,9 +108,9 @@ def matrix_hook_olm_command(): "Matrix olm encryption command", # Synopsis ("info all|blacklisted|private|unverified|verified ||" - "blacklist ||" - "unverify ||" - "verify "), + "blacklist ||" + "unverify ||" + "verify "), # Description (" info: show info about known devices and their keys\n" "blacklist: blacklist a device\n" @@ -104,21 +120,38 @@ def matrix_hook_olm_command(): # Completions ('info all|blacklisted|private|unverified|verified ||' 'blacklist %(device_ids) ||' - 'unverify %(device_ids) ||' - 'verify %(device_ids)'), + 'unverify %(user_ids) %(device_ids) ||' + 'verify %(user_ids) %(device_ids)'), # Function name 'matrix_olm_command_cb', '') def olm_cmd_parse_args(args): - split_args = args.split() + parser = WeechatArgParse(prog="olm") + subparsers = parser.add_subparsers(dest="subcommand") - command = split_args.pop(0) if split_args else "info" + info_parser = subparsers.add_parser("info") + info_parser.add_argument( + "category", nargs="?", default="private", + choices=[ + "all", + "blacklisted", + "private", + "unverified", + "verified" + ]) + info_parser.add_argument("filter", nargs="?") - rest_args = split_args if split_args else [] + verify_parser = subparsers.add_parser("verify") + verify_parser.add_argument("user_filter") + verify_parser.add_argument("device_filter", nargs="?") - return command, rest_args + try: + parsed_args = parser.parse_args(args.split()) + return parsed_args + except ParseError: + return None def grouper(iterable, n, fillvalue=None): @@ -133,51 +166,58 @@ def partition_key(key): return ' '.join(''.join(g) for g in groups) +def olm_info_command(server, args): + olm = server.olm + + if args.category == "private": + device_msg = (" - Device ID: {}\n".format(server.device_id) + if server.device_id else "") + id_key = partition_key(olm.account.identity_keys()["curve25519"]) + fp_key = partition_key(olm.account.identity_keys()["ed25519"]) + message = ("{prefix}matrix: Identity keys:\n" + " - User: {user}\n" + "{device_msg}" + " - Identity key: {id_key}\n" + " - Fingerprint key: {fp_key}\n").format( + prefix=W.prefix("network"), + user=server.user, + device_msg=device_msg, + id_key=id_key, + fp_key=fp_key) + W.prnt(server.server_buffer, message) + elif args.category == "all": + for user, keys in olm.device_keys.items(): + message = ("{prefix}matrix: Identity keys:\n" + " - User: {user}\n").format( + prefix=W.prefix("network"), + user=user) + W.prnt(server.server_buffer, message) + + for key in keys: + id_key = partition_key(key.keys["curve25519"]) + fp_key = partition_key(key.keys["ed25519"]) + device_msg = (" - Device ID: {}\n".format( + key.device_id) if key.device_id else "") + message = ("{device_msg}" + " - Identity key: {id_key}\n" + " - Fingerprint key: {fp_key}\n\n").format( + device_msg=device_msg, + id_key=id_key, + fp_key=fp_key) + W.prnt(server.server_buffer, message) + + @own_buffer_or_error @utf8_decode def matrix_olm_command_cb(server_name, buffer, args): server = SERVERS[server_name] - command, args = olm_cmd_parse_args(args) + parsed_args = olm_cmd_parse_args(args) - if not command or command == "info": - olm = server.olm + if not parsed_args: + return W.WEECHAT_RC_OK - if not args or args[0] == "private": - device_msg = (" - Device ID: {}\n".format(server.device_id) - if server.device_id else "") - id_key = partition_key(olm.account.identity_keys()["curve25519"]) - fp_key = partition_key(olm.account.identity_keys()["ed25519"]) - message = ("{prefix}matrix: Identity keys:\n" - " - User: {user}\n" - "{device_msg}" - " - Identity key: {id_key}\n" - " - Fingerprint key: {fp_key}\n").format( - prefix=W.prefix("network"), - user=server.user, - device_msg=device_msg, - id_key=id_key, - fp_key=fp_key) - W.prnt(server.server_buffer, message) - elif args[0] == "all": - for user, keys in olm.device_keys.items(): - message = ("{prefix}matrix: Identity keys:\n" - " - User: {user}\n").format( - prefix=W.prefix("network"), - user=user) - W.prnt(server.server_buffer, message) - - for key in keys: - id_key = partition_key(key.keys["curve25519"]) - fp_key = partition_key(key.keys["ed25519"]) - device_msg = (" - Device ID: {}\n".format( - key.device_id) if key.device_id else "") - message = ("{device_msg}" - " - Identity key: {id_key}\n" - " - Fingerprint key: {fp_key}\n\n").format( - device_msg=device_msg, - id_key=id_key, - fp_key=fp_key) - W.prnt(server.server_buffer, message) + if not parsed_args.subcommand or parsed_args.subcommand == "info": + olm_info_command(server, parsed_args) else: message = ("{prefix}matrix: Command not implemented.".format( prefix=W.prefix("error"))) @@ -192,6 +232,7 @@ class EncryptionError(Exception): class DeviceStore(object): def __init__(self, filename): + # type: (str) -> None self._entries = [] self._filename = filename From bb06293031422c5821f33638b115b641c0bb7417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 17 May 2018 12:59:07 +0200 Subject: [PATCH 028/269] encryption: Initial device verification support. --- matrix/encryption.py | 66 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 7734838..cd3ffa7 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -27,6 +27,7 @@ import argparse from builtins import str, bytes from collections import defaultdict +from itertools import chain from functools import wraps from future.moves.itertools import zip_longest @@ -207,6 +208,45 @@ def olm_info_command(server, args): W.prnt(server.server_buffer, message) +def olm_verify_command(server, args): + olm = server.olm + devices = olm.device_keys + filtered_devices = [] + + if args.user_filter == "*": + filtered_devices = devices.values() + else: + device_keys = filter(lambda x: args.user_filter in x, devices) + filtered_devices = [devices[x] for x in device_keys] + + filtered_devices = chain.from_iterable(filtered_devices) + + if args.device_filter and args.device_filter != "*": + filtered_devices = filter( + lambda x: args.device_filter in x.device_id, + filtered_devices + ) + + key_list = [] + + for device in list(filtered_devices): + if olm.verify_device(device): + key_list.append(str(device)) + + if not key_list: + message = ("{prefix}matrix: No matching unverified devices " + "found.").format(prefix=W.prefix("error")) + W.prnt(server.server_buffer, message) + return + + key_str = "\n - ".join(key_list) + message = ("{prefix}matrix: Verified keys:\n" + " - {key_str}").format(prefix=W.prefix("join"), + key_str=key_str) + + W.prnt(server.server_buffer, message) + + @own_buffer_or_error @utf8_decode def matrix_olm_command_cb(server_name, buffer, args): @@ -218,6 +258,8 @@ def matrix_olm_command_cb(server_name, buffer, args): if not parsed_args.subcommand or parsed_args.subcommand == "info": olm_info_command(server, parsed_args) + elif parsed_args.subcommand == "verify": + olm_verify_command(server, parsed_args) else: message = ("{prefix}matrix: Command not implemented.".format( prefix=W.prefix("error"))) @@ -351,7 +393,7 @@ class StoreEntry(object): def to_line(self): # type: () -> str key_type = "matrix-{}".format(self.key_type) - line = "{} {} {} {}".format( + line = "{} {} {} {}\n".format( self.user_id, self.device_id, key_type, @@ -391,6 +433,15 @@ class OlmDeviceKey(): self.device_id = device_id self.keys = key_dict + def __str__(self): + # type: () -> str + return "{} {} {}".format( + self.user_id, self.device_id, self.keys["ed25519"]) + + def __repr__(self): + # type: () -> str + return str(self) + class OneTimeKey(): def __init__(self, user_id, device_id, key): @@ -443,6 +494,9 @@ class Olm(): self.inbound_group_sessions = inbound_group_sessions self.outbound_group_sessions = {} + trust_file_path = "{}_{}.trusted_devices".format(user, device_id) + self.trust_db = DeviceStore(os.path.join(session_path, trust_file_path)) + def _create_session(self, sender, sender_key, message): W.prnt("", "matrix: Creating session for {}".format(sender)) session = InboundSession(self.account, message, sender_key) @@ -452,6 +506,16 @@ class Olm(): return session + def verify_device(self, device): + if self.trust_db.check(device): + return False + + self.trust_db.add(device) + return True + + def unverify_device(self, device): + self.trust_db.remove(device) + def create_session(self, user_id, device_id, one_time_key): W.prnt("", "matrix: Creating session for {}".format(user_id)) id_key = None From c8fb416f88ee58fc55ce191df8223b8c464cc8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 17 May 2018 14:34:34 +0200 Subject: [PATCH 029/269] encryption: Disallow sending messages until the devices are verified. --- matrix/encryption.py | 13 ++++++++++++ matrix/server.py | 49 ++++++++++++++++++++++++++++---------------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index cd3ffa7..292689d 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -59,6 +59,10 @@ class ParseError(Exception): pass +class OlmTrustError(Exception): + pass + + class WeechatArgParse(argparse.ArgumentParser): def print_usage(self, file): pass @@ -470,6 +474,7 @@ class Olm(): self.session_path = session_path self.database = database self.device_keys = {} + self.shared_sessions = [] if not database: db_file = "{}_{}.db".format(user, device_id) @@ -615,7 +620,12 @@ class Olm(): if room_id not in self.outbound_group_sessions: self.create_outbound_group_session(room_id) + + if self.outbound_group_sessions[room_id].id not in self.shared_sessions: to_device_dict = self.share_group_session(room_id, own_id, users) + self.shared_sessions.append( + self.outbound_group_sessions[room_id].id + ) session = self.outbound_group_sessions[room_id] @@ -681,6 +691,9 @@ class Olm(): if not self.sessions[user][key.device_id]: continue + if not self.trust_db.check(key): + raise OlmTrustError + device_payload_dict = payload_dict.copy() # TODO sort the sessions session = self.sessions[user][key.device_id][0] diff --git a/matrix/server.py b/matrix/server.py index b84bc0a..98c6c69 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -44,7 +44,12 @@ from matrix.api import ( MatrixKeyClaimMessage ) -from matrix.encryption import Olm, EncryptionError, encrypt_enabled +from matrix.encryption import ( + Olm, + EncryptionError, + OlmTrustError, + encrypt_enabled +) try: FileNotFoundError @@ -518,26 +523,34 @@ class MatrixServer: W.prnt("", "matrix: Encrypting message") - payload_dict, to_device_dict = self.olm.group_encrypt( - room_id, - plaintext_dict, - self.user_id, - room.users.keys() - ) + try: + payload_dict, to_device_dict = self.olm.group_encrypt( + room_id, + plaintext_dict, + self.user_id, + room.users.keys() + ) + + if to_device_dict: + W.prnt("", "matrix: Megolm session missing for room.") + message = MatrixToDeviceMessage(self.client, to_device_dict) + self.send_queue.append(message) + + message = MatrixEncryptedMessage( + self.client, + room_id, + formatted_data, + payload_dict + ) - if to_device_dict: - W.prnt("", "matrix: Megolm session missing for room.") - message = MatrixToDeviceMessage(self.client, to_device_dict) self.send_queue.append(message) - message = MatrixEncryptedMessage( - self.client, - room_id, - formatted_data, - payload_dict - ) - - self.send_queue.append(message) + except OlmTrustError: + m = ("{prefix}matrix: Untrusted devices found in room, " + "verification is needed before sending a message").format( + prefix=W.prefix("error")) + W.prnt(self.server_buffer, m) + return @encrypt_enabled def upload_keys(self, device_keys=False, one_time_keys=False): From 87a8c1a5c0d3c4a4b1134711531086eaf04eaf74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 17 May 2018 16:33:58 +0200 Subject: [PATCH 030/269] encryption: Add completion for device verification. --- main.py | 3 ++- matrix/completion.py | 59 ++++++++++++++++++++++++++++++++++++++++++++ matrix/encryption.py | 2 +- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 5b9b02a..12007ad 100644 --- a/main.py +++ b/main.py @@ -64,7 +64,8 @@ from matrix.bar_items import (init_bar_items, matrix_bar_item_name, from matrix.completion import ( init_completion, matrix_command_completion_cb, matrix_server_command_completion_cb, matrix_debug_completion_cb, - matrix_message_completion_cb, matrix_server_completion_cb) + matrix_message_completion_cb, matrix_server_completion_cb, + matrix_olm_user_completion_cb, matrix_olm_device_completion_cb) from matrix.utils import (key_from_value, server_buffer_prnt, prnt_debug, server_buffer_set_title) diff --git a/matrix/completion.py b/matrix/completion.py index 8212f41..454664b 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -118,6 +118,59 @@ def matrix_message_completion_cb(data, completion_item, buffer, completion): return W.WEECHAT_RC_OK +def server_from_buffer(buffer): + for server in SERVERS.values(): + if buffer in server.buffers.values(): + return server + elif buffer == server.server_buffer: + return server + return None + + +@utf8_decode +def matrix_olm_user_completion_cb(data, completion_item, buffer, completion): + server = server_from_buffer(buffer) + + if not server: + return W.WEECHAT_RC_OK + + olm = server.olm + + for user in olm.device_keys: + W.hook_completion_list_add(completion, user, 0, + W.WEECHAT_LIST_POS_SORT) + + return W.WEECHAT_RC_OK + + +@utf8_decode +def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): + server = server_from_buffer(buffer) + + if not server: + return W.WEECHAT_RC_OK + + olm = server.olm + + args = W.hook_completion_get_string(completion, "args") + + fields = args.split() + + if len(fields) < 2: + return W.WEECHAT_RC_OK + + user = fields[1] + + if user not in olm.device_keys: + return W.WEECHAT_RC_OK + + for device in olm.device_keys[user]: + W.hook_completion_list_add(completion, device.device_id, 0, + W.WEECHAT_LIST_POS_SORT) + + return W.WEECHAT_RC_OK + + def init_completion(): W.hook_completion("matrix_server_commands", "Matrix server completion", "matrix_server_command_completion_cb", "") @@ -133,3 +186,9 @@ def init_completion(): W.hook_completion("matrix_debug_types", "Matrix debugging type completion", "matrix_debug_completion_cb", "") + + W.hook_completion("olm_user_ids", "Matrix olm user id completion", + "matrix_olm_user_completion_cb", "") + + W.hook_completion("olm_devices", "Matrix olm device id completion", + "matrix_olm_device_completion_cb", "") diff --git a/matrix/encryption.py b/matrix/encryption.py index 292689d..9c867b0 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -126,7 +126,7 @@ def matrix_hook_olm_command(): ('info all|blacklisted|private|unverified|verified ||' 'blacklist %(device_ids) ||' 'unverify %(user_ids) %(device_ids) ||' - 'verify %(user_ids) %(device_ids)'), + 'verify %(olm_user_ids) %(olm_devices)'), # Function name 'matrix_olm_command_cb', '') From 86ae77c38c71d6804b6855c72b04c75611614175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Mon, 4 Jun 2018 22:34:18 +0200 Subject: [PATCH 031/269] encryption: Update for some olm lib API changes. --- matrix/encryption.py | 18 +++++++++++------- matrix/server.py | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 9c867b0..2970041 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -39,10 +39,14 @@ except NameError: import matrix.globals try: - from olm.account import Account, OlmAccountError - from olm.session import (Session, InboundSession, OutboundSession, - OlmSessionError, OlmPreKeyMessage) - from olm.group_session import ( + from olm import ( + Account, + OlmAccountError, + Session, + InboundSession, + OutboundSession, + OlmSessionError, + OlmPreKeyMessage, InboundGroupSession, OutboundGroupSession, OlmGroupSessionError @@ -787,7 +791,7 @@ class Olm(): cursor.execute("""update olmsessions set pickle=? where user = ? and session_id = ? and device_id = ?""", - (session.pickle(), user, session.id(), + (session.pickle(), user, session.id, device_id)) self.database.commit() @@ -800,7 +804,7 @@ class Olm(): for session in session_dict.values(): cursor.execute("""update inbound_group_sessions set pickle=? where room_id = ? and session_id = ?""", - (session.pickle(), room_id, session.id())) + (session.pickle(), room_id, session.id)) self.database.commit() cursor.close() @@ -809,7 +813,7 @@ class Olm(): cursor = self.database.cursor() cursor.execute("insert into olmsessions values(?,?,?,?)", - (user, device_id, session.id(), session.pickle())) + (user, device_id, session.id, session.pickle())) self.database.commit() diff --git a/matrix/server.py b/matrix/server.py index 98c6c69..163b4be 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -556,7 +556,7 @@ class MatrixServer: def upload_keys(self, device_keys=False, one_time_keys=False): keys = self.olm.account.identity_keys() if device_keys else None - one_time_keys = (self.olm.account.one_time_keys()["curve25519"] if + one_time_keys = (self.olm.account.one_time_keys["curve25519"] if one_time_keys else None) message = MatrixKeyUploadMessage(self.client, self.user_id, @@ -566,7 +566,7 @@ class MatrixServer: @encrypt_enabled def check_one_time_keys(self, key_count): - max_keys = self.olm.account.max_one_time_keys() + max_keys = self.olm.account.max_one_time_keys key_count = (max_keys / 2) - key_count From 5c6389fc47aece6a734b30358ba66e5570570425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 7 Jun 2018 14:43:38 +0200 Subject: [PATCH 032/269] encryption: Fix account loading for python3. --- matrix/encryption.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 2970041..3733aca 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -759,15 +759,31 @@ class Olm(): inbound_group_sessions = defaultdict(dict) try: - account = Account.from_pickle(bytes(account_pickle, "utf-8")) + try: + account_pickle = bytes(account_pickle, "utf-8") + except TypeError: + pass + + account = Account.from_pickle(account_pickle) for db_session in db_sessions: + session_pickle = db_session[2] + try: + session_pickle = bytes(session_pickle, "utf-8") + except TypeError: + pass + sessions[db_session[0]][db_session[1]].append( - Session.from_pickle(bytes(db_session[2], "utf-8"))) + Session.from_pickle(session_pickle)) for db_session in db_inbound_group_sessions: - session = InboundGroupSession.from_pickle( - bytes(db_session[1], "utf-8")) + session_pickle = db_session[1] + try: + session_pickle = bytes(session_pickle, "utf-8") + except TypeError: + pass + + session = InboundGroupSession.from_pickle(session_pickle) inbound_group_sessions[db_session[0]][session.id] = session return cls(user, device_id, session_path, database, account, From 90a9e607b18549232ca7083dec3f65fa155ec89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Mon, 18 Jun 2018 20:41:18 +0200 Subject: [PATCH 033/269] encryption: Switch to the new olm API. --- matrix/encryption.py | 10 +++++----- matrix/events.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 3733aca..9ff5112 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -181,8 +181,8 @@ def olm_info_command(server, args): if args.category == "private": device_msg = (" - Device ID: {}\n".format(server.device_id) if server.device_id else "") - id_key = partition_key(olm.account.identity_keys()["curve25519"]) - fp_key = partition_key(olm.account.identity_keys()["ed25519"]) + id_key = partition_key(olm.account.identity_keys["curve25519"]) + fp_key = partition_key(olm.account.identity_keys["ed25519"]) message = ("{prefix}matrix: Identity keys:\n" " - User: {user}\n" "{device_msg}" @@ -637,7 +637,7 @@ class Olm(): payload_dict = { "algorithm": "m.megolm.v1.aes-sha2", - "sender_key": self.account.identity_keys()["curve25519"], + "sender_key": self.account.identity_keys["curve25519"], "ciphertext": ciphertext, "session_id": session.id, "device_id": self.device_id @@ -676,7 +676,7 @@ class Olm(): "sender": own_id, "sender_device": self.device_id, "keys": { - "ed25519": self.account.identity_keys()["ed25519"] + "ed25519": self.account.identity_keys["ed25519"] } } @@ -712,7 +712,7 @@ class Olm(): olm_dict = { "algorithm": "m.olm.v1.curve25519-aes-sha2", - "sender_key": self.account.identity_keys()["curve25519"], + "sender_key": self.account.identity_keys["curve25519"], "ciphertext": { key.keys["curve25519"]: { "type": (0 if isinstance( diff --git a/matrix/events.py b/matrix/events.py index 56cea83..9b4330e 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -553,7 +553,7 @@ class MatrixSyncEvent(MatrixEvent): @staticmethod def _get_olm_device_event(server, parsed_dict): - device_key = server.olm.account.identity_keys()["curve25519"] + device_key = server.olm.account.identity_keys["curve25519"] if device_key not in parsed_dict["content"]["ciphertext"]: return None @@ -585,7 +585,7 @@ class MatrixSyncEvent(MatrixEvent): if (sender != decrypted_sender or server.user_id != decrypted_recepient or - olm.account.identity_keys()["ed25519"] != + olm.account.identity_keys["ed25519"] != decrypted_recepient_key): error_message = ("{prefix}matrix: Mismatch in decrypted Olm " "message").format(prefix=W.prefix("error")) From e6e996997ff013547a4f9728bab0ca7b1bc1343c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Mon, 18 Jun 2018 20:41:46 +0200 Subject: [PATCH 034/269] rooms: Encrypted events can be redacted as well. --- matrix/rooms.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/matrix/rooms.py b/matrix/rooms.py index 3313f12..86c616d 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -250,7 +250,9 @@ class RoomInfo(): state_event = None message_event = None - if event_dict["type"] == "m.room.message": + if "redacted_by" in event_dict["unsigned"]: + message_event = RoomRedactedMessageEvent.from_dict(event_dict) + elif event_dict["type"] == "m.room.message": message_event = RoomInfo._message_from_event(event_dict) elif event_dict["type"] == "m.room.member": state_event, message_event = ( @@ -309,10 +311,11 @@ class RoomInfo(): message_events.append(s_event) except (ValueError, TypeError, KeyError) as error: message = ("{prefix}matrix: Error parsing " - "room event of type {type}: {error}").format( + "room event of type {type}: {error}\n{event}").format( prefix=W.prefix("error"), type=event["type"], - error=pformat(error)) + error=pformat(error), + event=pformat(event)) W.prnt("", message) raise From e7208ded623eeb9eec8f2f2110d00268d3110808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Wed, 4 Jul 2018 22:09:43 +0200 Subject: [PATCH 035/269] matrix: Add weechat buffer abstractions. --- main.py | 4 +- matrix/_weechat.py | 82 ++++++--- matrix/buffer.py | 402 +++++++++++++++++++++++++++++++++++++++++++++ matrix/globals.py | 1 + 4 files changed, 468 insertions(+), 21 deletions(-) create mode 100644 matrix/buffer.py diff --git a/main.py b/main.py index 12007ad..7d924c8 100644 --- a/main.py +++ b/main.py @@ -78,10 +78,10 @@ from matrix.config import (matrix_config_init, matrix_config_read, import matrix.globals -from matrix.globals import W, SERVERS +from matrix.globals import W, SERVERS, SCRIPT_NAME # yapf: disable -WEECHAT_SCRIPT_NAME = "matrix" # type: str +WEECHAT_SCRIPT_NAME = SCRIPT_NAME WEECHAT_SCRIPT_DESCRIPTION = "matrix chat plugin" # type: str WEECHAT_SCRIPT_AUTHOR = "Damir Jelić " # type: str WEECHAT_SCRIPT_VERSION = "0.1" # type: str diff --git a/matrix/_weechat.py b/matrix/_weechat.py index 7b17d8c..6914e37 100644 --- a/matrix/_weechat.py +++ b/matrix/_weechat.py @@ -1,25 +1,31 @@ +import random +import string +import datetime + + +weechat_base_colors = { + "black": "0", + "red": "1", + "green": "2", + "brown": "3", + "blue": "4", + "magenta": "5", + "cyan": "6", + "default": "7", + "gray": "8", + "lightred": "9", + "lightgreen": "10", + "yellow": "11", + "lightblue": "12", + "lightmagenta": "13", + "lightcyan": "14", + "white": "15" +} + + def color(color_name): # type: (str) -> str # yapf: disable - weechat_base_colors = { - "black": "0", - "red": "1", - "green": "2", - "brown": "3", - "blue": "4", - "magenta": "5", - "cyan": "6", - "default": "7", - "gray": "8", - "lightred": "9", - "lightgreen": "10", - "yellow": "11", - "lightblue": "12", - "lightmagenta": "13", - "lightcyan": "14", - "white": "15" - } - escape_codes = [] reset_code = "0" @@ -111,6 +117,15 @@ def prnt(_, string): print(string) +def prnt_date_tags(_, date, tags_string, data): + message = "{} {} [{}]".format( + datetime.datetime.fromtimestamp(date), + data, + tags_string + ) + print(message) + + def config_search_section(*args, **kwargs): pass @@ -124,4 +139,33 @@ def mkdir_home(*args, **kwargs): def info_get(info, *args): + if info == "nick_color_name": + return random.choice(list(weechat_base_colors.keys())) + return "" + + +def buffer_new(*args, **kwargs): + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(8) + ) + + +def buffer_set(*args, **kwargs): + return + + +def nicklist_add_group(*args, **kwargs): + return + + +def nicklist_add_nick(*args, **kwargs): + return + + +def nicklist_remove_nick(*args, **kwargs): + return + + +def nicklist_search_nick(*args, **kwargs): + return buffer_new(args, kwargs) diff --git a/matrix/buffer.py b/matrix/buffer.py new file mode 100644 index 0000000..cb20d62 --- /dev/null +++ b/matrix/buffer.py @@ -0,0 +1,402 @@ +# -*- coding: utf-8 -*- + +# Weechat Matrix Protocol Script +# Copyright © 2018 Damir Jelić +# +# Permission to use, copy, modify, and/or distribute this software for +# any purpose with or without fee is hereby granted, provided that the +# above copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import unicode_literals + +import time + +from .globals import W, SERVERS, SCRIPT_NAME +from .utf import utf8_decode +from .colors import Formatted +from builtins import super + + +@utf8_decode +def room_buffer_input_cb(server_name, buffer, input_data): + server = SERVERS[server_name] + room, room_buffer = server.find_room(buffer) + + if not room_buffer: + # TODO log error + return + + if not server.connected: + room_buffer.error("You are not connected to the server") + return W.WEECHAT_RC_ERROR + + formatted_data = Formatted.from_input_line(input_data) + + if room.encrypted: + server.send_room_message(room.id, formatted_data) + return W.WEECHAT_RC_OK + + return W.WEECHAT_RC_OK + + +@utf8_decode +def room_buffer_close_cb(data, buffer): + return W.WEECHAT_RC_OK + + +class WeechatUser(object): + def __init__(self, nick, host=None, prefix=""): + # type: (str, str, str) -> None + self.nick = nick + self.host = host + self.prefix = prefix + self.color = W.info_get("nick_color_name", nick) + + +class RoomUser(WeechatUser): + def __init__(self, nick, user_id=None, power_level=0): + # type: (str, str, int) -> None + prefix = self._get_prefix(power_level) + return super().__init__(nick, user_id, prefix) + + @staticmethod + def _get_prefix(power_level): + # type: (int) -> str + if power_level >= 100: + return "&" + elif power_level >= 50: + return "@" + elif power_level > 0: + return "+" + return "" + + +class WeechatChannelBuffer(object): + tags = { + "message": [ + SCRIPT_NAME + "_message", + "notify_message", + "log1" + ], + "old_message": [ + SCRIPT_NAME + "_message", + "notify_message", + "no_log", + "no_highlight" + ], + "join": [ + SCRIPT_NAME + "_join", + "log4" + ], + "part": [ + SCRIPT_NAME + "_leave", + "log4" + ], + "kick": [ + SCRIPT_NAME + "_kick", + "log4" + ], + "invite": [ + SCRIPT_NAME + "_invite", + "log4" + ] + } + + membership_messages = { + "join": "has joined", + "part": "has left", + "kick": "has been kicked", + "invite": "has been invited" + } + + def __init__(self, name, server_name, user): + # type: (str, str, str) + self._ptr = W.buffer_new( + name, + "room_buffer_input_cb", + server_name, + "room_buffer_input_cb", + server_name, + ) + + self.name = "" + self.users = {} # type: Dict[str, RoomUser] + + W.buffer_set(self._ptr, "localvar_set_type", 'channel') + W.buffer_set(self._ptr, "type", 'formatted') + + W.buffer_set(self._ptr, "localvar_set_channel", name) + + W.buffer_set(self._ptr, "localvar_set_nick", user) + + W.buffer_set(self._ptr, "localvar_set_server", server_name) + + # short_name = strip_matrix_server(room_id) + # W.buffer_set(self._ptr, "short_name", short_name) + + W.nicklist_add_group( + self._ptr, + '', + "000|o", + "weechat.color.nicklist_group", + 1 + ) + W.nicklist_add_group( + self._ptr, + '', + "001|h", + "weechat.color.nicklist_group", + 1 + ) + W.nicklist_add_group( + self._ptr, + '', + "002|v", + "weechat.color.nicklist_group", + 1 + ) + W.nicklist_add_group( + self._ptr, + '', + "999|...", + "weechat.color.nicklist_group", + 1 + ) + + W.buffer_set(self._ptr, "nicklist", "1") + W.buffer_set(self._ptr, "nicklist_display_groups", "0") + + # TODO make this configurable + W.buffer_set( + self._ptr, + "highlight_tags_restrict", + SCRIPT_NAME + "_message" + ) + + def _print(self, string): + # type: (str) -> None + """ Print a string to the room buffer """ + W.prnt(self._ptr, string) + + def print_date_tags(self, data, date=None, tags=None): + # type: (str, Optional[int], Optional[List[str]]) -> None + date = date or int(time.time()) + tags = tags or [] + + tags_string = ",".join(tags) + W.prnt_date_tags(self._ptr, date, tags_string, data) + + def error(self, string): + # type: (str) -> None + """ Print an error to the room buffer """ + message = "{prefix}{script}: {message}".format( + W.prefix("error"), + SCRIPT_NAME, + string + ) + + self._print(message) + + @staticmethod + def _color_for_tags(color): + # type: (str) -> str + if color == "weechat.color.chat_nick_self": + option = W.config_get(color) + return W.config_string(option) + + return color + + def _message_tags(self, user, message_type): + # type: (str, RoomUser, str) -> List[str] + tags = self.tags[message_type].copy() + + tags.append("nick_{nick}".format(nick=user.nick)) + + color = self._color_for_tags(user.color) + + if message_type != "action": + tags.append("prefix_nick_{color}".format(color=color)) + + return tags + + def _get_user(self, nick): + # type: (str) -> RoomUser + if nick in self.users: + return self.users[nick] + + # A message from a non joined user + return RoomUser(nick) + + def message(self, nick, message, date): + # type: (str, str, int, str) -> None + user = self._get_user(nick) + tags = self._message_tags(user, "message") + + prefix_string = ("" if not user.prefix else "{}{}{}".format( + W.color(self._get_prefix_color(user.prefix)), + user.prefix, + W.color("reset") + )) + + data = "{prefix}{color}{author}{ncolor}\t{msg}".format( + prefix=prefix_string, + color=W.color(user.color), + author=user.nick, + ncolor=W.color("reset"), + msg=message) + + self.print_date_tags(data, date, tags) + + def notice(self, nick, message, date): + # type: (str, str, int) -> None + data = "{color}{message}{ncolor}".format( + color=W.color("irc.color.notice"), + message=message, + ncolor=W.color("reset")) + + self.message(nick, data, date) + + def action(self, nick, message, date): + # type: (str, str, int) -> None + user = self._get_user(nick) + tags = self._message_tags(user, "action") + + nick_prefix = ("" if not user.prefix else "{}{}{}".format( + W.color(self._get_prefix_color(user.prefix)), + user.prefix, + W.color("reset") + )) + + data = ("{prefix}{nick_prefix}{nick_color}{author}" + "{ncolor} {msg}").format( + prefix=W.prefix("action"), + nick_prefix=nick_prefix, + nick_color=W.color(user.color), + author=nick, + ncolor=W.color("reset"), + msg=self.message) + + self.print_date_tags(data, date, tags) + + @staticmethod + def _get_nicklist_group(user): + # type: (WeechatUser) -> str + group_name = "999|..." + + if user.prefix == "&": + group_name = "000|o" + elif user.power_level == "@": + group_name = "001|h" + elif user.power_level > "+": + group_name = "002|v" + + return group_name + + @staticmethod + def _get_prefix_color(prefix): + # type: (str) -> str + # TODO make this configurable + color = "" + + if prefix == "&": + color = "lightgreen" + elif prefix == "@": + color = "lightgreen" + elif prefix == "+": + color = "yellow" + + return color + + def _add_user_to_nicklist(self, user): + # type: (WeechatUser) -> None + nick_pointer = W.nicklist_search_nick(self._ptr, "", user.nick) + + if not nick_pointer: + group = W.nicklist_search_group( + self._ptr, + "", + self._get_nicklist_group(user) + ) + prefix = user.prefix if user.prefix else " " + W.nicklist_add_nick( + self._ptr, + group, + user.nick, + user.color, + prefix, + self._get_prefix_color(user.prefix), + 1 + ) + + def _membership_message(self, user, message_type): + # type: (WeechatUser, str) -> str + action_color = ("green" if message_type == "join" + or message_type == "invite" else "red") + + membership_message = self.membership_messages[message_type] + + message = ("{prefix}{color}{author}{ncolor} " + "{del_color}({host_color}{host}{del_color})" + "{action_color} {message} " + "{channel_color}{room}{ncolor}").format( + prefix=W.prefix(message_type), + color=W.color(user.color), + author=user.nick, + ncolor=W.color("reset"), + del_color=W.color("chat_delimiters"), + host_color=W.color("chat_host"), + host=user.host, + action_color=W.color(action_color), + message=membership_message, + channel_color=W.color("chat_channel"), + room=self.name) + + return message + + def join(self, user, date, message=True, extra_tags=[]): + # type: (WeechatUser, int, Optional[bool], Optional[List[str]]) -> None + self._add_user_to_nicklist(user) + self.users[user.nick] = user + + if message: + tags = self._message_tags(user, "join") + message = self._membership_message(user, "join") + self.print_date_tags(message, date, tags) + + def invite(self, user, date, extra_tags=[]): + # type: (WeechatUser, int, Optional[bool], Optional[List[str]]) -> None + tags = self._message_tags(user, "invite") + message = self._membership_message(user, "invite") + self.print_date_tags(message, date, tags + extra_tags) + + def _remove_user_from_nicklist(self, user): + # type: (WeechatUser) -> None + pass + + def _leave(self, user, date, message, leave_type, extra_tags): + # type: (WeechatUser, int, bool, str, List[str]) -> None + self._remove_user_from_nicklist(user) + + if message: + tags = self._message_tags(user, leave_type) + message = self._membership_message(user, leave_type) + self.print_date_tags(message, date, tags + extra_tags) + + if user.nick in self.users: + del self.users[user.nick] + + def part(self, user, date, message=True, extra_tags=[]): + # type: (WeechatUser, int, Optional[bool], Optional[List[str]]) -> None + self._leave(user, date, message, "leave", extra_tags) + + def kick(self, user, date, message=True, extra_tags=[]): + # type: (WeechatUser, int, Optional[bool], Optional[List[str]]) -> None + self._leave(user, date, message, "kick", extra_tags=[]) diff --git a/matrix/globals.py b/matrix/globals.py index e7f7a29..efca89e 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -32,3 +32,4 @@ OPTIONS = PluginOptions() # type: PluginOptions SERVERS = dict() # type: Dict[str, MatrixServer] CONFIG = None # type: weechat.config ENCRYPTION = True # type: bool +SCRIPT_NAME = "matrix" # type: str From 38d6a14a33702fbb7bcef587181e51b8ff44145e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 5 Jul 2018 15:13:19 +0200 Subject: [PATCH 036/269] matrix: Change the way responses and events are executed. --- main.py | 1 + matrix/buffer.py | 77 ++++++++++++++++++++---- matrix/encryption.py | 5 +- matrix/events.py | 5 +- matrix/rooms.py | 138 +++++++++++++------------------------------ matrix/server.py | 124 ++++++++++++++++++++++++++++++++------ 6 files changed, 218 insertions(+), 132 deletions(-) diff --git a/main.py b/main.py index 7d924c8..ac6c95b 100644 --- a/main.py +++ b/main.py @@ -46,6 +46,7 @@ from matrix.commands import (hook_commands, hook_page_up, matrix_command_cb, matrix_command_pgup_cb, matrix_redact_command_cb, matrix_command_buf_clear_cb, matrix_me_command_cb, matrix_command_kick_cb) +from matrix.buffer import room_buffer_input_cb, room_buffer_close_cb from matrix.server import ( MatrixServer, diff --git a/matrix/buffer.py b/matrix/buffer.py index cb20d62..447235c 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -28,7 +28,7 @@ from builtins import super @utf8_decode def room_buffer_input_cb(server_name, buffer, input_data): server = SERVERS[server_name] - room, room_buffer = server.find_room(buffer) + room, room_buffer = server.find_room_from_ptr(buffer) if not room_buffer: # TODO log error @@ -40,9 +40,7 @@ def room_buffer_input_cb(server_name, buffer, input_data): formatted_data = Formatted.from_input_line(input_data) - if room.encrypted: - server.send_room_message(room.id, formatted_data) - return W.WEECHAT_RC_OK + server.send_room_message(room, formatted_data) return W.WEECHAT_RC_OK @@ -86,6 +84,13 @@ class WeechatChannelBuffer(object): "notify_message", "log1" ], + "self_message": [ + SCRIPT_NAME + "_message", + "notify_none", + "no_highlight", + "self_msg", + "log1" + ], "old_message": [ SCRIPT_NAME + "_message", "notify_message", @@ -107,6 +112,10 @@ class WeechatChannelBuffer(object): "invite": [ SCRIPT_NAME + "_invite", "log4" + ], + "topic": [ + SCRIPT_NAME + "_topic", + "log3", ] } @@ -123,13 +132,17 @@ class WeechatChannelBuffer(object): name, "room_buffer_input_cb", server_name, - "room_buffer_input_cb", + "room_buffer_close_cb", server_name, ) self.name = "" self.users = {} # type: Dict[str, RoomUser] + self.topic = "" + self.topic_author = "" + self.topic_date = None + W.buffer_set(self._ptr, "localvar_set_type", 'channel') W.buffer_set(self._ptr, "type", 'formatted') @@ -216,7 +229,7 @@ class WeechatChannelBuffer(object): def _message_tags(self, user, message_type): # type: (str, RoomUser, str) -> List[str] - tags = self.tags[message_type].copy() + tags = list(self.tags[message_type]) tags.append("nick_{nick}".format(nick=user.nick)) @@ -235,10 +248,10 @@ class WeechatChannelBuffer(object): # A message from a non joined user return RoomUser(nick) - def message(self, nick, message, date): + def message(self, nick, message, date, tags=[]): # type: (str, str, int, str) -> None user = self._get_user(nick) - tags = self._message_tags(user, "message") + tags = tags or self._message_tags(user, "message") prefix_string = ("" if not user.prefix else "{}{}{}".format( W.color(self._get_prefix_color(user.prefix)), @@ -264,10 +277,10 @@ class WeechatChannelBuffer(object): self.message(nick, data, date) - def action(self, nick, message, date): + def action(self, nick, message, date, tags=[]): # type: (str, str, int) -> None user = self._get_user(nick) - tags = self._message_tags(user, "action") + tags = tags or self._message_tags(user, "action") nick_prefix = ("" if not user.prefix else "{}{}{}".format( W.color(self._get_prefix_color(user.prefix)), @@ -282,7 +295,7 @@ class WeechatChannelBuffer(object): nick_color=W.color(user.color), author=nick, ncolor=W.color("reset"), - msg=self.message) + msg=message) self.print_date_tags(data, date, tags) @@ -293,9 +306,9 @@ class WeechatChannelBuffer(object): if user.prefix == "&": group_name = "000|o" - elif user.power_level == "@": + elif user.prefix == "@": group_name = "001|h" - elif user.power_level > "+": + elif user.prefix > "+": group_name = "002|v" return group_name @@ -400,3 +413,41 @@ class WeechatChannelBuffer(object): def kick(self, user, date, message=True, extra_tags=[]): # type: (WeechatUser, int, Optional[bool], Optional[List[str]]) -> None self._leave(user, date, message, "kick", extra_tags=[]) + + def _print_topic(self, nick, topic, date): + user = self._get_user(nick) + tags = self._message_tags(user, "topic") + + data = ("{prefix}{nick} has changed " + "the topic for {chan_color}{room}{ncolor} " + "to \"{topic}\"").format( + prefix=W.prefix("network"), + nick=user.nick, + chan_color=W.color("chat_channel"), + ncolor=W.color("reset"), + room=self.name, + topic=topic + ) + + self.print_date_tags(data, date, tags) + + def topic(self, nick, topic, date, message=True): + W.buffer_set(self._ptr, "title", topic) + + if message: + self._print_topic(nick, topic, date) + + self.topic = topic + self.topic_author = nick + self.topic_date = date + + def self_message(self, nick, message, date): + user = self._get_user(nick) + tags = self._message_tags(user, "self_message") + self.message(nick, message, date, tags) + + def self_action(self, nick, message, date): + user = self._get_user(nick) + tags = self._message_tags(user, "self_message") + tags.append(SCRIPT_NAME + "_action") + self.action(nick, message, date, tags) diff --git a/matrix/encryption.py b/matrix/encryption.py index 9ff5112..d8a332e 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -73,8 +73,9 @@ class WeechatArgParse(argparse.ArgumentParser): def error(self, message): m = ("{prefix}Error: {message} for command {command} " - "(see /help {command})").format(prefix=W.prefix("error"), - message=message, command=self.prog) + "(see /help {command})").format(prefix=W.prefix("error"), + message=message, + command=self.prog) W.prnt("", m) raise ParseError diff --git a/matrix/events.py b/matrix/events.py index 9b4330e..31f1f81 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -27,11 +27,12 @@ from operator import itemgetter from matrix.globals import W from matrix.utils import (tags_for_message, sanitize_id, sanitize_token, sanitize_text, tags_from_line_data) -from matrix.rooms import (matrix_create_room_buffer, RoomInfo, RoomMessageText, +from matrix.rooms import (RoomInfo, RoomMessageText, RoomMessageEvent, RoomRedactedMessageEvent, RoomMessageEmote) from matrix.encryption import OlmDeviceKey, OneTimeKey +from .buffer import RoomUser try: from olm.session import OlmMessage, OlmPreKeyMessage @@ -657,7 +658,7 @@ class MatrixSyncEvent(MatrixEvent): info = self.joined_room_infos.pop() if info.room_id not in server.buffers: - matrix_create_room_buffer(server, info.room_id) + server.create_room_buffer(info.room_id) room = server.rooms[info.room_id] diff --git a/matrix/rooms.py b/matrix/rooms.py index 86c616d..434f943 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -156,45 +156,15 @@ class MatrixUser: # yapf: enable -def matrix_create_room_buffer(server, room_id): - # type: (MatrixServer, str) -> None - buf = W.buffer_new(room_id, "room_input_cb", server.name, "room_close_cb", - server.name) - - W.buffer_set(buf, "localvar_set_type", 'channel') - W.buffer_set(buf, "type", 'formatted') - - W.buffer_set(buf, "localvar_set_channel", room_id) - - W.buffer_set(buf, "localvar_set_nick", server.user) - - W.buffer_set(buf, "localvar_set_server", server.name) - - short_name = strip_matrix_server(room_id) - W.buffer_set(buf, "short_name", short_name) - - W.nicklist_add_group(buf, '', "000|o", "weechat.color.nicklist_group", 1) - W.nicklist_add_group(buf, '', "001|h", "weechat.color.nicklist_group", 1) - W.nicklist_add_group(buf, '', "002|v", "weechat.color.nicklist_group", 1) - W.nicklist_add_group(buf, '', "999|...", "weechat.color.nicklist_group", 1) - - W.buffer_set(buf, "nicklist", "1") - W.buffer_set(buf, "nicklist_display_groups", "0") - - # TODO make this configurable - W.buffer_set(buf, "highlight_tags_restrict", "matrix_message") - - server.buffers[room_id] = buf - server.rooms[room_id] = MatrixRoom(room_id) - - class RoomInfo(): - def __init__(self, room_id, prev_batch, events): + def __init__(self, room_id, prev_batch, state, timeline): # type: (str, str, List[Any], List[Any]) -> None self.room_id = room_id self.prev_batch = prev_batch - self.events = deque(events) + + self.state = deque(state) + self.timeline = deque(timeline) @staticmethod def _message_from_event(event): @@ -219,66 +189,40 @@ class RoomInfo(): raise ValueError if event_dict["content"]["membership"] == "join": - event = RoomMemberJoin.from_dict(event_dict) - - try: - message = RoomMembershipMessage( - event.event_id, event.sender, event.timestamp, - "has joined", "join") - - return event, message - except AttributeError: - return event, None + return RoomMemberJoin.from_dict(event_dict) elif event_dict["content"]["membership"] == "leave": - event = RoomMemberLeave.from_dict(event_dict) + return RoomMemberLeave.from_dict(event_dict) - try: - msg = ("has left" if event.sender == event.leaving_user else - "has been kicked") - message = RoomMembershipMessage( - event.event_id, event.leaving_user, event.timestamp, msg, "quit") - return event, message - except AttributeError: - return event, None - - return None, None + return None @staticmethod def parse_event(olm, room_id, event_dict): # type: (Dict[Any, Any]) -> (RoomEvent, RoomEvent) - state_event = None - message_event = None + event = None if "redacted_by" in event_dict["unsigned"]: - message_event = RoomRedactedMessageEvent.from_dict(event_dict) + event = RoomRedactedMessageEvent.from_dict(event_dict) elif event_dict["type"] == "m.room.message": - message_event = RoomInfo._message_from_event(event_dict) + event = RoomInfo._message_from_event(event_dict) elif event_dict["type"] == "m.room.member": - state_event, message_event = ( - RoomInfo._membership_from_dict(event_dict)) + event = RoomInfo._membership_from_dict(event_dict) elif event_dict["type"] == "m.room.power_levels": - state_event = RoomPowerLevels.from_dict(event_dict) + event = RoomPowerLevels.from_dict(event_dict) elif event_dict["type"] == "m.room.topic": - state_event = RoomTopicEvent.from_dict(event_dict) - message_event = RoomTopiceMessage( - state_event.event_id, - state_event.sender, - state_event.timestamp, - state_event.topic) + event = RoomTopicEvent.from_dict(event_dict) elif event_dict["type"] == "m.room.redaction": - message_event = RoomRedactionEvent.from_dict(event_dict) + event = RoomRedactionEvent.from_dict(event_dict) elif event_dict["type"] == "m.room.name": - state_event = RoomNameEvent.from_dict(event_dict) + event = RoomNameEvent.from_dict(event_dict) elif event_dict["type"] == "m.room.canonical_alias": - state_event = RoomAliasEvent.from_dict(event_dict) + event = RoomAliasEvent.from_dict(event_dict) elif event_dict["type"] == "m.room.encryption": - state_event = RoomEncryptionEvent.from_dict(event_dict) + event = RoomEncryptionEvent.from_dict(event_dict) elif event_dict["type"] == "m.room.encrypted": - state_event, message_event = RoomInfo._decrypt_event(olm, room_id, - event_dict) + event = RoomInfo._decrypt_event(olm, room_id, event_dict) - return state_event, message_event + return event @staticmethod def _decrypt_event(olm, room_id, event_dict): @@ -287,7 +231,7 @@ class RoomInfo(): plaintext = olm.group_decrypt(room_id, session_id, ciphertext) if not plaintext: - return None, None + return None parsed_plaintext = json.loads(plaintext, encoding="utf-8") @@ -297,18 +241,13 @@ class RoomInfo(): return RoomInfo.parse_event(olm, room_id, event_dict) @staticmethod - def _parse_events(olm, room_id, parsed_dict, messages=True, state=True): - state_events = [] - message_events = [] - - if not messages and not state: - return [] + def _parse_events(olm, room_id, parsed_dict): + events = [] try: for event in parsed_dict: - m_event, s_event = RoomInfo.parse_event(olm, room_id, event) - state_events.append(m_event) - message_events.append(s_event) + e = RoomInfo.parse_event(olm, room_id, event) + events.append(e) except (ValueError, TypeError, KeyError) as error: message = ("{prefix}matrix: Error parsing " "room event of type {type}: {error}\n{event}").format( @@ -319,14 +258,6 @@ class RoomInfo(): W.prnt("", message) raise - events = [] - - if state: - events = events + state_events - - if messages: - events = events + message_events - return events @classmethod @@ -336,12 +267,23 @@ class RoomInfo(): state_dict = parsed_dict['state']['events'] timeline_dict = parsed_dict['timeline']['events'] - state_events = RoomInfo._parse_events(olm, room_id, state_dict, messages=False) - timeline_events = RoomInfo._parse_events(olm, room_id, timeline_dict) + state_events = RoomInfo._parse_events( + olm, + room_id, + state_dict + ) + timeline_events = RoomInfo._parse_events( + olm, + room_id, + timeline_dict + ) - events = state_events + timeline_events - - return cls(room_id, prev_batch, list(filter(None, events))) + return cls( + room_id, + prev_batch, + list(filter(None, state_events)), + list(filter(None, timeline_events)) + ) class RoomEvent(): diff --git a/matrix/server.py b/matrix/server.py index 163b4be..0af619d 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -29,10 +29,19 @@ from http_parser.pyparser import HttpParser from matrix.plugin_options import Option, DebugType from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, - create_server_buffer, tags_for_message) + create_server_buffer, tags_for_message, + server_ts_to_weechat, shorten_sender) from matrix.utf import utf8_decode from matrix.globals import W, SERVERS, OPTIONS import matrix.api as API +from .buffer import WeechatChannelBuffer, RoomUser +from .rooms import ( + MatrixRoom, + RoomMessageText, + RoomMessageEmote, + MatrixUser, + RoomMemberJoin +) from matrix.api import ( MatrixClient, MatrixSyncMessage, @@ -40,10 +49,13 @@ from matrix.api import ( MatrixKeyUploadMessage, MatrixKeyQueryMessage, MatrixToDeviceMessage, + MatrixSendMessage, MatrixEncryptedMessage, MatrixKeyClaimMessage ) +from .events import MatrixSendEvent + from matrix.encryption import ( Olm, EncryptionError, @@ -77,6 +89,7 @@ class MatrixServer: self.password = "" # type: str self.rooms = dict() # type: Dict[str, MatrixRoom] + self.room_buffers = dict() # type: Dict[str, WeechatChannelBuffer] self.buffers = dict() # type: Dict[str, weechat.buffer] self.server_buffer = None # type: weechat.buffer self.fd_hook = None # type: weechat.hook @@ -433,7 +446,7 @@ class MatrixServer: if self.server_buffer: message = ("{prefix}matrix: disconnected from server" - ).format(prefix=W.prefix("network")) + ).format(prefix=W.prefix("network")) server_buffer_prnt(self, message) if reconnect: @@ -486,16 +499,20 @@ class MatrixServer: message = MatrixSyncMessage(self.client, self.next_batch, limit) self.send_queue.append(message) + def _send_unencrypted_message(self, room_id, formatted_data): + message = MatrixSendMessage( + self.client, room_id=room_id, formatted_message=formatted_data) + self.send_or_queue(message) + def send_room_message( self, - room_id, + room, formatted_data, already_claimed=False ): # type: (str, Formatted) -> None - room = self.rooms[room_id] - if not room.encrypted: + self._send_unencrypted_message(room.room_id, formatted_data) return # TODO don't send messages unless all the devices are verified @@ -505,8 +522,8 @@ class MatrixServer: W.prnt("", "{prefix}matrix: Olm session missing for room, can't" " encrypt message.") W.prnt("", pprint.pformat(missing)) - self.encryption_queue[room_id].append(formatted_data) - message = MatrixKeyClaimMessage(self.client, room_id, missing) + self.encryption_queue[room.room_id].append(formatted_data) + message = MatrixKeyClaimMessage(self.client, room.room_id, missing) self.send_or_queue(message) return @@ -525,7 +542,7 @@ class MatrixServer: try: payload_dict, to_device_dict = self.olm.group_encrypt( - room_id, + room.room_id, plaintext_dict, self.user_id, room.users.keys() @@ -538,7 +555,7 @@ class MatrixServer: message = MatrixEncryptedMessage( self.client, - room_id, + room.room_id, formatted_data, payload_dict ) @@ -612,19 +629,46 @@ class MatrixServer: server_buffer_prnt(self, pprint.pformat(message.request.payload)) server_buffer_prnt(self, pprint.pformat(message.response.body)) + def handle_room_event(self, room, room_buffer, event, is_state_event): + if isinstance(event, RoomMemberJoin): + if event.sender in room.users: + user = room.users[event.sender] + if event.display_name: + user.display_name = event.display_name + else: + short_name = shorten_sender(event.sender) + user = MatrixUser(short_name, event.display_name) + buffer_user = RoomUser(user.name, event.sender) + room.users[event.sender] = user + + if self.user_id == event.sender: + buffer_user.color = "weechat.color.chat_nick_self" + user.nick_color = "weechat.color.chat_nick_self" + + room_buffer.join( + buffer_user, + server_ts_to_weechat(event.timestamp), + not is_state_event + ) + else: + tags = tags_for_message("message") + event.execute(self, room, room_buffer._ptr, tags) + def _loop_events(self, info, n): for i in range(n+1): + is_state = False try: - event = info.events.popleft() + event = info.state.popleft() + is_state = True except IndexError: - return i + try: + event = info.timeline.popleft() + except IndexError: + return i - room = self.rooms[info.room_id] - buf = self.buffers[info.room_id] - - tags = tags_for_message("message") - event.execute(self, room, buf, tags) + room, room_buffer = self.find_room_from_id(info.room_id) + self.handle_room_event(room, room_buffer, event, is_state) self.event_queue.appendleft(info) return i @@ -657,6 +701,32 @@ class MatrixServer: return + def handle_own_messages(self, room_buffer, message): + if isinstance(message, RoomMessageText): + msg = (message.formatted_message.to_weechat() + if message.formatted_message + else message.message) + + date = server_ts_to_weechat(message.timestamp) + room_buffer.self_message(self.user, msg, date) + + return + elif isinstance(message, RoomMessageEmote): + date = server_ts_to_weechat(message.timestamp) + room_buffer.self_action(self.user, message.message, date) + + return + + raise NotImplementedError("Unsupported message of type {}".format( + type(message))) + + def handle_matrix_response(self, response): + if isinstance(response, MatrixSendEvent): + _, room_buffer = self.find_room_from_id(response.room_id) + self.handle_own_messages(room_buffer, response.message) + else: + response.execute() + def handle_response(self, message): # type: (MatrixMessage) -> None @@ -674,7 +744,7 @@ class MatrixServer: return event = message.event - event.execute() + self.handle_matrix_response(event) else: status_code = message.response.status if status_code == 504: @@ -705,6 +775,26 @@ class MatrixServer: return + def create_room_buffer(self, room_id): + buf = WeechatChannelBuffer(room_id, self.name, self.user) + # TODO this should turned into a propper class + self.room_buffers[room_id] = buf + self.buffers[room_id] = buf._ptr + self.rooms[room_id] = MatrixRoom(room_id) + pass + + def find_room_from_ptr(self, pointer): + room_id = key_from_value(self.buffers, pointer) + room = self.rooms[room_id] + room_buffer = self.room_buffers[room_id] + + return room, room_buffer + + def find_room_from_id(self, room_id): + room = self.rooms[room_id] + room_buffer = self.room_buffers[room_id] + return room, room_buffer + @utf8_decode def matrix_config_server_read_cb(data, config_file, section, option_name, From 24b412d73c16adbd7fc04f3a316931808bd071fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 5 Jul 2018 15:46:45 +0200 Subject: [PATCH 037/269] matrix: Change the way the topic is changed. --- matrix/buffer.py | 19 +++++++++-- matrix/rooms.py | 85 ------------------------------------------------ matrix/server.py | 47 ++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 90 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 447235c..e08d038 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -139,7 +139,6 @@ class WeechatChannelBuffer(object): self.name = "" self.users = {} # type: Dict[str, RoomUser] - self.topic = "" self.topic_author = "" self.topic_date = None @@ -425,15 +424,21 @@ class WeechatChannelBuffer(object): nick=user.nick, chan_color=W.color("chat_channel"), ncolor=W.color("reset"), - room=self.name, + room=self.short_name, topic=topic ) self.print_date_tags(data, date, tags) - def topic(self, nick, topic, date, message=True): + @property + def topic(self): + return W.buffer_get_string(self._ptr, "title") + + @topic.setter + def topic(self, topic): W.buffer_set(self._ptr, "title", topic) + def change_topic(self, nick, topic, date, message=True): if message: self._print_topic(nick, topic, date) @@ -451,3 +456,11 @@ class WeechatChannelBuffer(object): tags = self._message_tags(user, "self_message") tags.append(SCRIPT_NAME + "_action") self.action(nick, message, date, tags) + + @property + def short_name(self): + return W.buffer_get_string(self._ptr, "short_name") + + @short_name.setter + def short_name(self, name): + W.buffer_set(self._ptr, "short_name", name) diff --git a/matrix/rooms.py b/matrix/rooms.py index 434f943..4900ccf 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -566,44 +566,6 @@ class RoomMemberJoin(RoomEvent): return cls(event_id, sender, timestamp, display_name) - def execute(self, server, room, buff, tags): - short_name = shorten_sender(self.sender) - - if self.sender in room.users: - user = room.users[self.sender] - if self.display_name: - user.display_name = self.display_name - else: - user = MatrixUser(short_name, self.display_name) - - if not user.nick_color: - if self.sender == server.user_id: - highlight_words = [self.sender, user.name] - - if self.display_name: - highlight_words.append(self.display_name) - - user.nick_color = "weechat.color.chat_nick_self" - W.buffer_set(buff, "highlight_words", ",".join(highlight_words)) - else: - user.nick_color = W.info_get("nick_color_name", user.name) - - room.users[self.sender] = user - - nick_pointer = W.nicklist_search_nick(buff, "", self.sender) - - if not nick_pointer: - add_user_to_nicklist(buff, self.sender, user) - - # calculate room display name and set it as the buffer list name - room_name = room.display_name(server.user_id) - - # A user has joined an encrypted room, we need to check for new devices - if room.encrypted: - server.device_check_timestamp = None - - W.buffer_set(buff, "short_name", room_name) - class RoomMemberLeave(RoomEvent): @@ -675,43 +637,6 @@ class RoomPowerLevels(RoomEvent): self._set_power_level(room, buff, level) -class RoomTopiceMessage(RoomEvent): - - def __init__(self, event_id, sender, timestamp, topic): - self.topic = topic - RoomEvent.__init__(self, event_id, sender, timestamp) - - def execute(self, server, room, buff, tags): - topic = self.topic - - nick, color_name = sender_to_nick_and_color(room, self.sender) - - author = ("{nick_color}{user}{ncolor}").format( - nick_color=W.color(color_name), user=nick, ncolor=W.color("reset")) - - # TODO print old topic if configured so - if room.is_named(): - message = ("{prefix}{nick} has changed " - "the topic for {chan_color}{room}{ncolor} " - "to \"{topic}\"").format( - prefix=W.prefix("network"), - nick=author, - chan_color=W.color("chat_channel"), - ncolor=W.color("reset"), - room=room.named_room_name(), - topic=topic) - else: - message = ('{prefix}{nick} has changed the topic to ' - '"{topic}"').format( - prefix=W.prefix("network"), - nick=author, - topic=topic) - - tags = ["matrix_topic", "log3", "matrix_id_{}".format(self.event_id)] - date = server_ts_to_weechat(self.timestamp) - W.prnt_date_tags(buff, date, ",".join(tags), message) - - class RoomTopicEvent(RoomEvent): def __init__(self, event_id, sender, timestamp, topic): @@ -728,16 +653,6 @@ class RoomTopicEvent(RoomEvent): return cls(event_id, sender, timestamp, topic) - def execute(self, server, room, buff, tags): - topic = self.topic - - W.buffer_set(buff, "title", topic) - - room.topic = topic - room.topic_author = self.sender - room.topic_date = datetime.fromtimestamp( - server_ts_to_weechat(self.timestamp)) - class RoomRedactionEvent(RoomEvent): diff --git a/matrix/server.py b/matrix/server.py index 0af619d..2cff3ce 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -40,7 +40,9 @@ from .rooms import ( RoomMessageText, RoomMessageEmote, MatrixUser, - RoomMemberJoin + RoomMemberJoin, + RoomMemberLeave, + RoomTopicEvent ) from matrix.api import ( MatrixClient, @@ -629,7 +631,13 @@ class MatrixServer: server_buffer_prnt(self, pprint.pformat(message.request.payload)) server_buffer_prnt(self, pprint.pformat(message.response.body)) - def handle_room_event(self, room, room_buffer, event, is_state_event): + def handle_room_membership_events( + self, + room, + room_buffer, + event, + is_state_event + ): if isinstance(event, RoomMemberJoin): if event.sender in room.users: user = room.users[event.sender] @@ -639,6 +647,8 @@ class MatrixServer: short_name = shorten_sender(event.sender) user = MatrixUser(short_name, event.display_name) buffer_user = RoomUser(user.name, event.sender) + # TODO remove this duplication + user.nick_color = buffer_user.color room.users[event.sender] = user if self.user_id == event.sender: @@ -650,6 +660,39 @@ class MatrixServer: server_ts_to_weechat(event.timestamp), not is_state_event ) + + # calculate room display name and set it as the buffer list name + room_name = room.display_name(self.user_id) + room_buffer.short_name = room_name + + # A user has joined an encrypted room, we need to check for + # new devices + if room.encrypted: + self.device_check_timestamp = None + elif isinstance(event, RoomMemberLeave): + pass + + def handle_room_event(self, room, room_buffer, event, is_state_event): + if isinstance(event, (RoomMemberJoin, RoomMemberLeave)): + self.handle_room_membership_events( + room, + room_buffer, + event, + is_state_event + ) + elif isinstance(event, RoomTopicEvent): + try: + user = room.users[event.sender] + nick = user.name + except KeyError: + nick = event.sender + + room_buffer.change_topic( + nick, + event.topic, + server_ts_to_weechat(event.timestamp), + not is_state_event + ) else: tags = tags_for_message("message") event.execute(self, room, room_buffer._ptr, tags) From ca9b60cd34d4b368d3ad9ffb18791cf2ec96bf7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 5 Jul 2018 17:02:01 +0200 Subject: [PATCH 038/269] server: More refactoring of the event handling (join/leave). --- matrix/buffer.py | 32 +++++++++++++---------- matrix/rooms.py | 66 +++++++++++++----------------------------------- matrix/server.py | 57 +++++++++++++++++++++-------------------- 3 files changed, 66 insertions(+), 89 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index e08d038..f27d20b 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -122,8 +122,8 @@ class WeechatChannelBuffer(object): membership_messages = { "join": "has joined", "part": "has left", - "kick": "has been kicked", - "invite": "has been invited" + "kick": "has been kicked from", + "invite": "has been invited to" } def __init__(self, name, server_name, user): @@ -352,6 +352,8 @@ class WeechatChannelBuffer(object): # type: (WeechatUser, str) -> str action_color = ("green" if message_type == "join" or message_type == "invite" else "red") + prefix = ("join" if message_type == "join" or message_type == "invite" + else "quit") membership_message = self.membership_messages[message_type] @@ -359,7 +361,7 @@ class WeechatChannelBuffer(object): "{del_color}({host_color}{host}{del_color})" "{action_color} {message} " "{channel_color}{room}{ncolor}").format( - prefix=W.prefix(message_type), + prefix=W.prefix(prefix), color=W.color(user.color), author=user.nick, ncolor=W.color("reset"), @@ -369,7 +371,7 @@ class WeechatChannelBuffer(object): action_color=W.color(action_color), message=membership_message, channel_color=W.color("chat_channel"), - room=self.name) + room=self.short_name) return message @@ -391,10 +393,14 @@ class WeechatChannelBuffer(object): def _remove_user_from_nicklist(self, user): # type: (WeechatUser) -> None - pass + nick_pointer = W.nicklist_search_nick(self._ptr, "", user.nick) - def _leave(self, user, date, message, leave_type, extra_tags): - # type: (WeechatUser, int, bool, str, List[str]) -> None + if nick_pointer: + W.nicklist_remove_nick(self._ptr, nick_pointer) + + def _leave(self, nick, date, message, leave_type, extra_tags): + # type: (str, int, bool, str, List[str]) -> None + user = self._get_user(nick) self._remove_user_from_nicklist(user) if message: @@ -405,13 +411,13 @@ class WeechatChannelBuffer(object): if user.nick in self.users: del self.users[user.nick] - def part(self, user, date, message=True, extra_tags=[]): - # type: (WeechatUser, int, Optional[bool], Optional[List[str]]) -> None - self._leave(user, date, message, "leave", extra_tags) + def part(self, nick, date, message=True, extra_tags=[]): + # type: (str, int, Optional[bool], Optional[List[str]]) -> None + self._leave(nick, date, message, "part", extra_tags) - def kick(self, user, date, message=True, extra_tags=[]): - # type: (WeechatUser, int, Optional[bool], Optional[List[str]]) -> None - self._leave(user, date, message, "kick", extra_tags=[]) + def kick(self, nick, date, message=True, extra_tags=[]): + # type: (str, int, Optional[bool], Optional[List[str]]) -> None + self._leave(nick, date, message, "kick", extra_tags=[]) def _print_topic(self, nick, topic, date): user = self._get_user(nick) diff --git a/matrix/rooms.py b/matrix/rooms.py index 4900ccf..f03251f 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -143,6 +143,20 @@ class MatrixRoom: """ return not self.is_named() + def handle_event(self, event): + if isinstance(event, RoomMemberJoin): + if event.sender in self.users: + user = self.users[event.sender] + if event.display_name: + user.display_name = event.display_name + else: + short_name = shorten_sender(event.sender) + user = MatrixUser(short_name, event.display_name) + self.users[event.sender] = user + elif isinstance(event, RoomMemberLeave): + if event.leaving_user in self.users: + del self.users[event.leaving_user] + class MatrixUser: @@ -513,43 +527,11 @@ class RoomMessageMedia(RoomMessageEvent): self._print_message(msg, room, buff, tags) -class RoomMembershipMessage(RoomEvent): - def __init__(self, event_id, sender, timestamp, message, prefix): - self.message = message - self.prefix = prefix - RoomEvent.__init__(self, event_id, sender, timestamp) - - def execute(self, server, room, buff, tags): - nick, color_name = sender_to_nick_and_color(room, self.sender) - event_tags = add_event_tags(self.event_id, nick, None, []) - # TODO this should be configurable - action_color = "red" if self.prefix == "quit" else "green" - - data = ("{prefix}{color}{author}{ncolor} " - "{del_color}({host_color}{user_id}{del_color})" - "{action_color} {message} " - "{channel_color}{room}{ncolor}").format( - prefix=W.prefix(self.prefix), - color=W.color(color_name), - author=nick, - ncolor=W.color("reset"), - del_color=W.color("chat_delimiters"), - host_color=W.color("chat_host"), - user_id=self.sender, - action_color=W.color(action_color), - message=self.message, - channel_color=W.color("chat_channel"), - room="" if room.is_group() else room.named_room_name()) - date = server_ts_to_weechat(self.timestamp) - tags_string = ",".join(event_tags) - - W.prnt_date_tags(buff, date, tags_string, data) - - class RoomMemberJoin(RoomEvent): - def __init__(self, event_id, sender, timestamp, display_name): + def __init__(self, event_id, sender, timestamp, display_name, state_key): self.display_name = display_name + self.state_key = state_key RoomEvent.__init__(self, event_id, sender, timestamp) @classmethod @@ -557,6 +539,7 @@ class RoomMemberJoin(RoomEvent): event_id = sanitize_id(event_dict["event_id"]) sender = sanitize_id(event_dict["sender"]) timestamp = sanitize_ts(event_dict["origin_server_ts"]) + state_key = sanitize_id(event_dict["state_key"]) display_name = None if event_dict["content"]: @@ -564,7 +547,7 @@ class RoomMemberJoin(RoomEvent): display_name = sanitize_text( event_dict["content"]["displayname"]) - return cls(event_id, sender, timestamp, display_name) + return cls(event_id, sender, timestamp, display_name, state_key) class RoomMemberLeave(RoomEvent): @@ -582,19 +565,6 @@ class RoomMemberLeave(RoomEvent): return cls(event_id, sender, leaving_user, timestamp) - def execute(self, server, room, buff, tags): - if self.leaving_user in room.users: - nick_pointer = W.nicklist_search_nick(buff, "", self.leaving_user) - - if nick_pointer: - W.nicklist_remove_nick(buff, nick_pointer) - - del room.users[self.leaving_user] - - # calculate room display name and set it as the buffer list name - room_name = room.display_name(server.user_id) - W.buffer_set(buff, "short_name", room_name) - class RoomPowerLevels(RoomEvent): diff --git a/matrix/server.py b/matrix/server.py index 2cff3ce..694e86b 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -639,38 +639,39 @@ class MatrixServer: is_state_event ): if isinstance(event, RoomMemberJoin): - if event.sender in room.users: - user = room.users[event.sender] - if event.display_name: - user.display_name = event.display_name - else: - short_name = shorten_sender(event.sender) - user = MatrixUser(short_name, event.display_name) - buffer_user = RoomUser(user.name, event.sender) - # TODO remove this duplication - user.nick_color = buffer_user.color - room.users[event.sender] = user + room.handle_event(event) + user = room.users[event.sender] + buffer_user = RoomUser(user.name, event.sender) + # TODO remove this duplication + user.nick_color = buffer_user.color - if self.user_id == event.sender: - buffer_user.color = "weechat.color.chat_nick_self" - user.nick_color = "weechat.color.chat_nick_self" + if self.user_id == event.sender: + buffer_user.color = "weechat.color.chat_nick_self" + user.nick_color = "weechat.color.chat_nick_self" - room_buffer.join( - buffer_user, - server_ts_to_weechat(event.timestamp), - not is_state_event - ) + room_buffer.join( + buffer_user, + server_ts_to_weechat(event.timestamp), + not is_state_event + ) - # calculate room display name and set it as the buffer list name - room_name = room.display_name(self.user_id) - room_buffer.short_name = room_name - - # A user has joined an encrypted room, we need to check for - # new devices - if room.encrypted: - self.device_check_timestamp = None elif isinstance(event, RoomMemberLeave): - pass + user = room.users[event.sender] + date = server_ts_to_weechat(event.timestamp) + + if event.sender == event.leaving_user: + room_buffer.part(user.name, date, not is_state_event) + else: + room_buffer.kick(user.name, date, not is_state_event) + + # calculate room display name and set it as the buffer list name + room_name = room.display_name(self.user_id) + room_buffer.short_name = room_name + + # A user has joined or left an encrypted room, we need to check for + # new devices and create a new group session + if room.encrypted: + self.device_check_timestamp = None def handle_room_event(self, room, room_buffer, event, is_state_event): if isinstance(event, (RoomMemberJoin, RoomMemberLeave)): From 3c4888c1fb65f6ee13db596c99c548536e70f585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 5 Jul 2018 18:32:24 +0200 Subject: [PATCH 039/269] rooms: Handle invites. --- matrix/buffer.py | 5 +++-- matrix/rooms.py | 52 ++++++++++++++++++++++++++++++++++++++++++++---- matrix/server.py | 45 ++++++++++++++++++++++++++++++++--------- 3 files changed, 87 insertions(+), 15 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index f27d20b..c659718 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -385,8 +385,9 @@ class WeechatChannelBuffer(object): message = self._membership_message(user, "join") self.print_date_tags(message, date, tags) - def invite(self, user, date, extra_tags=[]): - # type: (WeechatUser, int, Optional[bool], Optional[List[str]]) -> None + def invite(self, nick, date, extra_tags=[]): + # type: (str, int, Optional[bool], Optional[List[str]]) -> None + user = self._get_user(nick) tags = self._message_tags(user, "invite") message = self._membership_message(user, "invite") self.print_date_tags(message, date, tags + extra_tags) diff --git a/matrix/rooms.py b/matrix/rooms.py index f03251f..0c2c29c 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -208,6 +208,9 @@ class RoomInfo(): elif event_dict["content"]["membership"] == "leave": return RoomMemberLeave.from_dict(event_dict) + elif event_dict["content"]["membership"] == "invite": + return RoomMemberInvite.from_dict(event_dict) + return None @staticmethod @@ -308,6 +311,10 @@ class RoomEvent(): self.timestamp = timestamp +class RoomMembershipEvent(RoomEvent): + pass + + class RoomRedactedMessageEvent(RoomEvent): def __init__(self, event_id, sender, timestamp, censor, reason=None): @@ -527,11 +534,22 @@ class RoomMessageMedia(RoomMessageEvent): self._print_message(msg, room, buff, tags) -class RoomMemberJoin(RoomEvent): +class RoomMemberJoin(RoomMembershipEvent): - def __init__(self, event_id, sender, timestamp, display_name, state_key): + def __init__( + self, + event_id, + sender, + timestamp, + display_name, + state_key, + content, + prev_content + ): self.display_name = display_name self.state_key = state_key + self.content = content + self.prev_content = prev_content RoomEvent.__init__(self, event_id, sender, timestamp) @classmethod @@ -541,16 +559,42 @@ class RoomMemberJoin(RoomEvent): timestamp = sanitize_ts(event_dict["origin_server_ts"]) state_key = sanitize_id(event_dict["state_key"]) display_name = None + content = event_dict["content"] + prev_content = (event_dict["unsigned"]["prev_content"] + if "prev_content" in event_dict["unsigned"] else None) if event_dict["content"]: if "display_name" in event_dict["content"]: display_name = sanitize_text( event_dict["content"]["displayname"]) - return cls(event_id, sender, timestamp, display_name, state_key) + return cls( + event_id, + sender, + timestamp, + display_name, + state_key, + content, + prev_content + ) -class RoomMemberLeave(RoomEvent): +class RoomMemberInvite(RoomMembershipEvent): + def __init__(self, event_id, sender, invited_user, timestamp): + self.invited_user = invited_user + RoomEvent.__init__(self, event_id, sender, timestamp) + + @classmethod + def from_dict(cls, event_dict): + event_id = sanitize_id(event_dict["event_id"]) + sender = sanitize_id(event_dict["sender"]) + invited_user = sanitize_id(event_dict["state_key"]) + timestamp = sanitize_ts(event_dict["origin_server_ts"]) + + return cls(event_id, sender, invited_user, timestamp) + + +class RoomMemberLeave(RoomMembershipEvent): def __init__(self, event_id, sender, leaving_user, timestamp): self.leaving_user = leaving_user diff --git a/matrix/server.py b/matrix/server.py index 694e86b..3a2e0b9 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -40,8 +40,10 @@ from .rooms import ( RoomMessageText, RoomMessageEmote, MatrixUser, + RoomMembershipEvent, RoomMemberJoin, RoomMemberLeave, + RoomMemberInvite, RoomTopicEvent ) from matrix.api import ( @@ -638,7 +640,7 @@ class MatrixServer: event, is_state_event ): - if isinstance(event, RoomMemberJoin): + def join(event, date, room, room_buffer, is_state_event): room.handle_event(event) user = room.users[event.sender] buffer_user = RoomUser(user.name, event.sender) @@ -655,14 +657,39 @@ class MatrixServer: not is_state_event ) - elif isinstance(event, RoomMemberLeave): - user = room.users[event.sender] - date = server_ts_to_weechat(event.timestamp) + date = server_ts_to_weechat(event.timestamp) - if event.sender == event.leaving_user: - room_buffer.part(user.name, date, not is_state_event) + joined = False + left = False + + if isinstance(event, RoomMemberJoin): + if event.prev_content and "membership" in event.prev_content: + if (event.prev_content["membership"] == "leave" + or event.prev_content["membership"] == "invite"): + join(event, date, room, room_buffer, is_state_event) + joined = True + else: + # TODO print out profile changes + return else: - room_buffer.kick(user.name, date, not is_state_event) + # No previous content for this user in this room, so he just + # joined. + join(event, date, room, room_buffer, is_state_event) + joined = True + + elif isinstance(event, RoomMemberLeave): + # TODO the nick can be a display name or a full sender name + nick = shorten_sender(event.sender) + if event.sender == event.leaving_user: + room_buffer.part(nick, date, not is_state_event) + else: + room_buffer.kick(nick, date, not is_state_event) + + left = True + + elif isinstance(event, RoomMemberInvite): + room_buffer.invite(event.sender, date) + return # calculate room display name and set it as the buffer list name room_name = room.display_name(self.user_id) @@ -670,11 +697,11 @@ class MatrixServer: # A user has joined or left an encrypted room, we need to check for # new devices and create a new group session - if room.encrypted: + if room.encrypted and (joined or left): self.device_check_timestamp = None def handle_room_event(self, room, room_buffer, event, is_state_event): - if isinstance(event, (RoomMemberJoin, RoomMemberLeave)): + if isinstance(event, RoomMembershipEvent): self.handle_room_membership_events( room, room_buffer, From e4d83b5cf44228f91f36ea7c543783c9c1069622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 5 Jul 2018 19:22:29 +0200 Subject: [PATCH 040/269] server: Fix invite message printing. --- matrix/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 3a2e0b9..755ab89 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -641,7 +641,6 @@ class MatrixServer: is_state_event ): def join(event, date, room, room_buffer, is_state_event): - room.handle_event(event) user = room.users[event.sender] buffer_user = RoomUser(user.name, event.sender) # TODO remove this duplication @@ -657,6 +656,7 @@ class MatrixServer: not is_state_event ) + room.handle_event(event) date = server_ts_to_weechat(event.timestamp) joined = False @@ -688,7 +688,7 @@ class MatrixServer: left = True elif isinstance(event, RoomMemberInvite): - room_buffer.invite(event.sender, date) + room_buffer.invite(event.invited_user, date) return # calculate room display name and set it as the buffer list name From ae5f97ce1d19291e8d5d8c90904f328c4283d229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Thu, 5 Jul 2018 19:23:36 +0200 Subject: [PATCH 041/269] server: Don't print out invites if they are from the state. --- matrix/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/matrix/server.py b/matrix/server.py index 755ab89..f5278c8 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -688,6 +688,9 @@ class MatrixServer: left = True elif isinstance(event, RoomMemberInvite): + if is_state_event: + return + room_buffer.invite(event.invited_user, date) return From 66507b23ce74002a67f713ea12c00b917ab314e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 6 Jul 2018 14:56:06 +0200 Subject: [PATCH 042/269] buffer: Add a matrix specific buffer class. --- matrix/buffer.py | 398 +++++++++++++++++++++++++++++++++++++++++++++-- matrix/rooms.py | 131 ++-------------- matrix/server.py | 127 ++------------- 3 files changed, 415 insertions(+), 241 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index c659718..0d39fe1 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -18,11 +18,29 @@ from __future__ import unicode_literals import time +from builtins import super +from functools import partial -from .globals import W, SERVERS, SCRIPT_NAME +from .globals import W, SERVERS, OPTIONS, SCRIPT_NAME from .utf import utf8_decode from .colors import Formatted -from builtins import super +from .utils import shorten_sender, server_ts_to_weechat, string_strikethrough +from .plugin_options import RedactType + + +from .rooms import ( + RoomNameEvent, + RoomAliasEvent, + RoomMembershipEvent, + RoomMemberJoin, + RoomMemberLeave, + RoomMemberInvite, + RoomTopicEvent, + RoomMessageText, + RoomMessageEmote, + RoomRedactionEvent, + RoomRedactedMessageEvent +) @utf8_decode @@ -91,6 +109,12 @@ class WeechatChannelBuffer(object): "self_msg", "log1" ], + "action": [ + SCRIPT_NAME + "_message", + SCRIPT_NAME + "_action", + "notify_message", + "log1", + ], "old_message": [ SCRIPT_NAME + "_message", "notify_message", @@ -126,6 +150,91 @@ class WeechatChannelBuffer(object): "invite": "has been invited to" } + class Line(object): + def __init__(self, pointer): + self._ptr = pointer + + @property + def _hdata(self): + return W.hdata_get("line_data") + + @property + def prefix(self): + return W.hdata_string(self._hdata, self._ptr, "prefix") + + @prefix.setter + def prefix(self, new_prefix): + new_data = {"prefix": new_prefix} + W.hdata_update(self._hdata, self._ptr, new_data) + + @property + def message(self): + return W.hdata_string(self._hdata, self._ptr, "message") + + @message.setter + def message(self, new_message): + # type: (str) -> None + new_data = {"message": new_message} + W.hdata_update(self._hdata, self._ptr, new_data) + + @property + def tags(self): + tags_count = W.hdata_get_var_array_size( + self._hdata, + self._ptr, + "tags_array" + ) + + tags = [ + W.hdata_string(self._hdata, self._ptr, "%d|tags_array" % i) + for i in range(tags_count) + ] + return tags + + @tags.setter + def tags(self, new_tags): + # type: (List[str]) -> None + new_data = {"tags_array": ",".join(new_tags)} + W.hdata_update(self._hdata, self._ptr, new_data) + + @property + def date(self): + # type: () -> int + return W.hdata_time(self._hdata, self._ptr, "date") + + @date.setter + def date(self, new_date): + # type: (int) -> None + new_data = {"date": new_date} + W.hdata_update(self._hdata, self._ptr, new_data) + + @property + def date_printed(self): + # type: () -> int + return W.hdata_time(self._hdata, self._ptr, "date_printed") + + @date_printed.setter + def date_printed(self, new_date): + # type: (int) -> None + new_data = {"date_printed": new_date} + W.hdata_update(self._hdata, self._ptr, new_data) + + @property + def highlight(self): + # type: () -> bool + return bool(W.hdata_char(self._hdata, self._ptr, "highlight")) + + def update(self, date, date_printed, tags, prefix, message): + new_data = { + "date": date, + "date_printed": date_printed, + "tags_array": ','.join(tags), + "prefix": prefix, + "message": message, + # "highlight": highlight + } + W.hdata_update(self._hdata, self._ptr, new_data) + def __init__(self, name, server_name, user): # type: (str, str, str) self._ptr = W.buffer_new( @@ -186,6 +295,8 @@ class WeechatChannelBuffer(object): W.buffer_set(self._ptr, "nicklist", "1") W.buffer_set(self._ptr, "nicklist_display_groups", "0") + W.buffer_set(self._ptr, "highlight_words", user) + # TODO make this configurable W.buffer_set( self._ptr, @@ -193,6 +304,36 @@ class WeechatChannelBuffer(object): SCRIPT_NAME + "_message" ) + @property + def _hdata(self): + return W.hdata_get("buffer") + + @property + def lines(self): + own_lines = W.hdata_pointer( + self._hdata, + self._ptr, + "own_lines" + ) + + if own_lines: + hdata_line = W.hdata_get("line") + + line_pointer = W.hdata_pointer( + W.hdata_get("lines"), own_lines, "last_line") + + while line_pointer: + data_pointer = W.hdata_pointer( + hdata_line, + line_pointer, + "data" + ) + + if data_pointer: + yield WeechatChannelBuffer.Line(data_pointer) + + line_pointer = W.hdata_move(hdata_line, line_pointer, -1) + def _print(self, string): # type: (str) -> None """ Print a string to the room buffer """ @@ -247,11 +388,7 @@ class WeechatChannelBuffer(object): # A message from a non joined user return RoomUser(nick) - def message(self, nick, message, date, tags=[]): - # type: (str, str, int, str) -> None - user = self._get_user(nick) - tags = tags or self._message_tags(user, "message") - + def _print_message(self, user, message, date, tags): prefix_string = ("" if not user.prefix else "{}{}{}".format( W.color(self._get_prefix_color(user.prefix)), user.prefix, @@ -267,6 +404,12 @@ class WeechatChannelBuffer(object): self.print_date_tags(data, date, tags) + def message(self, nick, message, date, extra_tags=[]): + # type: (str, str, int, str) -> None + user = self._get_user(nick) + tags = self._message_tags(user, "message") + extra_tags + self._print_message(user, message, date, tags) + def notice(self, nick, message, date): # type: (str, str, int) -> None data = "{color}{message}{ncolor}".format( @@ -276,11 +419,7 @@ class WeechatChannelBuffer(object): self.message(nick, data, date) - def action(self, nick, message, date, tags=[]): - # type: (str, str, int) -> None - user = self._get_user(nick) - tags = tags or self._message_tags(user, "action") - + def _print_action(self, user, message, date, tags): nick_prefix = ("" if not user.prefix else "{}{}{}".format( W.color(self._get_prefix_color(user.prefix)), user.prefix, @@ -292,12 +431,18 @@ class WeechatChannelBuffer(object): prefix=W.prefix("action"), nick_prefix=nick_prefix, nick_color=W.color(user.color), - author=nick, + author=user.nick, ncolor=W.color("reset"), msg=message) self.print_date_tags(data, date, tags) + def action(self, nick, message, date, extra_tags=[]): + # type: (str, str, int) -> None + user = self._get_user(nick) + tags = self._message_tags(user, "action") + extra_tags + self._print_action(user, message, date, tags) + @staticmethod def _get_nicklist_group(user): # type: (WeechatUser) -> str @@ -456,13 +601,13 @@ class WeechatChannelBuffer(object): def self_message(self, nick, message, date): user = self._get_user(nick) tags = self._message_tags(user, "self_message") - self.message(nick, message, date, tags) + self._print_message(user, message, date, tags) def self_action(self, nick, message, date): user = self._get_user(nick) tags = self._message_tags(user, "self_message") tags.append(SCRIPT_NAME + "_action") - self.action(nick, message, date, tags) + self._print_action(user, message, date, tags) @property def short_name(self): @@ -471,3 +616,226 @@ class WeechatChannelBuffer(object): @short_name.setter def short_name(self, name): W.buffer_set(self._ptr, "short_name", name) + + def find_lines(self, predicate): + lines = [] + for line in self.lines: + if predicate(line): + lines.append(line) + + return lines + + +class RoomBuffer(object): + def __init__(self, room, server_name): + self.room = room + user = shorten_sender(self.room.own_user_id) + self.weechat_buffer = WeechatChannelBuffer( + room.room_id, + server_name, + user + ) + + def handle_membership_events(self, event, is_state): + def join(event, date, is_state): + user = self.room.users[event.sender] + buffer_user = RoomUser(user.name, event.sender) + # TODO remove this duplication + user.nick_color = buffer_user.color + + if self.room.own_user_id == event.sender: + buffer_user.color = "weechat.color.chat_nick_self" + user.nick_color = "weechat.color.chat_nick_self" + + self.weechat_buffer.join( + buffer_user, + server_ts_to_weechat(event.timestamp), + not is_state + ) + + date = server_ts_to_weechat(event.timestamp) + + if isinstance(event, RoomMemberJoin): + if event.prev_content and "membership" in event.prev_content: + if (event.prev_content["membership"] == "leave" + or event.prev_content["membership"] == "invite"): + join(event, date, is_state) + else: + # TODO print out profile changes + return + else: + # No previous content for this user in this room, so he just + # joined. + join(event, date, is_state) + + elif isinstance(event, RoomMemberLeave): + # TODO the nick can be a display name or a full sender name + nick = shorten_sender(event.sender) + if event.sender == event.leaving_user: + self.weechat_buffer.part(nick, date, not is_state) + else: + self.weechat_buffer.kick(nick, date, not is_state) + + elif isinstance(event, RoomMemberInvite): + if is_state: + return + + self.weechat_buffer.invite(event.invited_user, date) + return + + room_name = self.room.display_name(self.room.own_user_id) + self.weechat_buffer.short_name = room_name + + def _redact_line(self, event): + def predicate(event_id, line): + def already_redacted(tags): + if SCRIPT_NAME + "_redacted" in tags: + return True + return False + + event_tag = SCRIPT_NAME + "_id_{}".format(event_id) + tags = line.tags + + if event_tag in tags and not already_redacted(tags): + return True + + return False + + lines = self.weechat_buffer.find_lines( + partial(predicate, event.redaction_id) + ) + + # No line to redact, return early + if not lines: + return + + # TODO multiple lines can contain a single matrix ID, we need to redact + # them all + line = lines[0] + + # TODO the censor may not be in the room anymore + censor = self.room.users[event.sender].name + message = line.message + tags = line.tags + + reason = ("" if not event.reason else + ", reason: \"{reason}\"".format(reason=event.reason)) + + redaction_msg = ("{del_color}<{log_color}Message redacted by: " + "{censor}{log_color}{reason}{del_color}>" + "{ncolor}").format( + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + log_color=W.color("logger.color.backlog_line"), + censor=censor, + reason=reason) + + new_message = "" + + if OPTIONS.redaction_type == RedactType.STRIKETHROUGH: + plaintext_msg = W.string_remove_color(message, '') + new_message = string_strikethrough(plaintext_msg) + elif OPTIONS.redaction_type == RedactType.NOTICE: + new_message = message + elif OPTIONS.redaction_type == RedactType.DELETE: + pass + + message = " ".join(s for s in [new_message, redaction_msg] if s) + + tags.append("matrix_redacted") + + line.message = message + line.tags = tags + + def _handle_redacted_message(self, event): + # TODO user doesn't have to be in the room anymore + user = self.room.users[event.sender] + date = server_ts_to_weechat(event.timestamp) + tags = self.get_event_tags(event) + tags.append(SCRIPT_NAME + "_redacted") + + reason = (", reason: \"{reason}\"".format(reason=event.reason) + if event.reason else "") + + censor = self.room.users[event.censor] + + data = ("{del_color}<{log_color}Message redacted by: " + "{censor}{log_color}{reason}{del_color}>{ncolor}").format( + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + log_color=W.color("logger.color.backlog_line"), + censor=censor.name, + reason=reason) + + self.weechat_buffer.message(user.name, data, date, tags) + + def _handle_topic(self, event, is_state): + try: + user = self.room.users[event.sender] + nick = user.name + except KeyError: + nick = event.sender + + self.weechat_buffer.change_topic( + nick, + event.topic, + server_ts_to_weechat(event.timestamp), + not is_state) + + @staticmethod + def get_event_tags(event): + return ["matrix_id_{}".format(event.event_id)] + + def handle_state_event(self, event): + if isinstance(event, RoomMembershipEvent): + self.handle_membership_events(event, True) + elif isinstance(event, RoomTopicEvent): + self._handle_topic(event, True) + + def handle_timeline_event(self, event): + if isinstance(event, RoomMembershipEvent): + self.handle_membership_events(event, False) + elif isinstance(event, (RoomNameEvent, RoomAliasEvent)): + room_name = self.room.display_name(self.room.own_user_id) + self.weechat_buffer.short_name = room_name + elif isinstance(event, RoomTopicEvent): + self._handle_topic(event, False) + elif isinstance(event, RoomMessageText): + user = self.room.users[event.sender] + data = (event.formatted_message.to_weechat() + if event.formatted_message else event.message) + + date = server_ts_to_weechat(event.timestamp) + self.weechat_buffer.message( + user.name, + data, + date, + self.get_event_tags(event) + ) + elif isinstance(event, RoomMessageEmote): + user = self.room.users[event.sender] + date = server_ts_to_weechat(event.timestamp) + self.weechat_buffer.action( + user.name, + event.message, + date, + self.get_event_tags(event) + ) + elif isinstance(event, RoomRedactionEvent): + self._redact_line(event) + elif isinstance(event, RoomRedactedMessageEvent): + self._handle_redacted_message(event) + + def self_message(self, message): + user = self.room.users[self.room.own_user_id] + data = (message.formatted_message.to_weechat() + if message.formatted_message + else message.message) + + date = server_ts_to_weechat(message.timestamp) + self.weechat_buffer.self_message(user.name, data, date) + + def self_action(self, message): + user = self.room.users[self.room.own_user_id] + date = server_ts_to_weechat(message.timestamp) + self.weechat_buffer.self_action(user.name, message.message, date) diff --git a/matrix/rooms.py b/matrix/rooms.py index 0c2c29c..b5e6556 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -40,10 +40,11 @@ PowerLevel = namedtuple('PowerLevel', ['user', 'level']) class MatrixRoom: - def __init__(self, room_id): + def __init__(self, room_id, own_user_id): # type: (str) -> None # yapf: disable self.room_id = room_id # type: str + self.own_user_id = own_user_id self.canonical_alias = None # type: str self.name = None # type: str self.topic = "" # type: str @@ -153,9 +154,23 @@ class MatrixRoom: short_name = shorten_sender(event.sender) user = MatrixUser(short_name, event.display_name) self.users[event.sender] = user + return True + elif isinstance(event, RoomMemberLeave): if event.leaving_user in self.users: del self.users[event.leaving_user] + return True + + elif isinstance(event, RoomNameEvent): + self.name = event.name + + elif isinstance(event, RoomAliasEvent): + self.canonical_alias = event.canonical_alias + + elif isinstance(event, RoomEncryptionEvent): + self.encrypted = True + + return False class MatrixUser: @@ -337,35 +352,6 @@ class RoomRedactedMessageEvent(RoomEvent): return cls(event_id, sender, timestamp, censor, reason) - def execute(self, server, room, buff, tags): - nick, color_name = sender_to_nick_and_color(room, self.sender) - color = color_for_tags(color_name) - date = server_ts_to_weechat(self.timestamp) - - event_tags = add_event_tags(self.event_id, nick, color, tags) - - reason = (", reason: \"{reason}\"".format(reason=self.reason) - if self.reason else "") - - censor, _ = sender_to_nick_and_color(room, self.censor) - - msg = ("{del_color}<{log_color}Message redacted by: " - "{censor}{log_color}{reason}{del_color}>{ncolor}").format( - del_color=W.color("chat_delimiters"), - ncolor=W.color("reset"), - log_color=W.color("logger.color.backlog_line"), - censor=censor, - reason=reason) - - event_tags.append("matrix_redacted") - - tags_string = ",".join(event_tags) - - data = "{author}\t{msg}".format(author=nick, msg=msg) - - W.prnt_date_tags(buff, date, tags_string, data) - - class RoomMessageEvent(RoomEvent): @classmethod @@ -464,12 +450,6 @@ class RoomMessageText(RoomMessageEvent): return cls(event_id, sender, timestamp, msg, formatted_msg) - def execute(self, server, room, buff, tags): - msg = (self.formatted_message.to_weechat() - if self.formatted_message else self.message) - - self._print_message(msg, room, buff, tags) - class RoomMessageEmote(RoomMessageSimple): @@ -688,60 +668,6 @@ class RoomRedactionEvent(RoomEvent): return cls(event_id, sender, timestamp, redaction_id, reason) - @staticmethod - def already_redacted(tags): - if "matrix_redacted" in tags: - return True - return False - - def _redact_line(self, data_pointer, tags, room, buff): - hdata_line_data = W.hdata_get('line_data') - - message = W.hdata_string(hdata_line_data, data_pointer, 'message') - censor, _ = sender_to_nick_and_color(room, self.sender) - - reason = ("" if not self.reason else - ", reason: \"{reason}\"".format(reason=self.reason)) - - redaction_msg = ("{del_color}<{log_color}Message redacted by: " - "{censor}{log_color}{reason}{del_color}>" - "{ncolor}").format( - del_color=W.color("chat_delimiters"), - ncolor=W.color("reset"), - log_color=W.color("logger.color.backlog_line"), - censor=censor, - reason=reason) - - new_message = "" - - if OPTIONS.redaction_type == RedactType.STRIKETHROUGH: - plaintext_msg = W.string_remove_color(message, '') - new_message = string_strikethrough(plaintext_msg) - elif OPTIONS.redaction_type == RedactType.NOTICE: - new_message = message - elif OPTIONS.redaction_type == RedactType.DELETE: - pass - - message = " ".join(s for s in [new_message, redaction_msg] if s) - - tags.append("matrix_redacted") - - new_data = {'tags_array': ','.join(tags), 'message': message} - - W.hdata_update(hdata_line_data, data_pointer, new_data) - - def execute(self, server, room, buff, tags): - data_pointer, tags = line_pointer_and_tags_from_event( - buff, self.redaction_id) - - if not data_pointer: - return - - if RoomRedactionEvent.already_redacted(tags): - return - - self._redact_line(data_pointer, tags, room, buff) - class RoomNameEvent(RoomEvent): @@ -759,18 +685,6 @@ class RoomNameEvent(RoomEvent): return cls(event_id, sender, timestamp, name) - def execute(self, server, room, buff, tags): - if not self.name: - return - - room.name = self.name - W.buffer_set(buff, "name", self.name) - W.buffer_set(buff, "localvar_set_channel", self.name) - - # calculate room display name and set it as the buffer list name - room_name = room.display_name(server.user_id) - W.buffer_set(buff, "short_name", room_name) - class RoomAliasEvent(RoomEvent): @@ -788,19 +702,6 @@ class RoomAliasEvent(RoomEvent): return cls(event_id, sender, timestamp, canonical_alias) - def execute(self, server, room, buff, tags): - if not self.canonical_alias: - return - - # TODO: What should we do with this? - # W.buffer_set(buff, "name", self.name) - # W.buffer_set(buff, "localvar_set_channel", self.name) - - # calculate room display name and set it as the buffer list name - room.canonical_alias = self.canonical_alias - room_name = room.display_name(server.user_id) - W.buffer_set(buff, "short_name", room_name) - class RoomEncryptionEvent(RoomEvent): diff --git a/matrix/server.py b/matrix/server.py index f5278c8..db7ed5e 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -34,7 +34,7 @@ from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, from matrix.utf import utf8_decode from matrix.globals import W, SERVERS, OPTIONS import matrix.api as API -from .buffer import WeechatChannelBuffer, RoomUser +from .buffer import RoomBuffer from .rooms import ( MatrixRoom, RoomMessageText, @@ -633,101 +633,6 @@ class MatrixServer: server_buffer_prnt(self, pprint.pformat(message.request.payload)) server_buffer_prnt(self, pprint.pformat(message.response.body)) - def handle_room_membership_events( - self, - room, - room_buffer, - event, - is_state_event - ): - def join(event, date, room, room_buffer, is_state_event): - user = room.users[event.sender] - buffer_user = RoomUser(user.name, event.sender) - # TODO remove this duplication - user.nick_color = buffer_user.color - - if self.user_id == event.sender: - buffer_user.color = "weechat.color.chat_nick_self" - user.nick_color = "weechat.color.chat_nick_self" - - room_buffer.join( - buffer_user, - server_ts_to_weechat(event.timestamp), - not is_state_event - ) - - room.handle_event(event) - date = server_ts_to_weechat(event.timestamp) - - joined = False - left = False - - if isinstance(event, RoomMemberJoin): - if event.prev_content and "membership" in event.prev_content: - if (event.prev_content["membership"] == "leave" - or event.prev_content["membership"] == "invite"): - join(event, date, room, room_buffer, is_state_event) - joined = True - else: - # TODO print out profile changes - return - else: - # No previous content for this user in this room, so he just - # joined. - join(event, date, room, room_buffer, is_state_event) - joined = True - - elif isinstance(event, RoomMemberLeave): - # TODO the nick can be a display name or a full sender name - nick = shorten_sender(event.sender) - if event.sender == event.leaving_user: - room_buffer.part(nick, date, not is_state_event) - else: - room_buffer.kick(nick, date, not is_state_event) - - left = True - - elif isinstance(event, RoomMemberInvite): - if is_state_event: - return - - room_buffer.invite(event.invited_user, date) - return - - # calculate room display name and set it as the buffer list name - room_name = room.display_name(self.user_id) - room_buffer.short_name = room_name - - # A user has joined or left an encrypted room, we need to check for - # new devices and create a new group session - if room.encrypted and (joined or left): - self.device_check_timestamp = None - - def handle_room_event(self, room, room_buffer, event, is_state_event): - if isinstance(event, RoomMembershipEvent): - self.handle_room_membership_events( - room, - room_buffer, - event, - is_state_event - ) - elif isinstance(event, RoomTopicEvent): - try: - user = room.users[event.sender] - nick = user.name - except KeyError: - nick = event.sender - - room_buffer.change_topic( - nick, - event.topic, - server_ts_to_weechat(event.timestamp), - not is_state_event - ) - else: - tags = tags_for_message("message") - event.execute(self, room, room_buffer._ptr, tags) - def _loop_events(self, info, n): for i in range(n+1): @@ -742,7 +647,15 @@ class MatrixServer: return i room, room_buffer = self.find_room_from_id(info.room_id) - self.handle_room_event(room, room_buffer, event, is_state) + # The room changed it's members, if the room is encrypted update + # the device list + if room.handle_event(event) and room.encrypted: + self.device_check_timestamp = None + + if is_state: + room_buffer.handle_state_event(event) + else: + room_buffer.handle_timeline_event(event) self.event_queue.appendleft(info) return i @@ -777,18 +690,10 @@ class MatrixServer: def handle_own_messages(self, room_buffer, message): if isinstance(message, RoomMessageText): - msg = (message.formatted_message.to_weechat() - if message.formatted_message - else message.message) - - date = server_ts_to_weechat(message.timestamp) - room_buffer.self_message(self.user, msg, date) - + room_buffer.self_message(message) return elif isinstance(message, RoomMessageEmote): - date = server_ts_to_weechat(message.timestamp) - room_buffer.self_action(self.user, message.message, date) - + room_buffer.self_action(message) return raise NotImplementedError("Unsupported message of type {}".format( @@ -850,12 +755,12 @@ class MatrixServer: return def create_room_buffer(self, room_id): - buf = WeechatChannelBuffer(room_id, self.name, self.user) + room = MatrixRoom(room_id, self.user_id) + buf = RoomBuffer(room, self.name) # TODO this should turned into a propper class self.room_buffers[room_id] = buf - self.buffers[room_id] = buf._ptr - self.rooms[room_id] = MatrixRoom(room_id) - pass + self.buffers[room_id] = buf.weechat_buffer._ptr + self.rooms[room_id] = room def find_room_from_ptr(self, pointer): room_id = key_from_value(self.buffers, pointer) From 1e3e66e3503b6a7758430860a4776f87d4f3eafc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 6 Jul 2018 15:03:00 +0200 Subject: [PATCH 043/269] buffer: Remember which name we display in the buffer. --- matrix/buffer.py | 59 ++++++++++++++++++++++++++++-------------------- matrix/rooms.py | 2 -- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 0d39fe1..19afc3d 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -629,6 +629,10 @@ class WeechatChannelBuffer(object): class RoomBuffer(object): def __init__(self, room, server_name): self.room = room + + # This dict remembers the connection from a user_id to the name we + # displayed in the buffer + self.displayed_nicks = {} user = shorten_sender(self.room.own_user_id) self.weechat_buffer = WeechatChannelBuffer( room.room_id, @@ -636,12 +640,21 @@ class RoomBuffer(object): user ) + def find_nick(self, user_id): + # type: (str) -> str + """Find a suitable nick from a user_id""" + if user_id in self.displayed_nicks: + return self.displayed_nicks[user_id] + + return user_id + def handle_membership_events(self, event, is_state): def join(event, date, is_state): user = self.room.users[event.sender] - buffer_user = RoomUser(user.name, event.sender) - # TODO remove this duplication - user.nick_color = buffer_user.color + buffer_user = RoomUser(user.name, event.sender, user.power_level) + # TODO we need to check that the displayed nick is unique if it + # isn't use the full user_id + self.displayed_nicks[event.sender] = user.name if self.room.own_user_id == event.sender: buffer_user.color = "weechat.color.chat_nick_self" @@ -669,13 +682,15 @@ class RoomBuffer(object): join(event, date, is_state) elif isinstance(event, RoomMemberLeave): - # TODO the nick can be a display name or a full sender name - nick = shorten_sender(event.sender) + nick = self.find_nick(event.leaving_user) if event.sender == event.leaving_user: self.weechat_buffer.part(nick, date, not is_state) else: self.weechat_buffer.kick(nick, date, not is_state) + if event.leaving_user in self.displayed_nicks: + del self.displayed_nicks[event.leaving_user] + elif isinstance(event, RoomMemberInvite): if is_state: return @@ -713,8 +728,7 @@ class RoomBuffer(object): # them all line = lines[0] - # TODO the censor may not be in the room anymore - censor = self.room.users[event.sender].name + censor = self.find_nick(event.sender) message = line.message tags = line.tags @@ -748,8 +762,7 @@ class RoomBuffer(object): line.tags = tags def _handle_redacted_message(self, event): - # TODO user doesn't have to be in the room anymore - user = self.room.users[event.sender] + nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.timestamp) tags = self.get_event_tags(event) tags.append(SCRIPT_NAME + "_redacted") @@ -757,24 +770,20 @@ class RoomBuffer(object): reason = (", reason: \"{reason}\"".format(reason=event.reason) if event.reason else "") - censor = self.room.users[event.censor] + censor = self.find_nick(event.censor) data = ("{del_color}<{log_color}Message redacted by: " "{censor}{log_color}{reason}{del_color}>{ncolor}").format( del_color=W.color("chat_delimiters"), ncolor=W.color("reset"), log_color=W.color("logger.color.backlog_line"), - censor=censor.name, + censor=censor, reason=reason) - self.weechat_buffer.message(user.name, data, date, tags) + self.weechat_buffer.message(nick, data, date, tags) def _handle_topic(self, event, is_state): - try: - user = self.room.users[event.sender] - nick = user.name - except KeyError: - nick = event.sender + nick = self.find_nick(event.sender) self.weechat_buffer.change_topic( nick, @@ -801,22 +810,22 @@ class RoomBuffer(object): elif isinstance(event, RoomTopicEvent): self._handle_topic(event, False) elif isinstance(event, RoomMessageText): - user = self.room.users[event.sender] + nick = self.find_nick(event.sender) data = (event.formatted_message.to_weechat() if event.formatted_message else event.message) date = server_ts_to_weechat(event.timestamp) self.weechat_buffer.message( - user.name, + nick, data, date, self.get_event_tags(event) ) elif isinstance(event, RoomMessageEmote): - user = self.room.users[event.sender] + nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.timestamp) self.weechat_buffer.action( - user.name, + nick, event.message, date, self.get_event_tags(event) @@ -827,15 +836,15 @@ class RoomBuffer(object): self._handle_redacted_message(event) def self_message(self, message): - user = self.room.users[self.room.own_user_id] + nick = self.find_nick(self.room.own_user_id) data = (message.formatted_message.to_weechat() if message.formatted_message else message.message) date = server_ts_to_weechat(message.timestamp) - self.weechat_buffer.self_message(user.name, data, date) + self.weechat_buffer.self_message(nick, data, date) def self_action(self, message): - user = self.room.users[self.room.own_user_id] + nick = self.find_nick(self.room.own_user_id) date = server_ts_to_weechat(message.timestamp) - self.weechat_buffer.self_action(user.name, message.message, date) + self.weechat_buffer.self_action(nick, message.message, date) diff --git a/matrix/rooms.py b/matrix/rooms.py index b5e6556..717d5ae 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -180,8 +180,6 @@ class MatrixUser: self.name = name # type: str self.display_name = display_name # type: str self.power_level = 0 # type: int - self.nick_color = "" # type: str - self.prefix = "" # type: str # yapf: enable From c7a8bc177f89b53c42adb8753ea20c289ffe321b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 6 Jul 2018 15:39:52 +0200 Subject: [PATCH 044/269] encryption: Update group decryption to unpack the message index. --- matrix/rooms.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/matrix/rooms.py b/matrix/rooms.py index 717d5ae..f1ff806 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -258,7 +258,11 @@ class RoomInfo(): def _decrypt_event(olm, room_id, event_dict): session_id = event_dict["content"]["session_id"] ciphertext = event_dict["content"]["ciphertext"] - plaintext = olm.group_decrypt(room_id, session_id, ciphertext) + plaintext, message_index = olm.group_decrypt( + room_id, + session_id, + ciphertext + ) if not plaintext: return None From b665cc4f0e4126b7f92ef772b1bf948aef990f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 6 Jul 2018 17:05:50 +0200 Subject: [PATCH 045/269] buffer: Add support for more message types. --- matrix/api.py | 15 ++++++------ matrix/buffer.py | 62 +++++++++++++++++++++++++++++++++++++++++++++--- matrix/rooms.py | 58 +++----------------------------------------- matrix/utils.py | 22 +++++++++++++++++ 4 files changed, 91 insertions(+), 66 deletions(-) diff --git a/matrix/api.py b/matrix/api.py index 0bbf4e6..0919bbb 100644 --- a/matrix/api.py +++ b/matrix/api.py @@ -333,7 +333,8 @@ class MatrixClient: return HttpRequest(RequestType.PUT, self.host, path, content) - def mxc_to_http(self, mxc): + @staticmethod + def mxc_to_http(mxc): # type: (str) -> str url = urlparse(mxc) @@ -343,13 +344,11 @@ class MatrixClient: if not url.netloc or not url.path: return None - http_url = ("https://{host}/_matrix/media/r0/download/" - "{server_name}{mediaId}").format( - host=self.host, - server_name=url.netloc, - mediaId=url.path) - - return http_url + return "https://{}/_matrix/media/r0/download/{}{}".format( + url.netloc, + url.netloc, + url.path + ) class MatrixMessage(): diff --git a/matrix/buffer.py b/matrix/buffer.py index 19afc3d..82a6d9e 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -21,10 +21,15 @@ import time from builtins import super from functools import partial -from .globals import W, SERVERS, OPTIONS, SCRIPT_NAME +from .globals import W, SERVERS, OPTIONS, SCRIPT_NAME, ENCRYPTION from .utf import utf8_decode from .colors import Formatted -from .utils import shorten_sender, server_ts_to_weechat, string_strikethrough +from .utils import ( + shorten_sender, + server_ts_to_weechat, + string_strikethrough, + mxc_to_http +) from .plugin_options import RedactType @@ -38,8 +43,12 @@ from .rooms import ( RoomTopicEvent, RoomMessageText, RoomMessageEmote, + RoomMessageNotice, + RoomMessageMedia, + RoomMessageUnknown, RoomRedactionEvent, - RoomRedactedMessageEvent + RoomRedactedMessageEvent, + RoomEncryptionEvent ) @@ -804,11 +813,14 @@ class RoomBuffer(object): def handle_timeline_event(self, event): if isinstance(event, RoomMembershipEvent): self.handle_membership_events(event, False) + elif isinstance(event, (RoomNameEvent, RoomAliasEvent)): room_name = self.room.display_name(self.room.own_user_id) self.weechat_buffer.short_name = room_name + elif isinstance(event, RoomTopicEvent): self._handle_topic(event, False) + elif isinstance(event, RoomMessageText): nick = self.find_nick(event.sender) data = (event.formatted_message.to_weechat() @@ -821,6 +833,7 @@ class RoomBuffer(object): date, self.get_event_tags(event) ) + elif isinstance(event, RoomMessageEmote): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.timestamp) @@ -830,11 +843,54 @@ class RoomBuffer(object): date, self.get_event_tags(event) ) + + elif isinstance(event, RoomMessageNotice): + nick = self.find_nick(event.sender) + date = server_ts_to_weechat(event.timestamp) + self.weechat_buffer.notice( + nick, + event.message, + date, + self.get_event_tags(event) + ) + elif isinstance(event, RoomMessageMedia): + nick = self.find_nick(event.sender) + date = server_ts_to_weechat(event.timestamp) + http_url = mxc_to_http(event.url) + url = http_url if http_url else event.url + + description = ("/{}".format(event.description) + if event.description else "") + data = "{url}{desc}".format(url=url, desc=description) + + self.weechat_buffer.message( + nick, + data, + date, + self.get_event_tags(event) + ) + + elif isinstance(event, RoomMessageUnknown): + nick = self.find_nick(event.sender) + date = server_ts_to_weechat(event.timestamp) + data = ("Unknown message of type {t}, body: {body}").format( + t=self.message_type, body=self.message) + elif isinstance(event, RoomRedactionEvent): self._redact_line(event) + elif isinstance(event, RoomRedactedMessageEvent): self._handle_redacted_message(event) + elif isinstance(event, RoomEncryptionEvent): + if ENCRYPTION: + return + + message = ("This room is encrypted, encryption is " + "currently unsuported. Message sending is disabled for " + "this room.") + self.weechat_buffer.error(message) + def self_message(self, message): nick = self.find_nick(self.room.own_user_id) data = (message.formatted_message.to_weechat() diff --git a/matrix/rooms.py b/matrix/rooms.py index f1ff806..8a1a7c5 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -15,17 +15,14 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import unicode_literals -from builtins import str import json from pprint import pformat from collections import namedtuple, deque -from datetime import datetime -from matrix.globals import W, OPTIONS -from matrix.plugin_options import RedactType +from matrix.globals import W from matrix.colors import Formatted from matrix.utils import ( @@ -47,9 +44,6 @@ class MatrixRoom: self.own_user_id = own_user_id self.canonical_alias = None # type: str self.name = None # type: str - self.topic = "" # type: str - self.topic_author = "" # type: str - self.topic_date = None # type: datetime.datetime self.prev_batch = "" # type: str self.users = dict() # type: Dict[str, MatrixUser] self.encrypted = False # type: bool @@ -115,7 +109,6 @@ class MatrixRoom: else: return "Empty room?" - def machine_name(self): """ Calculate an unambiguous, unique machine name for a room. @@ -454,36 +447,11 @@ class RoomMessageText(RoomMessageEvent): class RoomMessageEmote(RoomMessageSimple): - - def execute(self, server, room, buff, tags): - nick, color_name = sender_to_nick_and_color(room, self.sender) - color = color_for_tags(color_name) - - event_tags = add_event_tags(self.event_id, nick, color, tags) - event_tags.append("matrix_action") - - tags_string = ",".join(event_tags) - - data = "{prefix}{nick_color}{author}{ncolor} {msg}".format( - prefix=W.prefix("action"), - nick_color=W.color(color_name), - author=nick, - ncolor=W.color("reset"), - msg=self.message) - - date = server_ts_to_weechat(self.timestamp) - W.prnt_date_tags(buff, date, tags_string, data) + pass class RoomMessageNotice(RoomMessageText): - - def execute(self, server, room, buff, tags): - msg = "{color}{message}{ncolor}".format( - color=W.color("irc.color.notice"), - message=self.message, - ncolor=W.color("reset")) - - self._print_message(msg, room, buff, tags) + pass class RoomMessageMedia(RoomMessageEvent): @@ -504,17 +472,6 @@ class RoomMessageMedia(RoomMessageEvent): return cls(event_id, sender, timestamp, mxc_url, description) - def execute(self, server, room, buff, tags): - http_url = server.client.mxc_to_http(self.url) - url = http_url if http_url else self.url - - description = (" ({})".format(self.description) - if self.description else "") - - msg = "{url}{desc}".format(url=url, desc=description) - - self._print_message(msg, room, buff, tags) - class RoomMemberJoin(RoomMembershipEvent): @@ -714,12 +671,3 @@ class RoomEncryptionEvent(RoomEvent): timestamp = sanitize_ts(event_dict["origin_server_ts"]) return cls(event_id, sender, timestamp) - - def execute(self, server, room, buff, tags): - room.encrypted = True - - message = ("{prefix}This room is encrypted, encryption is " - "currently unsuported. Message sending is disabled for " - "this room.").format(prefix=W.prefix("error")) - - W.prnt(buff, message) diff --git a/matrix/utils.py b/matrix/utils.py index 9d8d437..15b3e6f 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -24,6 +24,11 @@ from matrix.globals import W, SERVERS, OPTIONS from matrix.plugin_options import ServerBufferType +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse + def key_from_value(dictionary, value): # type: (Dict[str, Any], Any) -> str @@ -350,3 +355,20 @@ def event_id_from_tags(tags): return tag[10:] return "" + + +def mxc_to_http(mxc): + # type: (str) -> str + url = urlparse(mxc) + + if url.scheme != "mxc": + return None + + if not url.netloc or not url.path: + return None + + return "https://{}/_matrix/media/r0/download/{}{}".format( + url.netloc, + url.netloc, + url.path + ) From eb8a83010dfa39d3cefdf265bffa503424670fc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 6 Jul 2018 18:16:13 +0200 Subject: [PATCH 046/269] buffer: Handle power level events. --- matrix/buffer.py | 37 ++++++++++++++++++++++++++++++++++++- matrix/rooms.py | 38 +++++++++++++------------------------- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 82a6d9e..aafdfc4 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -48,7 +48,8 @@ from .rooms import ( RoomMessageUnknown, RoomRedactionEvent, RoomRedactedMessageEvent, - RoomEncryptionEvent + RoomEncryptionEvent, + RoomPowerLevels ) @@ -92,6 +93,22 @@ class RoomUser(WeechatUser): prefix = self._get_prefix(power_level) return super().__init__(nick, user_id, prefix) + @property + def power_level(self): + # This shouldn't be used since it's a lossy function. It's only here + # for the setter + if self.prefix == "&": + return 100 + if self.prefix == "@": + return 50 + if self.prefix == "+": + return 10 + return 0 + + @power_level.setter + def power_level(self, level): + self.prefix = self._get_prefix(level) + @staticmethod def _get_prefix(power_level): # type: (int) -> str @@ -804,11 +821,26 @@ class RoomBuffer(object): def get_event_tags(event): return ["matrix_id_{}".format(event.event_id)] + def _handle_power_level(self, event): + for user_id in self.room.power_levels: + if user_id in self.displayed_nicks: + nick = self.find_nick(user_id) + + user = self.weechat_buffer.users[nick] + user.power_level = self.room.power_levels[user_id] + + # There is no way to change the group of a user without + # removing him from the nicklist + self.weechat_buffer._remove_user_from_nicklist(user) + self.weechat_buffer._add_user_to_nicklist(user) + def handle_state_event(self, event): if isinstance(event, RoomMembershipEvent): self.handle_membership_events(event, True) elif isinstance(event, RoomTopicEvent): self._handle_topic(event, True) + elif isinstance(event, RoomPowerLevels): + self._handle_power_level(event) def handle_timeline_event(self, event): if isinstance(event, RoomMembershipEvent): @@ -891,6 +923,9 @@ class RoomBuffer(object): "this room.") self.weechat_buffer.error(message) + elif isinstance(event, RoomPowerLevels): + self._handle_power_level(event) + def self_message(self, message): nick = self.find_nick(self.room.own_user_id) data = (message.formatted_message.to_weechat() diff --git a/matrix/rooms.py b/matrix/rooms.py index 8a1a7c5..10c9a51 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -48,6 +48,7 @@ class MatrixRoom: self.users = dict() # type: Dict[str, MatrixUser] self.encrypted = False # type: bool self.backlog_pending = False # type: bool + self.power_levels = {} # yapf: enable def display_name(self, own_user_id): @@ -145,6 +146,9 @@ class MatrixRoom: user.display_name = event.display_name else: short_name = shorten_sender(event.sender) + # TODO the default power level doesn't have to be 0 + level = (self.power_levels[event.sender] if event.sender in + self.power_levels else 0) user = MatrixUser(short_name, event.display_name) self.users[event.sender] = user return True @@ -163,6 +167,14 @@ class MatrixRoom: elif isinstance(event, RoomEncryptionEvent): self.encrypted = True + elif isinstance(event, RoomPowerLevels): + self.power_levels = event.power_levels + + # Update the power levels of the joined users + for user_id, level in self.power_levels.items(): + if user_id in self.users: + self.users[user_id].power_level = level + return False @@ -560,35 +572,11 @@ class RoomPowerLevels(RoomEvent): event_id = sanitize_id(event_dict["event_id"]) sender = sanitize_id(event_dict["sender"]) timestamp = sanitize_ts(event_dict["origin_server_ts"]) - power_levels = [] - for user, level in event_dict["content"]["users"].items(): - power_levels.append( - PowerLevel(sanitize_id(user), sanitize_power_level(level))) + power_levels = event_dict["content"].pop("users") return cls(event_id, sender, timestamp, power_levels) - def _set_power_level(self, room, buff, power_level): - user_id = power_level.user - level = power_level.level - - if user_id not in room.users: - return - - user = room.users[user_id] - user.power_level = level - user.prefix = get_prefix_for_level(level) - - nick_pointer = W.nicklist_search_nick(buff, "", user_id) - - if nick_pointer: - W.nicklist_remove_nick(buff, nick_pointer) - add_user_to_nicklist(buff, user_id, user) - - def execute(self, server, room, buff, tags): - for level in self.power_levels: - self._set_power_level(room, buff, level) - class RoomTopicEvent(RoomEvent): From bc1ad45918d48699955acebf133fd06e4d316309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 6 Jul 2018 18:28:41 +0200 Subject: [PATCH 047/269] buffer: Fix handling of unknown message types. --- matrix/buffer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index aafdfc4..c2ed1cd 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -885,6 +885,7 @@ class RoomBuffer(object): date, self.get_event_tags(event) ) + elif isinstance(event, RoomMessageMedia): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.timestamp) @@ -906,7 +907,15 @@ class RoomBuffer(object): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.timestamp) data = ("Unknown message of type {t}, body: {body}").format( - t=self.message_type, body=self.message) + t=event.message_type, + body=event.message + ) + self.weechat_buffer.message( + nick, + data, + date, + self.get_event_tags(event) + ) elif isinstance(event, RoomRedactionEvent): self._redact_line(event) @@ -926,6 +935,9 @@ class RoomBuffer(object): elif isinstance(event, RoomPowerLevels): self._handle_power_level(event) + else: + W.prnt("", "Unhandled event of type {}.".format(type(event))) + def self_message(self, message): nick = self.find_nick(self.room.own_user_id) data = (message.formatted_message.to_weechat() From e6485c73bb60539fd9fe0a6fb9a0f248014e4e3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 6 Jul 2018 18:39:43 +0200 Subject: [PATCH 048/269] buffer: Add support for multiple users with the same base name. --- matrix/buffer.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index c2ed1cd..96d192f 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -677,10 +677,16 @@ class RoomBuffer(object): def handle_membership_events(self, event, is_state): def join(event, date, is_state): user = self.room.users[event.sender] - buffer_user = RoomUser(user.name, event.sender, user.power_level) - # TODO we need to check that the displayed nick is unique if it - # isn't use the full user_id - self.displayed_nicks[event.sender] = user.name + + # TODO make this configurable + if user.name in self.displayed_nicks.values(): + # Use the full user id, but don't include the @ + nick = event.sender[1:] + else: + nick = user.name + + buffer_user = RoomUser(nick, event.sender, user.power_level) + self.displayed_nicks[event.sender] = nick if self.room.own_user_id == event.sender: buffer_user.color = "weechat.color.chat_nick_self" From d70e651d3e02857e39129f34003dcbdc2e20cfde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 6 Jul 2018 21:12:21 +0200 Subject: [PATCH 049/269] rooms: Remove membership events, just leave the original one. --- matrix/buffer.py | 22 +++++------ matrix/rooms.py | 97 +++++++++++------------------------------------- matrix/server.py | 6 --- 3 files changed, 31 insertions(+), 94 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 96d192f..a170623 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -37,9 +37,6 @@ from .rooms import ( RoomNameEvent, RoomAliasEvent, RoomMembershipEvent, - RoomMemberJoin, - RoomMemberLeave, - RoomMemberInvite, RoomTopicEvent, RoomMessageText, RoomMessageEmote, @@ -700,10 +697,9 @@ class RoomBuffer(object): date = server_ts_to_weechat(event.timestamp) - if isinstance(event, RoomMemberJoin): + if event.content["membership"] == "join": if event.prev_content and "membership" in event.prev_content: - if (event.prev_content["membership"] == "leave" - or event.prev_content["membership"] == "invite"): + if (event.prev_content["membership"] != "join"): join(event, date, is_state) else: # TODO print out profile changes @@ -713,21 +709,21 @@ class RoomBuffer(object): # joined. join(event, date, is_state) - elif isinstance(event, RoomMemberLeave): - nick = self.find_nick(event.leaving_user) - if event.sender == event.leaving_user: + elif event.content["membership"] == "leave": + nick = self.find_nick(event.state_key) + if event.sender == event.state_key: self.weechat_buffer.part(nick, date, not is_state) else: self.weechat_buffer.kick(nick, date, not is_state) - if event.leaving_user in self.displayed_nicks: - del self.displayed_nicks[event.leaving_user] + if event.state_key in self.displayed_nicks: + del self.displayed_nicks[event.state_key] - elif isinstance(event, RoomMemberInvite): + elif event.content["membership"] == "invite": if is_state: return - self.weechat_buffer.invite(event.invited_user, date) + self.weechat_buffer.invite(event.state_key, date) return room_name = self.room.display_name(self.room.own_user_id) diff --git a/matrix/rooms.py b/matrix/rooms.py index 10c9a51..1ca50e6 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -138,26 +138,36 @@ class MatrixRoom: """ return not self.is_named() - def handle_event(self, event): - if isinstance(event, RoomMemberJoin): + def _handle_membership(self, event): + if event.content["membership"] == "join": if event.sender in self.users: user = self.users[event.sender] - if event.display_name: - user.display_name = event.display_name + if "display_name" in event.content: + user.display_name = event.content["display_name"] else: short_name = shorten_sender(event.sender) # TODO the default power level doesn't have to be 0 level = (self.power_levels[event.sender] if event.sender in self.power_levels else 0) - user = MatrixUser(short_name, event.display_name) + display_name = (event.content["display_name"] + if "display_name" in event.content else None) + + user = MatrixUser(short_name, display_name, level) self.users[event.sender] = user return True - elif isinstance(event, RoomMemberLeave): - if event.leaving_user in self.users: - del self.users[event.leaving_user] + elif event.content["membership"] == "leave": + if event.state_key in self.users: + del self.users[event.state_key] return True + elif event.content["membership"] == "invite": + pass + + def handle_event(self, event): + if isinstance(event, RoomMembershipEvent): + self._handle_membership(event) + elif isinstance(event, RoomNameEvent): self.name = event.name @@ -180,11 +190,11 @@ class MatrixRoom: class MatrixUser: - def __init__(self, name, display_name): + def __init__(self, name, display_name=None, power_level=0): # yapf: disable self.name = name # type: str self.display_name = display_name # type: str - self.power_level = 0 # type: int + self.power_level = power_level # type: int # yapf: enable @@ -213,24 +223,6 @@ class RoomInfo(): return RoomMessageEvent.from_dict(event) - @staticmethod - def _membership_from_dict(event_dict): - if (event_dict["content"]["membership"] not in [ - "invite", "join", "knock", "leave", "ban" - ]): - raise ValueError - - if event_dict["content"]["membership"] == "join": - return RoomMemberJoin.from_dict(event_dict) - - elif event_dict["content"]["membership"] == "leave": - return RoomMemberLeave.from_dict(event_dict) - - elif event_dict["content"]["membership"] == "invite": - return RoomMemberInvite.from_dict(event_dict) - - return None - @staticmethod def parse_event(olm, room_id, event_dict): # type: (Dict[Any, Any]) -> (RoomEvent, RoomEvent) @@ -241,7 +233,7 @@ class RoomInfo(): elif event_dict["type"] == "m.room.message": event = RoomInfo._message_from_event(event_dict) elif event_dict["type"] == "m.room.member": - event = RoomInfo._membership_from_dict(event_dict) + event = RoomMembershipEvent.from_dict(event_dict) elif event_dict["type"] == "m.room.power_levels": event = RoomPowerLevels.from_dict(event_dict) elif event_dict["type"] == "m.room.topic": @@ -333,10 +325,6 @@ class RoomEvent(): self.timestamp = timestamp -class RoomMembershipEvent(RoomEvent): - pass - - class RoomRedactedMessageEvent(RoomEvent): def __init__(self, event_id, sender, timestamp, censor, reason=None): @@ -485,19 +473,16 @@ class RoomMessageMedia(RoomMessageEvent): return cls(event_id, sender, timestamp, mxc_url, description) -class RoomMemberJoin(RoomMembershipEvent): - +class RoomMembershipEvent(RoomEvent): def __init__( self, event_id, sender, timestamp, - display_name, state_key, content, prev_content ): - self.display_name = display_name self.state_key = state_key self.content = content self.prev_content = prev_content @@ -509,58 +494,20 @@ class RoomMemberJoin(RoomMembershipEvent): sender = sanitize_id(event_dict["sender"]) timestamp = sanitize_ts(event_dict["origin_server_ts"]) state_key = sanitize_id(event_dict["state_key"]) - display_name = None content = event_dict["content"] prev_content = (event_dict["unsigned"]["prev_content"] if "prev_content" in event_dict["unsigned"] else None) - if event_dict["content"]: - if "display_name" in event_dict["content"]: - display_name = sanitize_text( - event_dict["content"]["displayname"]) - return cls( event_id, sender, timestamp, - display_name, state_key, content, prev_content ) -class RoomMemberInvite(RoomMembershipEvent): - def __init__(self, event_id, sender, invited_user, timestamp): - self.invited_user = invited_user - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event_dict): - event_id = sanitize_id(event_dict["event_id"]) - sender = sanitize_id(event_dict["sender"]) - invited_user = sanitize_id(event_dict["state_key"]) - timestamp = sanitize_ts(event_dict["origin_server_ts"]) - - return cls(event_id, sender, invited_user, timestamp) - - -class RoomMemberLeave(RoomMembershipEvent): - - def __init__(self, event_id, sender, leaving_user, timestamp): - self.leaving_user = leaving_user - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event_dict): - event_id = sanitize_id(event_dict["event_id"]) - sender = sanitize_id(event_dict["sender"]) - leaving_user = sanitize_id(event_dict["state_key"]) - timestamp = sanitize_ts(event_dict["origin_server_ts"]) - - return cls(event_id, sender, leaving_user, timestamp) - - class RoomPowerLevels(RoomEvent): def __init__(self, event_id, sender, timestamp, power_levels): diff --git a/matrix/server.py b/matrix/server.py index db7ed5e..e540915 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -39,12 +39,6 @@ from .rooms import ( MatrixRoom, RoomMessageText, RoomMessageEmote, - MatrixUser, - RoomMembershipEvent, - RoomMemberJoin, - RoomMemberLeave, - RoomMemberInvite, - RoomTopicEvent ) from matrix.api import ( MatrixClient, From 810607dc082b97128b23c341c3f40ffd6152b606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?poljar=20=28Damir=20Jeli=C4=87=29?= Date: Fri, 6 Jul 2018 21:34:27 +0200 Subject: [PATCH 050/269] rooms: Add BadEvent class. This way if one event in a room is not per spec, we don't end up in a broken state. --- matrix/rooms.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/matrix/rooms.py b/matrix/rooms.py index 1ca50e6..09e61a4 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -287,7 +287,8 @@ class RoomInfo(): error=pformat(error), event=pformat(event)) W.prnt("", message) - raise + e = BadEvent.from_dict(event) + events.append(e) return events @@ -325,6 +326,22 @@ class RoomEvent(): self.timestamp = timestamp +class BadEvent(RoomEvent): + def __init__(self, event_id, sender, timestamp, source): + self.source = source + + def from_dict(cls, event): + event_id = (sanitize_id(event["event_id"]) + if "event_id" in event else None) + sender = (sanitize_id(event["sender"]) + if "sender" in event else None) + timestamp = (sanitize_id(event["origin_server_ts"]) + if "origin_server_ts" in event else None) + source = json.dumps(event) + + return cls(event_id, sender, timestamp, source) + + class RoomRedactedMessageEvent(RoomEvent): def __init__(self, event_id, sender, timestamp, censor, reason=None): From c64af4d953ec57f9d6b16d1ad5e450fbabcfa2d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 11 Jul 2018 12:58:43 +0200 Subject: [PATCH 051/269] events: Fix encrypted message sending. --- matrix/events.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matrix/events.py b/matrix/events.py index 31f1f81..fb24c0c 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -387,7 +387,8 @@ class MatrixKeyClaimEvent(MatrixEvent): while server.encryption_queue[self.room_id]: formatted_message = server.encryption_queue[self.room_id].popleft() - server.send_room_message(self.room_id, formatted_message, True) + room, _ = server.find_room_from_id(self.room_id) + server.send_room_message(room, formatted_message, True) class MatrixToDeviceEvent(MatrixEvent): From 9622ccc9d98e4424dffedbd13fad6cf6a2ce53c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 13 Jul 2018 13:21:51 +0200 Subject: [PATCH 052/269] buffer: Add backlog functionality back. --- matrix/buffer.py | 123 ++++++++++++++++++++++++++++++++++++++++++----- matrix/events.py | 79 ------------------------------ matrix/server.py | 13 ++++- 3 files changed, 123 insertions(+), 92 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index a170623..de5c71a 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -228,7 +228,7 @@ class WeechatChannelBuffer(object): @date.setter def date(self, new_date): # type: (int) -> None - new_data = {"date": new_date} + new_data = {"date": str(new_date)} W.hdata_update(self._hdata, self._ptr, new_data) @property @@ -239,7 +239,7 @@ class WeechatChannelBuffer(object): @date_printed.setter def date_printed(self, new_date): # type: (int) -> None - new_data = {"date_printed": new_date} + new_data = {"date_printed": str(new_date)} W.hdata_update(self._hdata, self._ptr, new_data) @property @@ -247,16 +247,32 @@ class WeechatChannelBuffer(object): # type: () -> bool return bool(W.hdata_char(self._hdata, self._ptr, "highlight")) - def update(self, date, date_printed, tags, prefix, message): - new_data = { - "date": date, - "date_printed": date_printed, - "tags_array": ','.join(tags), - "prefix": prefix, - "message": message, - # "highlight": highlight - } - W.hdata_update(self._hdata, self._ptr, new_data) + def update( + self, + date=None, + date_printed=None, + tags=None, + prefix=None, + message=None, + highlight=None + ): + new_data = {} + + if date: + new_data["date"] = str(date) + if date_printed: + new_data["date_printed"] = str(date_printed) + if tags: + new_data["tags_array"] = ','.join(tags) + if prefix: + new_data["prefix"] = prefix + if message: + new_data["message"] = message + if highlight: + new_data["highlight"] = highlight + + if new_data: + W.hdata_update(self._hdata, self._ptr, new_data) def __init__(self, name, server_name, user): # type: (str, str, str) @@ -953,3 +969,86 @@ class RoomBuffer(object): nick = self.find_nick(self.room.own_user_id) date = server_ts_to_weechat(message.timestamp) self.weechat_buffer.self_action(nick, message.message, date) + + def old_redacted(self, event): + tags = [ + SCRIPT_NAME + "_message", + "notify_message", + "no_log", + "no_highlight" + ] + reason = (", reason: \"{reason}\"".format(reason=event.reason) + if event.reason else "") + + censor = self.find_nick(event.censor) + + data = ("{del_color}<{log_color}Message redacted by: " + "{censor}{log_color}{reason}{del_color}>{ncolor}").format( + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + log_color=W.color("logger.color.backlog_line"), + censor=censor, + reason=reason) + + tags += self.get_event_tags(event) + nick = self.find_nick(event.sender) + user = self.weechat_buffer._get_user(nick) + date = server_ts_to_weechat(event.timestamp) + self.weechat_buffer._print_message(user, data, date, tags) + + def old_message(self, event): + tags = [ + SCRIPT_NAME + "_message", + "notify_message", + "no_log", + "no_highlight" + ] + tags += self.get_event_tags(event) + nick = self.find_nick(event.sender) + data = (event.formatted_message.to_weechat() + if event.formatted_message else event.message) + user = self.weechat_buffer._get_user(nick) + date = server_ts_to_weechat(event.timestamp) + self.weechat_buffer._print_message(user, data, date, tags) + + def sort_messages(self): + class LineCopy(object): + def __init__( + self, + date, + date_printed, + tags, + prefix, + message, + highlight + ): + self.date = date + self.date_printed = date_printed + self.tags = tags + self.prefix = prefix + self.message = message + self.highlight = highlight + + @classmethod + def from_line(cls, line): + return cls(line.date, line.date_printed, line.tags, + line.prefix, line.message, line.highlight) + + lines = [ + LineCopy.from_line(line) for line in self.weechat_buffer.lines + ] + sorted_lines = sorted(lines, key=lambda line: line.date, reverse=True) + + for n, line in enumerate(self.weechat_buffer.lines): + new = sorted_lines[n] + line.update(new.date, new.date_printed, new.tags, new.prefix, + new.message) + + def handle_backlog(self, events): + for event in events: + if isinstance(event, RoomMessageText): + self.old_message(event) + elif isinstance(event, RoomRedactedMessageEvent): + self.old_redacted(event) + + self.sort_messages() diff --git a/matrix/events.py b/matrix/events.py index fb24c0c..0a38034 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -427,72 +427,6 @@ class MatrixBacklogEvent(MatrixEvent): return RoomMessageEvent.from_dict(event_dict) - @staticmethod - def buffer_sort_messages(buff): - lines = [] - - own_lines = W.hdata_pointer(W.hdata_get('buffer'), buff, 'own_lines') - - if own_lines: - hdata_line = W.hdata_get('line') - hdata_line_data = W.hdata_get('line_data') - line = W.hdata_pointer( - W.hdata_get('lines'), own_lines, 'first_line') - - while line: - data = W.hdata_pointer(hdata_line, line, 'data') - - line_data = {} - - if data: - date = W.hdata_time(hdata_line_data, data, 'date') - print_date = W.hdata_time(hdata_line_data, data, - 'date_printed') - tags = tags_from_line_data(data) - prefix = W.hdata_string(hdata_line_data, data, 'prefix') - message = W.hdata_string(hdata_line_data, data, 'message') - highlight = W.hdata_char(hdata_line_data, data, "highlight") - - line_data = { - 'date': date, - 'date_printed': print_date, - 'tags_array': ','.join(tags), - 'prefix': prefix, - 'message': message, - 'highlight': highlight - } - - lines.append(line_data) - - line = W.hdata_move(hdata_line, line, 1) - - sorted_lines = sorted(lines, key=itemgetter('date')) - lines = [] - - # We need to convert the dates to a string for hdata_update(), this - # will reverse the list at the same time - while sorted_lines: - line = sorted_lines.pop() - new_line = {k: str(v) for k, v in line.items()} - lines.append(new_line) - - MatrixBacklogEvent.update_buffer_lines(lines, own_lines) - - @staticmethod - def update_buffer_lines(new_lines, own_lines): - hdata_line = W.hdata_get('line') - hdata_line_data = W.hdata_get('line_data') - - line = W.hdata_pointer(W.hdata_get('lines'), own_lines, 'first_line') - - while line: - data = W.hdata_pointer(hdata_line, line, 'data') - - if data: - W.hdata_update(hdata_line_data, data, new_lines.pop()) - - line = W.hdata_move(hdata_line, line, 1) - @classmethod def from_dict(cls, server, room_id, parsed_dict): try: @@ -515,19 +449,6 @@ class MatrixBacklogEvent(MatrixEvent): return MatrixErrorEvent.from_dict(server, "Error fetching backlog", False, parsed_dict) - def execute(self): - room = self.server.rooms[self.room_id] - buff = self.server.buffers[self.room_id] - tags = tags_for_message("backlog") - - for event in self.events: - event.execute(self.server, room, buff, list(tags)) - - room.prev_batch = self.end_token - MatrixBacklogEvent.buffer_sort_messages(buff) - room.backlog_pending = False - W.bar_item_update("buffer_modes") - class MatrixSyncEvent(MatrixEvent): diff --git a/matrix/server.py b/matrix/server.py index e540915..ca51595 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -52,7 +52,10 @@ from matrix.api import ( MatrixKeyClaimMessage ) -from .events import MatrixSendEvent +from .events import ( + MatrixSendEvent, + MatrixBacklogEvent +) from matrix.encryption import ( Olm, @@ -697,6 +700,14 @@ class MatrixServer: if isinstance(response, MatrixSendEvent): _, room_buffer = self.find_room_from_id(response.room_id) self.handle_own_messages(room_buffer, response.message) + + elif isinstance(response, MatrixBacklogEvent): + room, room_buffer = self.find_room_from_id(response.room_id) + room_buffer.handle_backlog(response.events) + room.prev_batch = response.end_token + room.backlog_pending = False + W.bar_item_update("buffer_modes") + else: response.execute() From 2a8cf83f64ea575be13ce11914fce2024c3cc4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 13 Jul 2018 14:26:08 +0200 Subject: [PATCH 053/269] matrix: Remove unused execute() methods. --- matrix/events.py | 10 ---------- matrix/rooms.py | 7 +------ 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/matrix/events.py b/matrix/events.py index 0a38034..90b78a1 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -150,16 +150,6 @@ class MatrixSendEvent(MatrixEvent): self.message = message MatrixEvent.__init__(self, server) - def execute(self): - tags = [ - "matrix_message", "notify_none", "no_highlight", "self_msg", "log1" - ] - - buff = self.server.buffers[self.room_id] - room = self.server.rooms[self.room_id] - - self.message.execute(self.server, room, buff, tags) - @classmethod def from_dict(cls, server, room_id, message, parsed_dict): try: diff --git a/matrix/rooms.py b/matrix/rooms.py index 09e61a4..07f9ef6 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -428,12 +428,7 @@ class RoomMessageSimple(RoomMessageEvent): class RoomMessageUnknown(RoomMessageSimple): - - def execute(self, server, room, buff, tags): - msg = ("Unknown message of type {t}, body: {body}").format( - t=self.message_type, body=self.message) - - self._print_message(msg, room, buff, tags) + pass class RoomMessageText(RoomMessageEvent): From 783f2d59e8fd6d57af2d0db7582178c146877d5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 13 Jul 2018 14:48:53 +0200 Subject: [PATCH 054/269] encryption: Fix for API change in the olm bindings. --- matrix/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index ca51595..e4e6270 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -572,7 +572,7 @@ class MatrixServer: @encrypt_enabled def upload_keys(self, device_keys=False, one_time_keys=False): - keys = self.olm.account.identity_keys() if device_keys else None + keys = self.olm.account.identity_keys if device_keys else None one_time_keys = (self.olm.account.one_time_keys["curve25519"] if one_time_keys else None) From 901b18ac066c109f20ab491a50310e34950d7ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 13 Jul 2018 15:31:45 +0200 Subject: [PATCH 055/269] rooms: Fix BadEvent creation. --- matrix/buffer.py | 3 ++- matrix/rooms.py | 32 ++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index de5c71a..4b4270a 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -954,7 +954,8 @@ class RoomBuffer(object): self._handle_power_level(event) else: - W.prnt("", "Unhandled event of type {}.".format(type(event))) + W.prnt("", "Unhandled event of type {}.".format( + type(event).__name__)) def self_message(self, message): nick = self.find_nick(self.room.own_user_id) diff --git a/matrix/rooms.py b/matrix/rooms.py index 07f9ef6..7bc33e0 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -275,20 +275,22 @@ class RoomInfo(): def _parse_events(olm, room_id, parsed_dict): events = [] - try: - for event in parsed_dict: + for event in parsed_dict: + try: e = RoomInfo.parse_event(olm, room_id, event) + except (ValueError, TypeError, KeyError) as error: + message = ("{prefix}matrix: Error parsing " + "room event of type {type}: " + "{error}\n{event}").format( + prefix=W.prefix("error"), + type=event["type"], + error=pformat(error), + event=pformat(event)) + W.prnt("", message) + e = BadEvent.from_dict(event) + + finally: events.append(e) - except (ValueError, TypeError, KeyError) as error: - message = ("{prefix}matrix: Error parsing " - "room event of type {type}: {error}\n{event}").format( - prefix=W.prefix("error"), - type=event["type"], - error=pformat(error), - event=pformat(event)) - W.prnt("", message) - e = BadEvent.from_dict(event) - events.append(e) return events @@ -318,7 +320,7 @@ class RoomInfo(): ) -class RoomEvent(): +class RoomEvent(object): def __init__(self, event_id, sender, timestamp): self.event_id = event_id @@ -328,14 +330,16 @@ class RoomEvent(): class BadEvent(RoomEvent): def __init__(self, event_id, sender, timestamp, source): + RoomEvent.__init__(self, event_id, sender, timestamp) self.source = source + @classmethod def from_dict(cls, event): event_id = (sanitize_id(event["event_id"]) if "event_id" in event else None) sender = (sanitize_id(event["sender"]) if "sender" in event else None) - timestamp = (sanitize_id(event["origin_server_ts"]) + timestamp = (sanitize_ts(event["origin_server_ts"]) if "origin_server_ts" in event else None) source = json.dumps(event) From 7e1fc69cbfae3432f2814f251ad7306796ca24e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 13 Jul 2018 15:39:18 +0200 Subject: [PATCH 056/269] encryption: Return a tuple even if group decryption failed. --- matrix/encryption.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index d8a332e..6c09e12 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -653,11 +653,9 @@ class Olm(): session = self.inbound_group_sessions[room_id][session_id] try: - plaintext = session.decrypt(ciphertext) + return session.decrypt(ciphertext) except OlmGroupSessionError: - return None - - return plaintext + return None, None def share_group_session(self, room_id, own_id, users): group_session = self.outbound_group_sessions[room_id] From 4323931309af6180f37515601950c1496722650a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 13 Jul 2018 15:46:21 +0200 Subject: [PATCH 057/269] encryption: Return a tuple as well if no decryption session is found. --- matrix/encryption.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/matrix/encryption.py b/matrix/encryption.py index 6c09e12..851baf7 100644 --- a/matrix/encryption.py +++ b/matrix/encryption.py @@ -649,13 +649,16 @@ class Olm(): @encrypt_enabled def group_decrypt(self, room_id, session_id, ciphertext): if session_id not in self.inbound_group_sessions[room_id]: - return None + return None, None session = self.inbound_group_sessions[room_id][session_id] + try: return session.decrypt(ciphertext) except OlmGroupSessionError: - return None, None + pass + + return None, None def share_group_session(self, room_id, own_id, users): group_session = self.outbound_group_sessions[room_id] From dec12a898e2ba6b82b0f4926849d39e232944074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 13 Jul 2018 16:05:18 +0200 Subject: [PATCH 058/269] rooms: Return an error event instead of None upon decryption failure. --- matrix/buffer.py | 15 ++++++++++++++- matrix/rooms.py | 20 +++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 4b4270a..44245c0 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -46,7 +46,8 @@ from .rooms import ( RoomRedactionEvent, RoomRedactedMessageEvent, RoomEncryptionEvent, - RoomPowerLevels + RoomPowerLevels, + UndecryptedEvent ) @@ -953,6 +954,18 @@ class RoomBuffer(object): elif isinstance(event, RoomPowerLevels): self._handle_power_level(event) + elif isinstance(event, UndecryptedEvent): + nick = self.find_nick(event.sender) + date = server_ts_to_weechat(event.timestamp) + data = ("Error decrypting event session " + "id: {}".format(event.session_id)) + self.weechat_buffer.message( + nick, + data, + date, + self.get_event_tags(event) + ) + else: W.prnt("", "Unhandled event of type {}.".format( type(event).__name__)) diff --git a/matrix/rooms.py b/matrix/rooms.py index 7bc33e0..67fc643 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -262,7 +262,7 @@ class RoomInfo(): ) if not plaintext: - return None + return UndecryptedEvent.from_dict(event_dict) parsed_plaintext = json.loads(plaintext, encoding="utf-8") @@ -328,6 +328,24 @@ class RoomEvent(object): self.timestamp = timestamp +class UndecryptedEvent(RoomEvent): + def __init__(self, event_id, sender, timestamp, session_id): + self.session_id = session_id + RoomEvent.__init__(self, event_id, sender, timestamp) + + @classmethod + def from_dict(cls, event): + event_id = (sanitize_id(event["event_id"]) + if "event_id" in event else None) + sender = (sanitize_id(event["sender"]) + if "sender" in event else None) + timestamp = (sanitize_ts(event["origin_server_ts"]) + if "origin_server_ts" in event else None) + session_id = event["content"]["session_id"] + + return cls(event_id, sender, timestamp, session_id) + + class BadEvent(RoomEvent): def __init__(self, event_id, sender, timestamp, source): RoomEvent.__init__(self, event_id, sender, timestamp) From 9a3dae6aab466815257f895674d770c79f1ee013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 13 Jul 2018 16:32:09 +0200 Subject: [PATCH 059/269] rooms: Return true on membership changes and encryption change in handle_events. --- matrix/rooms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matrix/rooms.py b/matrix/rooms.py index 67fc643..e15018c 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -166,7 +166,7 @@ class MatrixRoom: def handle_event(self, event): if isinstance(event, RoomMembershipEvent): - self._handle_membership(event) + return self._handle_membership(event) elif isinstance(event, RoomNameEvent): self.name = event.name @@ -176,6 +176,7 @@ class MatrixRoom: elif isinstance(event, RoomEncryptionEvent): self.encrypted = True + return True elif isinstance(event, RoomPowerLevels): self.power_levels = event.power_levels From 6fc4b0b12cd0c189a9a049a4b49b82567f3763de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 14 Jul 2018 10:51:15 +0200 Subject: [PATCH 060/269] buffer: Expose the error method from the RoomBuffer. --- matrix/buffer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index 44245c0..f882117 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -1066,3 +1066,7 @@ class RoomBuffer(object): self.old_redacted(event) self.sort_messages() + + def error(self, string): + # type: (str) -> None + self.weechat_buffer.error(string) From 3993ce830dddf00d2bf10f86dc6791a3b7f4afde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 14 Jul 2018 10:51:40 +0200 Subject: [PATCH 061/269] rooms: Don't use finally to catch generated events. --- matrix/rooms.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/matrix/rooms.py b/matrix/rooms.py index e15018c..6ad94e5 100644 --- a/matrix/rooms.py +++ b/matrix/rooms.py @@ -290,8 +290,7 @@ class RoomInfo(): W.prnt("", message) e = BadEvent.from_dict(event) - finally: - events.append(e) + events.append(e) return events From 91eec1ad85a000a4af34783e9884ad273337832c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 20 Jul 2018 15:53:47 +0200 Subject: [PATCH 062/269] server: Start the nio migration. --- matrix/events.py | 58 ----------------------- matrix/server.py | 120 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 105 insertions(+), 73 deletions(-) diff --git a/matrix/events.py b/matrix/events.py index 90b78a1..0d4ab31 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -56,15 +56,6 @@ class MatrixErrorEvent(MatrixEvent): self.fatal = fatal MatrixEvent.__init__(self, server) - def execute(self): - message = ("{prefix}matrix: {error}").format( - prefix=W.prefix("error"), error=self.error_message) - - W.prnt(self.server.server_buffer, message) - - if self.fatal: - self.server.disconnect(reconnect=False) - @classmethod def from_dict(cls, server, error_prefix, fatal, parsed_dict): try: @@ -113,25 +104,6 @@ class MatrixLoginEvent(MatrixEvent): self.device_id = device_id MatrixEvent.__init__(self, server) - def execute(self): - self.server.access_token = self.access_token - self.server.user_id = self.user_id - self.server.client.access_token = self.access_token - self.server.device_id = self.device_id - self.server.save_device_id() - - message = "{prefix}matrix: Logged in as {user}".format( - prefix=W.prefix("network"), user=self.user_id) - - W.prnt(self.server.server_buffer, message) - - if not self.server.olm: - self.server.create_olm() - self.server.store_olm() - self.server.upload_keys(device_keys=True, one_time_keys=False) - - self.server.sync() - @classmethod def from_dict(cls, server, parsed_dict): try: @@ -562,33 +534,3 @@ class MatrixSyncEvent(MatrixEvent): except (KeyError, ValueError, TypeError): return MatrixErrorEvent.from_dict(server, "Error syncing", False, parsed_dict) - - def _queue_joined_info(self): - server = self.server - - while self.joined_room_infos: - info = self.joined_room_infos.pop() - - if info.room_id not in server.buffers: - server.create_room_buffer(info.room_id) - - room = server.rooms[info.room_id] - - if not room.prev_batch: - room.prev_batch = info.prev_batch - - server.event_queue.append(info) - - def execute(self): - server = self.server - - # we got the same batch again, nothing to do - if self.next_batch == server.next_batch: - server.sync() - return - - self._queue_joined_info() - server.next_batch = self.next_batch - server.check_one_time_keys(self.one_time_key_count) - - server.handle_events() diff --git a/matrix/server.py b/matrix/server.py index e4e6270..be48fb9 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -23,10 +23,13 @@ import socket import time import datetime import pprint +import json from collections import deque, defaultdict from http_parser.pyparser import HttpParser +from nio import Client, LoginResponse + from matrix.plugin_options import Option, DebugType from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, create_server_buffer, tags_for_message, @@ -53,8 +56,13 @@ from matrix.api import ( ) from .events import ( + MatrixLoginEvent, + MatrixSyncEvent, MatrixSendEvent, - MatrixBacklogEvent + MatrixBacklogEvent, + MatrixErrorEvent, + MatrixEmoteEvent, + MatrixJoinEvent ) from matrix.encryption import ( @@ -108,6 +116,7 @@ class MatrixServer: self.ssl_context = ssl.create_default_context() # type: ssl.SSLContext self.client = None + self.nio_client = Client() # type: Option[Client] self.access_token = None # type: str self.next_batch = None # type: str self.transaction_id = 0 # type: int @@ -250,6 +259,7 @@ class MatrixServer: host = ':'.join([self.address, str(self.port)]) user_agent = 'weechat-matrix/{version}'.format(version="0.1") self.client = MatrixClient(host, user_agent=user_agent) + # self.nio_client = Client() def update_option(self, option, option_name): if option_name == "address": @@ -278,11 +288,13 @@ class MatrixServer: value = W.config_string(option) self.user = value self.access_token = "" + self.nio_client.user = value + self.nio_client.access_token = "" self._load_device_id() - if self.device_id: - self._load_olm() + # if self.device_id: + # self._load_olm() elif option_name == "password": value = W.config_string(option) @@ -696,8 +708,67 @@ class MatrixServer: raise NotImplementedError("Unsupported message of type {}".format( type(message))) + def _handle_erorr_response(self, response): + message = ("{prefix}matrix: {error}").format( + prefix=W.prefix("error"), error=self.error_message) + + W.prnt(self.server.server_buffer, message) + + if self.fatal: + self.server.disconnect(reconnect=False) + + def _handle_login(self, response): + self.access_token = response.access_token + self.user_id = response.user_id + self.client.access_token = response.access_token + self.device_id = response.device_id + self.save_device_id() + + message = "{prefix}matrix: Logged in as {user}".format( + prefix=W.prefix("network"), user=self.user_id) + + W.prnt(self.server_buffer, message) + + # if not self.olm: + # self.create_olm() + # self.store_olm() + # self.upload_keys(device_keys=True, one_time_keys=False) + + self.sync() + + def _queue_joined_info(self, response): + while response.joined_room_infos: + info = response.joined_room_infos.pop() + + if info.room_id not in self.buffers: + self.create_room_buffer(info.room_id) + + room = self.rooms[info.room_id] + + if not room.prev_batch: + room.prev_batch = info.prev_batch + + self.event_queue.append(info) + + def _handle_sync(self, response): + # we got the same batch again, nothing to do + if self.next_batch == response.next_batch: + self.sync() + return + + self._queue_joined_info(response) + self.next_batch = response.next_batch + # self.check_one_time_keys(response.one_time_key_count) + self.handle_events() + def handle_matrix_response(self, response): - if isinstance(response, MatrixSendEvent): + if isinstance(response, MatrixLoginEvent): + self._handle_login(response) + + elif isinstance(response, MatrixSyncEvent): + self._handle_sync(response) + + elif isinstance(response, MatrixSendEvent): _, room_buffer = self.find_room_from_id(response.room_id) self.handle_own_messages(room_buffer, response.message) @@ -708,8 +779,22 @@ class MatrixServer: room.backlog_pending = False W.bar_item_update("buffer_modes") - else: - response.execute() + elif isinstance(response, MatrixErrorEvent): + self._handle_erorr_response(response) + + def nio_receive(self): + response = self.nio_client.next_response() + + if isinstance(response, LoginResponse): + self._handle_login(response) + + def nio_parse_response(self, response): + if isinstance(response, MatrixLoginMessage): + self.nio_client.receive("login", response.response.body) + + self.nio_receive() + + return def handle_response(self, message): # type: (MatrixMessage) -> None @@ -718,17 +803,22 @@ class MatrixServer: if ('content-type' in message.response.headers and message.response.headers['content-type'] == 'application/json'): - ret, error = message.decode_body(self) - if not ret: - message = ("{prefix}matrix: Error decoding json response from " - "server: {error}").format( - prefix=W.prefix("error"), error=error) - W.prnt(self.server_buffer, message) - return + if isinstance(message, MatrixLoginMessage): + self.nio_parse_response(message) - event = message.event - self.handle_matrix_response(event) + else: + ret, error = message.decode_body(self) + + if not ret: + message = ("{prefix}matrix: Error decoding json response" + " from server: {error}").format( + prefix=W.prefix("error"), error=error) + W.prnt(self.server_buffer, message) + return + + event = message.event + self.handle_matrix_response(event) else: status_code = message.response.status if status_code == 504: From 45be743c077dfb573dccb09555dcc1da1303de26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 20 Jul 2018 17:51:48 +0200 Subject: [PATCH 063/269] server: Continue nio migration. --- main.py | 34 --- matrix/bar_items.py | 13 +- matrix/buffer.py | 223 ++++++++------- matrix/commands.py | 66 ++--- matrix/events.py | 156 +---------- matrix/rooms.py | 642 -------------------------------------------- matrix/server.py | 134 +++------ 7 files changed, 199 insertions(+), 1069 deletions(-) delete mode 100644 matrix/rooms.py diff --git a/main.py b/main.py index ac6c95b..ff9db4e 100644 --- a/main.py +++ b/main.py @@ -145,14 +145,6 @@ def print_certificate_info(buff, sock, cert): W.prnt(buff, message) - -@utf8_decode -def matrix_event_timer_cb(server_name, remaining_calls): - server = SERVERS[server_name] - server.handle_events() - return W.WEECHAT_RC_OK - - def wrap_socket(server, file_descriptor): # type: (MatrixServer, int) -> None sock = None # type: socket.socket @@ -399,32 +391,6 @@ def connect_cb(data, status, gnutls_rc, sock, error, ip_address): return W.WEECHAT_RC_OK -@utf8_decode -def room_input_cb(server_name, buffer, input_data): - server = SERVERS[server_name] - - if not server.connected: - message = "{prefix}matrix: you are not connected to the server".format( - prefix=W.prefix("error")) - W.prnt(buffer, message) - return W.WEECHAT_RC_ERROR - - room_id = key_from_value(server.buffers, buffer) - room = server.rooms[room_id] - - formatted_data = Formatted.from_input_line(input_data) - - if room.encrypted: - server.send_room_message(room_id, formatted_data) - return W.WEECHAT_RC_OK - - message = MatrixSendMessage( - server.client, room_id=room_id, formatted_message=formatted_data) - - server.send_or_queue(message) - return W.WEECHAT_RC_OK - - @utf8_decode def room_close_cb(data, buffer): W.prnt("", diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 1f3cfff..5ae77b2 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -45,13 +45,12 @@ def matrix_bar_item_name(data, item, window, buffer, extra_info): color = ("status_name_ssl" if server.ssl_context.check_hostname else "status_name") - room_id = key_from_value(server.buffers, buffer) - - room = server.rooms[room_id] + room_buffer = server.find_room_from_ptr(buffer) + room = room_buffer.room return "{color}{name}".format( color=W.color(color), - name=room.display_name(server.user_id)) + name=room.display_name()) elif buffer == server.server_buffer: color = ("status_name_ssl" @@ -92,14 +91,14 @@ def matrix_bar_item_buffer_modes(data, item, window, buffer, extra_info): # pylint: disable=unused-argument for server in SERVERS.values(): if buffer in server.buffers.values(): - room_id = key_from_value(server.buffers, buffer) - room = server.rooms[room_id] + room_buffer = server.find_room_from_ptr(buffer) + room = room_buffer.room modes = [] if room.encrypted: modes.append("🔐") - if room.backlog_pending: + if room_buffer.backlog_pending: modes.append("⏳") return "".join(modes) diff --git a/matrix/buffer.py b/matrix/buffer.py index f882117..6aa2973 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -20,6 +20,7 @@ from __future__ import unicode_literals import time from builtins import super from functools import partial +from typing import NamedTuple from .globals import W, SERVERS, OPTIONS, SCRIPT_NAME, ENCRYPTION from .utf import utf8_decode @@ -32,29 +33,35 @@ from .utils import ( ) from .plugin_options import RedactType - -from .rooms import ( - RoomNameEvent, - RoomAliasEvent, - RoomMembershipEvent, - RoomTopicEvent, +from nio import ( RoomMessageText, - RoomMessageEmote, - RoomMessageNotice, - RoomMessageMedia, - RoomMessageUnknown, - RoomRedactionEvent, - RoomRedactedMessageEvent, + RoomMemberEvent, + PowerLevelsEvent, RoomEncryptionEvent, - RoomPowerLevels, - UndecryptedEvent + RedactedEvent, + RoomAliasEvent, + RoomTopicEvent, + RoomMessageEmote, + RoomNameEvent ) +OwnMessage = NamedTuple("OwnMessage", [ + ("sender", str), + ("age", int), + ("event_id", str), + ("formatted_message", Formatted) +]) + + +class OwnAction(OwnMessage): + pass + + @utf8_decode def room_buffer_input_cb(server_name, buffer, input_data): server = SERVERS[server_name] - room, room_buffer = server.find_room_from_ptr(buffer) + room_buffer = server.find_room_from_ptr(buffer) if not room_buffer: # TODO log error @@ -66,7 +73,7 @@ def room_buffer_input_cb(server_name, buffer, input_data): formatted_data = Formatted.from_input_line(input_data) - server.send_room_message(room, formatted_data) + server.send_room_message(room_buffer.room, formatted_data) return W.WEECHAT_RC_OK @@ -669,6 +676,7 @@ class WeechatChannelBuffer(object): class RoomBuffer(object): def __init__(self, room, server_name): self.room = room + self.backlog_pending = False # This dict remembers the connection from a user_id to the name we # displayed in the buffer @@ -692,12 +700,14 @@ class RoomBuffer(object): def join(event, date, is_state): user = self.room.users[event.sender] + short_name = shorten_sender(user.user_id) + # TODO make this configurable - if user.name in self.displayed_nicks.values(): + if short_name in self.displayed_nicks.values(): # Use the full user id, but don't include the @ nick = event.sender[1:] else: - nick = user.name + nick = short_name buffer_user = RoomUser(nick, event.sender, user.power_level) self.displayed_nicks[event.sender] = nick @@ -708,11 +718,11 @@ class RoomBuffer(object): self.weechat_buffer.join( buffer_user, - server_ts_to_weechat(event.timestamp), + server_ts_to_weechat(event.server_timestamp), not is_state ) - date = server_ts_to_weechat(event.timestamp) + date = server_ts_to_weechat(event.server_timestamp) if event.content["membership"] == "join": if event.prev_content and "membership" in event.prev_content: @@ -743,7 +753,7 @@ class RoomBuffer(object): self.weechat_buffer.invite(event.state_key, date) return - room_name = self.room.display_name(self.room.own_user_id) + room_name = self.room.display_name() self.weechat_buffer.short_name = room_name def _redact_line(self, event): @@ -808,14 +818,14 @@ class RoomBuffer(object): def _handle_redacted_message(self, event): nick = self.find_nick(event.sender) - date = server_ts_to_weechat(event.timestamp) + date = server_ts_to_weechat(event.server_timestamp) tags = self.get_event_tags(event) tags.append(SCRIPT_NAME + "_redacted") reason = (", reason: \"{reason}\"".format(reason=event.reason) if event.reason else "") - censor = self.find_nick(event.censor) + censor = self.find_nick(event.redacter) data = ("{del_color}<{log_color}Message redacted by: " "{censor}{log_color}{reason}{del_color}>{ncolor}").format( @@ -833,7 +843,7 @@ class RoomBuffer(object): self.weechat_buffer.change_topic( nick, event.topic, - server_ts_to_weechat(event.timestamp), + server_ts_to_weechat(event.server_timestamp), not is_state) @staticmethod @@ -841,12 +851,13 @@ class RoomBuffer(object): return ["matrix_id_{}".format(event.event_id)] def _handle_power_level(self, event): - for user_id in self.room.power_levels: + for user_id in self.room.power_levels.users: if user_id in self.displayed_nicks: nick = self.find_nick(user_id) user = self.weechat_buffer.users[nick] - user.power_level = self.room.power_levels[user_id] + user.power_level = self.room.power_levels.get_user_level( + user_id) # There is no way to change the group of a user without # removing him from the nicklist @@ -854,67 +865,48 @@ class RoomBuffer(object): self.weechat_buffer._add_user_to_nicklist(user) def handle_state_event(self, event): - if isinstance(event, RoomMembershipEvent): + if isinstance(event, RoomMemberEvent): self.handle_membership_events(event, True) elif isinstance(event, RoomTopicEvent): self._handle_topic(event, True) - elif isinstance(event, RoomPowerLevels): + elif isinstance(event, PowerLevelsEvent): self._handle_power_level(event) def handle_timeline_event(self, event): - if isinstance(event, RoomMembershipEvent): + if isinstance(event, RoomMemberEvent): self.handle_membership_events(event, False) elif isinstance(event, (RoomNameEvent, RoomAliasEvent)): - room_name = self.room.display_name(self.room.own_user_id) + room_name = self.room.display_name() self.weechat_buffer.short_name = room_name elif isinstance(event, RoomTopicEvent): self._handle_topic(event, False) - elif isinstance(event, RoomMessageText): - nick = self.find_nick(event.sender) - data = (event.formatted_message.to_weechat() - if event.formatted_message else event.message) - - date = server_ts_to_weechat(event.timestamp) - self.weechat_buffer.message( - nick, - data, - date, - self.get_event_tags(event) - ) - + # Emotes are a subclass of RoomMessageText, so put them before the text + # ones elif isinstance(event, RoomMessageEmote): nick = self.find_nick(event.sender) - date = server_ts_to_weechat(event.timestamp) + date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer.action( nick, - event.message, + event.body, date, self.get_event_tags(event) ) - elif isinstance(event, RoomMessageNotice): + + elif isinstance(event, RoomMessageText): nick = self.find_nick(event.sender) - date = server_ts_to_weechat(event.timestamp) - self.weechat_buffer.notice( - nick, - event.message, - date, - self.get_event_tags(event) - ) + formatted = None - elif isinstance(event, RoomMessageMedia): - nick = self.find_nick(event.sender) - date = server_ts_to_weechat(event.timestamp) - http_url = mxc_to_http(event.url) - url = http_url if http_url else event.url + if event.formatted_body: + formatted = Formatted.from_html(event.formatted_body) - description = ("/{}".format(event.description) - if event.description else "") - data = "{url}{desc}".format(url=url, desc=description) + data = (formatted.to_weechat() + if formatted else event.body) + date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer.message( nick, data, @@ -922,24 +914,51 @@ class RoomBuffer(object): self.get_event_tags(event) ) - elif isinstance(event, RoomMessageUnknown): - nick = self.find_nick(event.sender) - date = server_ts_to_weechat(event.timestamp) - data = ("Unknown message of type {t}, body: {body}").format( - t=event.message_type, - body=event.message - ) - self.weechat_buffer.message( - nick, - data, - date, - self.get_event_tags(event) - ) + # elif isinstance(event, RoomMessageNotice): + # nick = self.find_nick(event.sender) + # date = server_ts_to_weechat(event.server_timestamp) + # self.weechat_buffer.notice( + # nick, + # event.message, + # date, + # self.get_event_tags(event) + # ) - elif isinstance(event, RoomRedactionEvent): - self._redact_line(event) + # elif isinstance(event, RoomMessageMedia): + # nick = self.find_nick(event.sender) + # date = server_ts_to_weechat(event.server_timestamp) + # http_url = mxc_to_http(event.url) + # url = http_url if http_url else event.url - elif isinstance(event, RoomRedactedMessageEvent): + # description = ("/{}".format(event.description) + # if event.description else "") + # data = "{url}{desc}".format(url=url, desc=description) + + # self.weechat_buffer.message( + # nick, + # data, + # date, + # self.get_event_tags(event) + # ) + + # elif isinstance(event, RoomMessageUnknown): + # nick = self.find_nick(event.sender) + # date = server_ts_to_weechat(event.server_timestamp) + # data = ("Unknown message of type {t}, body: {body}").format( + # t=event.message_type, + # body=event.message + # ) + # self.weechat_buffer.message( + # nick, + # data, + # date, + # self.get_event_tags(event) + # ) + + # elif isinstance(event, RoomRedactionEvent): + # self._redact_line(event) + + elif isinstance(event, RedactedEvent): self._handle_redacted_message(event) elif isinstance(event, RoomEncryptionEvent): @@ -951,38 +970,44 @@ class RoomBuffer(object): "this room.") self.weechat_buffer.error(message) - elif isinstance(event, RoomPowerLevels): + elif isinstance(event, PowerLevelsEvent): self._handle_power_level(event) - elif isinstance(event, UndecryptedEvent): - nick = self.find_nick(event.sender) - date = server_ts_to_weechat(event.timestamp) - data = ("Error decrypting event session " - "id: {}".format(event.session_id)) - self.weechat_buffer.message( - nick, - data, - date, - self.get_event_tags(event) - ) + # elif isinstance(event, UndecryptedEvent): + # nick = self.find_nick(event.sender) + # date = server_ts_to_weechat(event.server_timestamp) + # data = ("Error decrypting event session " + # "id: {}".format(event.session_id)) + # self.weechat_buffer.message( + # nick, + # data, + # date, + # self.get_event_tags(event) + # ) else: W.prnt("", "Unhandled event of type {}.".format( type(event).__name__)) def self_message(self, message): + # type: (OwnMessage) -> None nick = self.find_nick(self.room.own_user_id) - data = (message.formatted_message.to_weechat() - if message.formatted_message - else message.message) + data = message.formatted_message.to_weechat() - date = server_ts_to_weechat(message.timestamp) + # TODO event_id tag is missing + date = message.age self.weechat_buffer.self_message(nick, data, date) def self_action(self, message): + # type: (OwnMessage) -> None nick = self.find_nick(self.room.own_user_id) - date = server_ts_to_weechat(message.timestamp) - self.weechat_buffer.self_action(nick, message.message, date) + date = message.age + # TODO event_id tag is missing + self.weechat_buffer.self_action( + nick, + message.formatted_message.to_weechat(), + date + ) def old_redacted(self, event): tags = [ @@ -994,7 +1019,7 @@ class RoomBuffer(object): reason = (", reason: \"{reason}\"".format(reason=event.reason) if event.reason else "") - censor = self.find_nick(event.censor) + censor = self.find_nick(event.redacter) data = ("{del_color}<{log_color}Message redacted by: " "{censor}{log_color}{reason}{del_color}>{ncolor}").format( @@ -1007,7 +1032,7 @@ class RoomBuffer(object): tags += self.get_event_tags(event) nick = self.find_nick(event.sender) user = self.weechat_buffer._get_user(nick) - date = server_ts_to_weechat(event.timestamp) + date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer._print_message(user, data, date, tags) def old_message(self, event): @@ -1022,7 +1047,7 @@ class RoomBuffer(object): data = (event.formatted_message.to_weechat() if event.formatted_message else event.message) user = self.weechat_buffer._get_user(nick) - date = server_ts_to_weechat(event.timestamp) + date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer._print_message(user, data, date, tags) def sort_messages(self): @@ -1062,7 +1087,7 @@ class RoomBuffer(object): for event in events: if isinstance(event, RoomMessageText): self.old_message(event) - elif isinstance(event, RoomRedactedMessageEvent): + elif isinstance(event, RedactedEvent): self.old_redacted(event) self.sort_messages() diff --git a/matrix/commands.py b/matrix/commands.py index 0046526..eb4baff 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -67,23 +67,23 @@ def hook_commands(): 'matrix_command_cb', '') - W.hook_command( - # Command name and short description - 'redact', - 'redact messages', - # Synopsis - ('[:""] []'), - # Description - ("message-number: number of message to redact (starting from 1 for\n" - " the last message received, counting up)\n" - " message-part: an initial part of the message (ignored, only used\n" - " as visual feedback when using completion)\n" - " reason: the redaction reason\n"), - # Completions - ('%(matrix_messages)'), - # Function name - 'matrix_redact_command_cb', - '') + # W.hook_command( + # # Command name and short description + # 'redact', + # 'redact messages', + # # Synopsis + # ('[:""] []'), + # # Description + # ("message-number: number of message to redact (starting from 1 for\n" + # " the last message received, counting up)\n" + # " message-part: an initial part of the message (ignored, only used\n" + # " as visual feedback when using completion)\n" + # " reason: the redaction reason\n"), + # # Completions + # ('%(matrix_messages)'), + # # Function name + # 'matrix_redact_command_cb', + # '') W.hook_command( # Command name and short description @@ -101,15 +101,15 @@ def hook_commands(): matrix_hook_olm_command() - W.hook_command_run('/topic', 'matrix_command_topic_cb', '') + # W.hook_command_run('/topic', 'matrix_command_topic_cb', '') W.hook_command_run('/buffer clear', 'matrix_command_buf_clear_cb', '') W.hook_command_run('/join', 'matrix_command_join_cb', '') W.hook_command_run('/part', 'matrix_command_part_cb', '') W.hook_command_run('/invite', 'matrix_command_invite_cb', '') W.hook_command_run('/kick', 'matrix_command_kick_cb', '') - if OPTIONS.enable_backlog: - hook_page_up() + # if OPTIONS.enable_backlog: + # hook_page_up() @utf8_decode @@ -123,7 +123,7 @@ def matrix_me_command_cb(data, buffer, args): W.prnt(server.server_buffer, message) return W.WEECHAT_RC_ERROR - room_id = key_from_value(server.buffers, buffer) + room_buffer = server.find_room_from_ptr(buffer) if not args: return W.WEECHAT_RC_OK @@ -131,10 +131,10 @@ def matrix_me_command_cb(data, buffer, args): formatted_data = Formatted.from_input_line(args) message = MatrixEmoteMessage( server.client, - room_id=room_id, + room_id=room_buffer.room.room_id, formatted_message=formatted_data) - if server.rooms[room_id].encrypted: + if room_buffer.room.encrypted: return W.WEECHAT_RC_OK server.send_or_queue(message) @@ -144,15 +144,16 @@ def matrix_me_command_cb(data, buffer, args): elif buffer == server.server_buffer: message = ("{prefix}matrix: command \"me\" must be " "executed on a Matrix channel buffer" - ).format(prefix=W.prefix("error")) + ).format(prefix=W.prefix("error")) W.prnt("", message) return W.WEECHAT_RC_OK def matrix_fetch_old_messages(server, room_id): - room = server.rooms[room_id] + room_buffer = server.find_room_from_id(room_id) + room = room_buffer.room - if room.backlog_pending: + if room_buffer.backlog_pending: return prev_batch = room.prev_batch @@ -165,7 +166,7 @@ def matrix_fetch_old_messages(server, room_id): room_id=room_id, token=prev_batch, limit=OPTIONS.backlog_limit) - room.backlog_pending = True + room_buffer.backlog_pending = True W.bar_item_update("buffer_modes") server.send_or_queue(message) @@ -913,14 +914,15 @@ def matrix_command_topic_cb(data, buffer, command): for server in SERVERS.values(): if buffer in server.buffers.values(): topic = None - room_id = key_from_value(server.buffers, buffer) + room_buffer = server.find_room_from_ptr(buffer) split_command = command.split(' ', 1) if len(split_command) == 2: topic = split_command[1] if not topic: - room = server.rooms[room_id] + room_buffer = server.find_room_from_ptr(buffer) + room = room_buffer.room if not room.topic: return W.WEECHAT_RC_OK @@ -938,8 +940,8 @@ def matrix_command_topic_cb(data, buffer, command): topic=room.topic) date = int(time.time()) - topic_date = room.topic_date.strftime("%a, %d %b %Y " - "%H:%M:%S") + topic_date = room_buffer.weechat_buffer.topic_date.strftime( + "%a, %d %b %Y %H:%M:%S") tags = "matrix_topic,log1" W.prnt_date_tags(buffer, date, tags, message) @@ -958,7 +960,7 @@ def matrix_command_topic_cb(data, buffer, command): return W.WEECHAT_RC_OK_EAT message = MatrixTopicMessage( - server.client, room_id=room_id, topic=topic) + server.client, room_id=room.room_id, topic=topic) server.send_or_queue(message) return W.WEECHAT_RC_OK_EAT diff --git a/matrix/events.py b/matrix/events.py index 0d4ab31..3221516 100644 --- a/matrix/events.py +++ b/matrix/events.py @@ -27,12 +27,9 @@ from operator import itemgetter from matrix.globals import W from matrix.utils import (tags_for_message, sanitize_id, sanitize_token, sanitize_text, tags_from_line_data) -from matrix.rooms import (RoomInfo, RoomMessageText, - RoomMessageEvent, RoomRedactedMessageEvent, - RoomMessageEmote) from matrix.encryption import OlmDeviceKey, OneTimeKey -from .buffer import RoomUser +from .buffer import RoomUser, OwnMessage, OwnAction try: from olm.session import OlmMessage, OlmPreKeyMessage @@ -96,25 +93,6 @@ class MatrixKeyUploadEvent(MatrixEvent): "keys", False, parsed_dict) -class MatrixLoginEvent(MatrixEvent): - - def __init__(self, server, user_id, device_id, access_token): - self.user_id = user_id - self.access_token = access_token - self.device_id = device_id - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, parsed_dict): - try: - return cls(server, sanitize_id(parsed_dict["user_id"]), - sanitize_id(parsed_dict["device_id"]), - sanitize_token(parsed_dict["access_token"])) - except (KeyError, TypeError, ValueError): - return MatrixErrorEvent.from_dict(server, "Error logging in", True, - parsed_dict) - - class MatrixSendEvent(MatrixEvent): def __init__(self, server, room_id, message): @@ -128,11 +106,9 @@ class MatrixSendEvent(MatrixEvent): event_id = sanitize_id(parsed_dict["event_id"]) sender = server.user_id age = 0 - plain_message = message.to_plain() formatted_message = message - message = RoomMessageText(event_id, sender, age, plain_message, - formatted_message) + message = OwnMessage(sender, age, event_id, formatted_message) return cls(server, room_id, message) except (KeyError, TypeError, ValueError): @@ -148,11 +124,9 @@ class MatrixEmoteEvent(MatrixSendEvent): event_id = sanitize_id(parsed_dict["event_id"]) sender = server.user_id age = 0 - plain_message = message.to_plain() formatted_message = message - message = RoomMessageEmote(event_id, sender, age, plain_message, - formatted_message) + message = OwnAction(sender, age, event_id, formatted_message) return cls(server, room_id, message) except (KeyError, TypeError, ValueError): @@ -410,127 +384,3 @@ class MatrixBacklogEvent(MatrixEvent): except (KeyError, ValueError, TypeError): return MatrixErrorEvent.from_dict(server, "Error fetching backlog", False, parsed_dict) - - -class MatrixSyncEvent(MatrixEvent): - - def __init__(self, server, next_batch, room_infos, invited_infos, - one_time_key_count): - self.next_batch = next_batch - self.joined_room_infos = room_infos - self.invited_room_infos = invited_infos - self.one_time_key_count = one_time_key_count - - MatrixEvent.__init__(self, server) - - @staticmethod - def _infos_from_dict(olm, parsed_dict): - join_infos = [] - invite_infos = [] - - for room_id, room_dict in parsed_dict['join'].items(): - if not room_id: - continue - - join_infos.append(RoomInfo.from_dict(olm, room_id, room_dict)) - - return (join_infos, invite_infos) - - @staticmethod - def _get_olm_device_event(server, parsed_dict): - device_key = server.olm.account.identity_keys["curve25519"] - - if device_key not in parsed_dict["content"]["ciphertext"]: - return None - - sender = sanitize_id(parsed_dict["sender"]) - sender_key = sanitize_id(parsed_dict["content"]["sender_key"]) - - ciphertext = parsed_dict["content"]["ciphertext"].pop(device_key) - - message = None - - if ciphertext["type"] == 0: - message = OlmPreKeyMessage(ciphertext["body"]) - elif ciphertext["type"] == 1: - message = OlmMessage(ciphertext["body"]) - else: - raise ValueError("Invalid Olm message type") - - olm = server.olm - plaintext = olm.decrypt(sender, sender_key, message) - - if not plaintext: - return None - - # TODO check sender key - decrypted_sender = plaintext["sender"] - decrypted_recepient = plaintext["recipient"] - decrypted_recepient_key = plaintext["recipient_keys"]["ed25519"] - - if (sender != decrypted_sender or - server.user_id != decrypted_recepient or - olm.account.identity_keys["ed25519"] != - decrypted_recepient_key): - error_message = ("{prefix}matrix: Mismatch in decrypted Olm " - "message").format(prefix=W.prefix("error")) - W.prnt("", error_message) - return None - - if plaintext["type"] != "m.room_key": - return None - - MatrixSyncEvent._handle_key_event(server, sender_key, plaintext) - - @staticmethod - def _handle_key_event(server, sender_key, parsed_dict): - # type: (MatrixServer, str, Dict[Any, Any] -> None - olm = server.olm - content = parsed_dict.pop("content") - - if content["algorithm"] != "m.megolm.v1.aes-sha2": - return - - room_id = content["room_id"] - session_id = content["session_id"] - session_key = content["session_key"] - - if session_id in olm.inbound_group_sessions[room_id]: - return - - olm.create_group_session(room_id, session_id, session_key) - - @staticmethod - def _get_to_device_events(server, parsed_dict): - # type: (MatrixServer, Dict[Any, Any]) -> None - for event in parsed_dict["events"]: - if event["type"] == "m.room.encrypted": - if (event["content"]["algorithm"] == - 'm.olm.v1.curve25519-aes-sha2'): - MatrixSyncEvent._get_olm_device_event(server, event) - - @classmethod - def from_dict(cls, server, parsed_dict): - try: - next_batch = sanitize_id(parsed_dict["next_batch"]) - one_time_key_count = 0 - - if "device_one_time_keys_count" in parsed_dict: - if ("signed_curve25519" in - parsed_dict["device_one_time_keys_count"]): - one_time_key_count = ( - parsed_dict["device_one_time_keys_count"]["signed_curve25519"]) - - MatrixSyncEvent._get_to_device_events( - server, parsed_dict.pop("to_device")) - - room_info_dict = parsed_dict["rooms"] - - join_infos, invite_infos = MatrixSyncEvent._infos_from_dict( - server.olm, room_info_dict) - - return cls(server, next_batch, join_infos, invite_infos, - one_time_key_count) - except (KeyError, ValueError, TypeError): - return MatrixErrorEvent.from_dict(server, "Error syncing", False, - parsed_dict) diff --git a/matrix/rooms.py b/matrix/rooms.py deleted file mode 100644 index 6ad94e5..0000000 --- a/matrix/rooms.py +++ /dev/null @@ -1,642 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2018 Damir Jelić -# -# Permission to use, copy, modify, and/or distribute this software for -# any purpose with or without fee is hereby granted, provided that the -# above copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -from __future__ import unicode_literals - -import json - -from pprint import pformat - -from collections import namedtuple, deque - -from matrix.globals import W - -from matrix.colors import Formatted -from matrix.utils import ( - strip_matrix_server, color_for_tags, server_ts_to_weechat, - sender_to_nick_and_color, add_event_tags, sanitize_id, sanitize_ts, - sanitize_string, sanitize_text, shorten_sender, add_user_to_nicklist, - get_prefix_for_level, sanitize_power_level, string_strikethrough, - line_pointer_and_tags_from_event, sender_to_prefix_and_color) - -PowerLevel = namedtuple('PowerLevel', ['user', 'level']) - - -class MatrixRoom: - - def __init__(self, room_id, own_user_id): - # type: (str) -> None - # yapf: disable - self.room_id = room_id # type: str - self.own_user_id = own_user_id - self.canonical_alias = None # type: str - self.name = None # type: str - self.prev_batch = "" # type: str - self.users = dict() # type: Dict[str, MatrixUser] - self.encrypted = False # type: bool - self.backlog_pending = False # type: bool - self.power_levels = {} - # yapf: enable - - def display_name(self, own_user_id): - """ - Calculate display name for a room. - - Prefer returning the room name if it exists, falling back to - a group-style name if not. - - Mostly follows: - https://matrix.org/docs/spec/client_server/r0.3.0.html#id268 - - An exception is that we prepend '#' before the room name to make it - visually distinct from private messages and unnamed groups of users - ("direct chats") in weechat's buffer list. - """ - if self.is_named(): - return self.named_room_name() - else: - return self.group_name(own_user_id) - - def named_room_name(self): - """ - Returns the name of the room, if it's a named room. Otherwise return - None. - """ - if self.name: - return "#" + self.name - elif self.canonical_alias: - return self.canonical_alias - else: - return None - - def group_name(self, own_user_id): - """ - Returns the group-style name of the room, i.e. a name based on the room - members. - """ - # Sort user display names, excluding our own user and using the - # mxid as the sorting key. - # - # TODO: Hook the user display name disambiguation algorithm here. - # Currently, we use the user display names as is, which may not be - # unique. - users = [user.name for mxid, user - in sorted(self.users.items(), key=lambda t: t[0]) - if mxid != own_user_id] - - num_users = len(users) - - if num_users == 1: - return users[0] - elif num_users == 2: - return " and ".join(users) - elif num_users >= 3: - return "{first_user} and {num} others".format( - first_user=users[0], - num=num_users-1) - else: - return "Empty room?" - - def machine_name(self): - """ - Calculate an unambiguous, unique machine name for a room. - - Either use the more human-friendly canonical alias, if it exists, or - the internal room ID if not. - """ - if self.canonical_alias: - return self.canonical_alias - else: - return self.room_id - - def is_named(self): - """ - Is this a named room? - - A named room is a room with either the name or a canonical alias set. - """ - return self.canonical_alias or self.name - - def is_group(self): - """ - Is this an ad hoc group of users? - - A group is an unnamed room with no canonical alias. - """ - return not self.is_named() - - def _handle_membership(self, event): - if event.content["membership"] == "join": - if event.sender in self.users: - user = self.users[event.sender] - if "display_name" in event.content: - user.display_name = event.content["display_name"] - else: - short_name = shorten_sender(event.sender) - # TODO the default power level doesn't have to be 0 - level = (self.power_levels[event.sender] if event.sender in - self.power_levels else 0) - display_name = (event.content["display_name"] - if "display_name" in event.content else None) - - user = MatrixUser(short_name, display_name, level) - self.users[event.sender] = user - return True - - elif event.content["membership"] == "leave": - if event.state_key in self.users: - del self.users[event.state_key] - return True - - elif event.content["membership"] == "invite": - pass - - def handle_event(self, event): - if isinstance(event, RoomMembershipEvent): - return self._handle_membership(event) - - elif isinstance(event, RoomNameEvent): - self.name = event.name - - elif isinstance(event, RoomAliasEvent): - self.canonical_alias = event.canonical_alias - - elif isinstance(event, RoomEncryptionEvent): - self.encrypted = True - return True - - elif isinstance(event, RoomPowerLevels): - self.power_levels = event.power_levels - - # Update the power levels of the joined users - for user_id, level in self.power_levels.items(): - if user_id in self.users: - self.users[user_id].power_level = level - - return False - - -class MatrixUser: - - def __init__(self, name, display_name=None, power_level=0): - # yapf: disable - self.name = name # type: str - self.display_name = display_name # type: str - self.power_level = power_level # type: int - # yapf: enable - - -class RoomInfo(): - - def __init__(self, room_id, prev_batch, state, timeline): - # type: (str, str, List[Any], List[Any]) -> None - self.room_id = room_id - self.prev_batch = prev_batch - - self.state = deque(state) - self.timeline = deque(timeline) - - @staticmethod - def _message_from_event(event): - # The transaction id will only be present for events that are send out - # from this client, since we print out our own messages as soon as we - # get a receive confirmation from the server we don't care about our - # own messages in a sync event. More info under: - # https://github.com/matrix-org/matrix-doc/blob/master/api/client-server/definitions/event.yaml#L53 - if "transaction_id" in event["unsigned"]: - return None - - if "redacted_by" in event["unsigned"]: - return RoomRedactedMessageEvent.from_dict(event) - - return RoomMessageEvent.from_dict(event) - - @staticmethod - def parse_event(olm, room_id, event_dict): - # type: (Dict[Any, Any]) -> (RoomEvent, RoomEvent) - event = None - - if "redacted_by" in event_dict["unsigned"]: - event = RoomRedactedMessageEvent.from_dict(event_dict) - elif event_dict["type"] == "m.room.message": - event = RoomInfo._message_from_event(event_dict) - elif event_dict["type"] == "m.room.member": - event = RoomMembershipEvent.from_dict(event_dict) - elif event_dict["type"] == "m.room.power_levels": - event = RoomPowerLevels.from_dict(event_dict) - elif event_dict["type"] == "m.room.topic": - event = RoomTopicEvent.from_dict(event_dict) - elif event_dict["type"] == "m.room.redaction": - event = RoomRedactionEvent.from_dict(event_dict) - elif event_dict["type"] == "m.room.name": - event = RoomNameEvent.from_dict(event_dict) - elif event_dict["type"] == "m.room.canonical_alias": - event = RoomAliasEvent.from_dict(event_dict) - elif event_dict["type"] == "m.room.encryption": - event = RoomEncryptionEvent.from_dict(event_dict) - elif event_dict["type"] == "m.room.encrypted": - event = RoomInfo._decrypt_event(olm, room_id, event_dict) - - return event - - @staticmethod - def _decrypt_event(olm, room_id, event_dict): - session_id = event_dict["content"]["session_id"] - ciphertext = event_dict["content"]["ciphertext"] - plaintext, message_index = olm.group_decrypt( - room_id, - session_id, - ciphertext - ) - - if not plaintext: - return UndecryptedEvent.from_dict(event_dict) - - parsed_plaintext = json.loads(plaintext, encoding="utf-8") - - event_dict["content"] = parsed_plaintext["content"] - event_dict["type"] = parsed_plaintext["type"] - - return RoomInfo.parse_event(olm, room_id, event_dict) - - @staticmethod - def _parse_events(olm, room_id, parsed_dict): - events = [] - - for event in parsed_dict: - try: - e = RoomInfo.parse_event(olm, room_id, event) - except (ValueError, TypeError, KeyError) as error: - message = ("{prefix}matrix: Error parsing " - "room event of type {type}: " - "{error}\n{event}").format( - prefix=W.prefix("error"), - type=event["type"], - error=pformat(error), - event=pformat(event)) - W.prnt("", message) - e = BadEvent.from_dict(event) - - events.append(e) - - return events - - @classmethod - def from_dict(cls, olm, room_id, parsed_dict): - prev_batch = sanitize_id(parsed_dict['timeline']['prev_batch']) - - state_dict = parsed_dict['state']['events'] - timeline_dict = parsed_dict['timeline']['events'] - - state_events = RoomInfo._parse_events( - olm, - room_id, - state_dict - ) - timeline_events = RoomInfo._parse_events( - olm, - room_id, - timeline_dict - ) - - return cls( - room_id, - prev_batch, - list(filter(None, state_events)), - list(filter(None, timeline_events)) - ) - - -class RoomEvent(object): - - def __init__(self, event_id, sender, timestamp): - self.event_id = event_id - self.sender = sender - self.timestamp = timestamp - - -class UndecryptedEvent(RoomEvent): - def __init__(self, event_id, sender, timestamp, session_id): - self.session_id = session_id - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event): - event_id = (sanitize_id(event["event_id"]) - if "event_id" in event else None) - sender = (sanitize_id(event["sender"]) - if "sender" in event else None) - timestamp = (sanitize_ts(event["origin_server_ts"]) - if "origin_server_ts" in event else None) - session_id = event["content"]["session_id"] - - return cls(event_id, sender, timestamp, session_id) - - -class BadEvent(RoomEvent): - def __init__(self, event_id, sender, timestamp, source): - RoomEvent.__init__(self, event_id, sender, timestamp) - self.source = source - - @classmethod - def from_dict(cls, event): - event_id = (sanitize_id(event["event_id"]) - if "event_id" in event else None) - sender = (sanitize_id(event["sender"]) - if "sender" in event else None) - timestamp = (sanitize_ts(event["origin_server_ts"]) - if "origin_server_ts" in event else None) - source = json.dumps(event) - - return cls(event_id, sender, timestamp, source) - - -class RoomRedactedMessageEvent(RoomEvent): - - def __init__(self, event_id, sender, timestamp, censor, reason=None): - self.censor = censor - self.reason = reason - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event): - event_id = sanitize_id(event["event_id"]) - sender = sanitize_id(event["sender"]) - timestamp = sanitize_ts(event["origin_server_ts"]) - - censor = sanitize_id(event['unsigned']['redacted_because']['sender']) - reason = None - - if 'reason' in event['unsigned']['redacted_because']['content']: - reason = sanitize_text( - event['unsigned']['redacted_because']['content']['reason']) - - return cls(event_id, sender, timestamp, censor, reason) - -class RoomMessageEvent(RoomEvent): - - @classmethod - def from_dict(cls, event): - if event['content']['msgtype'] == 'm.text': - return RoomMessageText.from_dict(event) - elif event['content']['msgtype'] == 'm.image': - return RoomMessageMedia.from_dict(event) - elif event['content']['msgtype'] == 'm.audio': - return RoomMessageMedia.from_dict(event) - elif event['content']['msgtype'] == 'm.file': - return RoomMessageMedia.from_dict(event) - elif event['content']['msgtype'] == 'm.video': - return RoomMessageMedia.from_dict(event) - elif event['content']['msgtype'] == 'm.emote': - return RoomMessageEmote.from_dict(event) - elif event['content']['msgtype'] == 'm.notice': - return RoomMessageNotice.from_dict(event) - return RoomMessageUnknown.from_dict(event) - - def _print_message(self, message, room, buff, tags): - nick, color_name = sender_to_nick_and_color(room, self.sender) - color = color_for_tags(color_name) - - event_tags = add_event_tags(self.event_id, nick, color, tags) - - tags_string = ",".join(event_tags) - - prefix, prefix_color = sender_to_prefix_and_color(room, self.sender) - - prefix_string = ("" if not prefix else "{}{}{}".format( - W.color(prefix_color), prefix, W.color("reset"))) - - data = "{prefix}{color}{author}{ncolor}\t{msg}".format( - prefix=prefix_string, - color=W.color(color_name), - author=nick, - ncolor=W.color("reset"), - msg=message) - - date = server_ts_to_weechat(self.timestamp) - W.prnt_date_tags(buff, date, tags_string, data) - - -class RoomMessageSimple(RoomMessageEvent): - - def __init__(self, event_id, sender, timestamp, message, message_type): - self.message = message - self.message_type = message_type - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event): - event_id = sanitize_id(event["event_id"]) - sender = sanitize_id(event["sender"]) - timestamp = sanitize_ts(event["origin_server_ts"]) - - message = sanitize_text(event["content"]["body"]) - message_type = sanitize_text(event["content"]["msgtype"]) - - return cls(event_id, sender, timestamp, message, message_type) - - -class RoomMessageUnknown(RoomMessageSimple): - pass - - -class RoomMessageText(RoomMessageEvent): - - def __init__(self, event_id, sender, timestamp, message, formatted_message=None): - self.message = message - self.formatted_message = formatted_message - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event): - event_id = sanitize_id(event["event_id"]) - sender = sanitize_id(event["sender"]) - timestamp = sanitize_ts(event["origin_server_ts"]) - - msg = "" - formatted_msg = None - - msg = sanitize_text(event['content']['body']) - - if ('format' in event['content'] and - 'formatted_body' in event['content']): - if event['content']['format'] == "org.matrix.custom.html": - formatted_msg = Formatted.from_html( - sanitize_text(event['content']['formatted_body'])) - - return cls(event_id, sender, timestamp, msg, formatted_msg) - - -class RoomMessageEmote(RoomMessageSimple): - pass - - -class RoomMessageNotice(RoomMessageText): - pass - - -class RoomMessageMedia(RoomMessageEvent): - - def __init__(self, event_id, sender, timestamp, url, description): - self.url = url - self.description = description - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event): - event_id = sanitize_id(event["event_id"]) - sender = sanitize_id(event["sender"]) - timestamp = sanitize_ts(event["origin_server_ts"]) - - mxc_url = sanitize_text(event['content']['url']) - description = sanitize_text(event["content"]["body"]) - - return cls(event_id, sender, timestamp, mxc_url, description) - - -class RoomMembershipEvent(RoomEvent): - def __init__( - self, - event_id, - sender, - timestamp, - state_key, - content, - prev_content - ): - self.state_key = state_key - self.content = content - self.prev_content = prev_content - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event_dict): - event_id = sanitize_id(event_dict["event_id"]) - sender = sanitize_id(event_dict["sender"]) - timestamp = sanitize_ts(event_dict["origin_server_ts"]) - state_key = sanitize_id(event_dict["state_key"]) - content = event_dict["content"] - prev_content = (event_dict["unsigned"]["prev_content"] - if "prev_content" in event_dict["unsigned"] else None) - - return cls( - event_id, - sender, - timestamp, - state_key, - content, - prev_content - ) - - -class RoomPowerLevels(RoomEvent): - - def __init__(self, event_id, sender, timestamp, power_levels): - self.power_levels = power_levels - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event_dict): - event_id = sanitize_id(event_dict["event_id"]) - sender = sanitize_id(event_dict["sender"]) - timestamp = sanitize_ts(event_dict["origin_server_ts"]) - - power_levels = event_dict["content"].pop("users") - - return cls(event_id, sender, timestamp, power_levels) - - -class RoomTopicEvent(RoomEvent): - - def __init__(self, event_id, sender, timestamp, topic): - self.topic = topic - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event_dict): - event_id = sanitize_id(event_dict["event_id"]) - sender = sanitize_id(event_dict["sender"]) - timestamp = sanitize_ts(event_dict["origin_server_ts"]) - - topic = sanitize_text(event_dict["content"]["topic"]) - - return cls(event_id, sender, timestamp, topic) - - -class RoomRedactionEvent(RoomEvent): - - def __init__(self, event_id, sender, timestamp, redaction_id, reason=None): - self.redaction_id = redaction_id - self.reason = reason - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event_dict): - event_id = sanitize_id(event_dict["event_id"]) - sender = sanitize_id(event_dict["sender"]) - timestamp = sanitize_ts(event_dict["origin_server_ts"]) - - redaction_id = sanitize_id(event_dict["redacts"]) - - reason = (sanitize_text(event_dict["content"]["reason"]) - if "reason" in event_dict["content"] else None) - - return cls(event_id, sender, timestamp, redaction_id, reason) - - -class RoomNameEvent(RoomEvent): - - def __init__(self, event_id, sender, timestamp, name): - self.name = name - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event_dict): - event_id = sanitize_id(event_dict["event_id"]) - sender = sanitize_id(event_dict["sender"]) - timestamp = sanitize_ts(event_dict["origin_server_ts"]) - - name = sanitize_string(event_dict['content']['name']) - - return cls(event_id, sender, timestamp, name) - - -class RoomAliasEvent(RoomEvent): - - def __init__(self, event_id, sender, timestamp, canonical_alias): - self.canonical_alias = canonical_alias - RoomEvent.__init__(self, event_id, sender, timestamp) - - @classmethod - def from_dict(cls, event_dict): - event_id = sanitize_id(event_dict["event_id"]) - sender = sanitize_id(event_dict["sender"]) - timestamp = sanitize_ts(event_dict["origin_server_ts"]) - - canonical_alias = sanitize_id(event_dict["content"]["alias"]) - - return cls(event_id, sender, timestamp, canonical_alias) - - -class RoomEncryptionEvent(RoomEvent): - - @classmethod - def from_dict(cls, event_dict): - event_id = sanitize_id(event_dict["event_id"]) - sender = sanitize_id(event_dict["sender"]) - timestamp = sanitize_ts(event_dict["origin_server_ts"]) - - return cls(event_id, sender, timestamp) diff --git a/matrix/server.py b/matrix/server.py index be48fb9..eb8ca05 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -28,7 +28,7 @@ import json from collections import deque, defaultdict from http_parser.pyparser import HttpParser -from nio import Client, LoginResponse +from nio import Client, LoginResponse, SyncRepsponse from matrix.plugin_options import Option, DebugType from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, @@ -37,12 +37,7 @@ from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, from matrix.utf import utf8_decode from matrix.globals import W, SERVERS, OPTIONS import matrix.api as API -from .buffer import RoomBuffer -from .rooms import ( - MatrixRoom, - RoomMessageText, - RoomMessageEmote, -) +from .buffer import RoomBuffer, OwnMessage, OwnAction from matrix.api import ( MatrixClient, MatrixSyncMessage, @@ -56,8 +51,6 @@ from matrix.api import ( ) from .events import ( - MatrixLoginEvent, - MatrixSyncEvent, MatrixSendEvent, MatrixBacklogEvent, MatrixErrorEvent, @@ -97,7 +90,6 @@ class MatrixServer: self.user = "" # type: str self.password = "" # type: str - self.rooms = dict() # type: Dict[str, MatrixRoom] self.room_buffers = dict() # type: Dict[str, WeechatChannelBuffer] self.buffers = dict() # type: Dict[str, weechat.buffer] self.server_buffer = None # type: weechat.buffer @@ -610,10 +602,10 @@ class MatrixServer: def query_keys(self): users = [] - for room in self.rooms.values(): - if not room.encrypted: + for room_buffer in self.room_buffers.values(): + if not room_buffer.room.encrypted: continue - users += list(room.users) + users += list(room_buffer.room.users) if not users: return @@ -642,68 +634,13 @@ class MatrixServer: server_buffer_prnt(self, pprint.pformat(message.request.payload)) server_buffer_prnt(self, pprint.pformat(message.response.body)) - def _loop_events(self, info, n): - - for i in range(n+1): - is_state = False - try: - event = info.state.popleft() - is_state = True - except IndexError: - try: - event = info.timeline.popleft() - except IndexError: - return i - - room, room_buffer = self.find_room_from_id(info.room_id) - # The room changed it's members, if the room is encrypted update - # the device list - if room.handle_event(event) and room.encrypted: - self.device_check_timestamp = None - - if is_state: - room_buffer.handle_state_event(event) - else: - room_buffer.handle_timeline_event(event) - - self.event_queue.appendleft(info) - return i - - def handle_events(self): - n = 25 - - while True: - try: - info = self.event_queue.popleft() - except IndexError: - if self.event_queue_timer: - W.unhook(self.event_queue_timer) - self.event_queue_timer = None - - self.sync() - return - - ret = self._loop_events(info, n) - - if ret < n: - n = n - ret - else: - self.event_queue.appendleft(info) - - if not self.event_queue_timer: - hook = W.hook_timer(1 * 100, 0, 0, "matrix_event_timer_cb", - self.name) - self.event_queue_timer = hook - - return - def handle_own_messages(self, room_buffer, message): - if isinstance(message, RoomMessageText): - room_buffer.self_message(message) - return - elif isinstance(message, RoomMessageEmote): + if isinstance(message, OwnAction): room_buffer.self_action(message) return + elif isinstance(message, OwnMessage): + room_buffer.self_message(message) + return raise NotImplementedError("Unsupported message of type {}".format( type(message))) @@ -736,19 +673,18 @@ class MatrixServer: self.sync() - def _queue_joined_info(self, response): - while response.joined_room_infos: - info = response.joined_room_infos.pop() + def _handle_room_info(self, response): + for room_id, join_info in response.rooms.join.items(): + if room_id not in self.buffers: + self.create_room_buffer(room_id) - if info.room_id not in self.buffers: - self.create_room_buffer(info.room_id) + room_buffer = self.find_room_from_id(room_id) - room = self.rooms[info.room_id] + for event in join_info.state: + room_buffer.handle_state_event(event) - if not room.prev_batch: - room.prev_batch = info.prev_batch - - self.event_queue.append(info) + for event in join_info.timeline.events: + room_buffer.handle_timeline_event(event) def _handle_sync(self, response): # we got the same batch again, nothing to do @@ -756,27 +692,20 @@ class MatrixServer: self.sync() return - self._queue_joined_info(response) + self._handle_room_info(response) + # self._queue_joined_info(response) self.next_batch = response.next_batch # self.check_one_time_keys(response.one_time_key_count) - self.handle_events() + # self.handle_events() def handle_matrix_response(self, response): - if isinstance(response, MatrixLoginEvent): - self._handle_login(response) - - elif isinstance(response, MatrixSyncEvent): - self._handle_sync(response) - - elif isinstance(response, MatrixSendEvent): - _, room_buffer = self.find_room_from_id(response.room_id) + if isinstance(response, MatrixSendEvent): + room_buffer = self.find_room_from_id(response.room_id) self.handle_own_messages(room_buffer, response.message) elif isinstance(response, MatrixBacklogEvent): - room, room_buffer = self.find_room_from_id(response.room_id) + room_buffer = self.find_room_from_id(response.room_id) room_buffer.handle_backlog(response.events) - room.prev_batch = response.end_token - room.backlog_pending = False W.bar_item_update("buffer_modes") elif isinstance(response, MatrixErrorEvent): @@ -787,10 +716,14 @@ class MatrixServer: if isinstance(response, LoginResponse): self._handle_login(response) + elif isinstance(response, SyncRepsponse): + self._handle_sync(response) def nio_parse_response(self, response): if isinstance(response, MatrixLoginMessage): self.nio_client.receive("login", response.response.body) + elif isinstance(response, MatrixSyncMessage): + self.nio_client.receive("sync", response.response.body) self.nio_receive() @@ -804,7 +737,7 @@ class MatrixServer: if ('content-type' in message.response.headers and message.response.headers['content-type'] == 'application/json'): - if isinstance(message, MatrixLoginMessage): + if isinstance(message, (MatrixLoginMessage, MatrixSyncMessage)): self.nio_parse_response(message) else: @@ -850,24 +783,21 @@ class MatrixServer: return def create_room_buffer(self, room_id): - room = MatrixRoom(room_id, self.user_id) + room = self.nio_client.rooms[room_id] buf = RoomBuffer(room, self.name) # TODO this should turned into a propper class self.room_buffers[room_id] = buf self.buffers[room_id] = buf.weechat_buffer._ptr - self.rooms[room_id] = room def find_room_from_ptr(self, pointer): room_id = key_from_value(self.buffers, pointer) - room = self.rooms[room_id] room_buffer = self.room_buffers[room_id] - return room, room_buffer + return room_buffer def find_room_from_id(self, room_id): - room = self.rooms[room_id] room_buffer = self.room_buffers[room_id] - return room, room_buffer + return room_buffer @utf8_decode From fc4c879e0dd4c0dbd863742501e2a8722874241d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 20 Jul 2018 19:14:32 +0200 Subject: [PATCH 064/269] matrix: Switch to the nio http client. --- main.py | 46 +-- matrix/api.py | 720 ---------------------------------- matrix/encryption.py | 913 ------------------------------------------- matrix/events.py | 386 ------------------ matrix/http.py | 93 ----- matrix/server.py | 379 ++---------------- 6 files changed, 55 insertions(+), 2482 deletions(-) delete mode 100644 matrix/api.py delete mode 100644 matrix/encryption.py delete mode 100644 matrix/events.py delete mode 100644 matrix/http.py diff --git a/main.py b/main.py index ff9db4e..781d643 100644 --- a/main.py +++ b/main.py @@ -32,6 +32,8 @@ from future.utils import bytes_to_native_str as n # pylint: disable=unused-import from typing import (List, Set, Dict, Tuple, Text, Optional, AnyStr, Deque, Any) +from nio import TransportType + from matrix.colors import Formatted from matrix.utf import utf8_decode from matrix.http import HttpResponse @@ -288,49 +290,31 @@ def receive_cb(server_name, file_descriptor): server.disconnect() break - received = len(data) # type: int - parsed_bytes = server.http_parser.execute(data, received) + server.client.receive(data) - assert parsed_bytes == received + response = server.client.next_response() - if server.http_parser.is_partial_body(): - server.http_buffer.append(server.http_parser.recv_body()) - - if server.http_parser.is_message_complete(): - status = server.http_parser.get_status_code() - headers = server.http_parser.get_headers() - body = b"".join(server.http_buffer) - - message = server.receive_queue.popleft() - message.response = HttpResponse(status, headers, body) - receive_time = time.time() - server.lag = (receive_time - message.send_time) * 1000 - server.lag_done = True - W.bar_item_update("lag") - message.receive_time = receive_time - - prnt_debug(DebugType.MESSAGING, server, - ("{prefix}Received message of type {t} and " - "status {s}").format( - prefix=W.prefix("error"), - t=message.__class__.__name__, - s=status)) - - # Message done, reset the parser state. - server.reset_parser() - - server.handle_response(message) + if response: + server.handle_response(response) break return W.WEECHAT_RC_OK def finalize_connection(server): - hook = W.hook_fd(server.socket.fileno(), 1, 0, 0, "receive_cb", server.name) + hook = W.hook_fd( + server.socket.fileno(), + 1, + 0, + 0, + "receive_cb", + server.name + ) server.fd_hook = hook server.connected = True server.connecting = False + server.client.connect(TransportType.HTTP) server.login() diff --git a/matrix/api.py b/matrix/api.py deleted file mode 100644 index 0919bbb..0000000 --- a/matrix/api.py +++ /dev/null @@ -1,720 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2018 Damir Jelić -# -# Permission to use, copy, modify, and/or distribute this software for -# any purpose with or without fee is hereby granted, provided that the -# above copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -from __future__ import unicode_literals -from builtins import str - -import time -import json -from enum import Enum, unique -from functools import partial - -try: - from json.decoder import JSONDecodeError -except ImportError: - JSONDecodeError = ValueError - -try: - from urllib import quote, urlencode - from urlparse import urlparse -except ImportError: - from urllib.parse import quote, urlencode, urlparse - -from matrix.http import RequestType, HttpRequest -import matrix.events as MatrixEvents - -MATRIX_API_PATH = "/_matrix/client/r0" # type: str - - -class MatrixClient: - - def __init__( - self, - host, # type: str - access_token="", # type: str - user_agent="" # type: str - ): - # type: (...) -> None - self.host = host - self.user_agent = user_agent - self.access_token = access_token - self.txn_id = 0 # type: int - - def _get_txn_id(self): - txn_id = self.txn_id - self.txn_id = self.txn_id + 1 - return txn_id - - def login(self, user, password, device_name="", device_id=""): - # type (str, str, str) -> HttpRequest - path = ("{api}/login").format(api=MATRIX_API_PATH) - - post_data = { - "type": "m.login.password", - "user": user, - "password": password - } - - if device_id: - post_data["device_id"] = device_id - - if device_name: - post_data["initial_device_display_name"] = device_name - - return HttpRequest(RequestType.POST, self.host, path, post_data) - - def sync(self, next_batch="", sync_filter=None): - # type: (str, Dict[Any, Any]) -> HttpRequest - assert self.access_token - - query_parameters = {"access_token": self.access_token} - - if sync_filter: - query_parameters["filter"] = json.dumps( - sync_filter, separators=(",", ":")) - - if next_batch: - query_parameters["since"] = next_batch - - path = ("{api}/sync?{query_params}").format( - api=MATRIX_API_PATH, query_params=urlencode(query_parameters)) - - return HttpRequest(RequestType.GET, self.host, path) - - def room_encrypted_message(self, room_id, content): - # type: (str, Dict[Any, Any]) -> HttpRequest - query_parameters = {"access_token": self.access_token} - - path = ("{api}/rooms/{room}/send/m.room.encrypted/" - "{tx_id}?{query_parameters}").format( - api=MATRIX_API_PATH, - room=quote(room_id), - tx_id=quote(str(self._get_txn_id())), - query_parameters=urlencode(query_parameters)) - - return HttpRequest(RequestType.PUT, self.host, path, content) - - def room_send_message(self, - room_id, - message_type, - content, - formatted_content=None): - # type: (str, str, str) -> HttpRequest - query_parameters = {"access_token": self.access_token} - - body = {"msgtype": message_type, "body": content} - - if formatted_content: - body["format"] = "org.matrix.custom.html" - body["formatted_body"] = formatted_content - - path = ("{api}/rooms/{room}/send/m.room.message/" - "{tx_id}?{query_parameters}").format( - api=MATRIX_API_PATH, - room=quote(room_id), - tx_id=quote(str(self._get_txn_id())), - query_parameters=urlencode(query_parameters)) - - return HttpRequest(RequestType.PUT, self.host, path, body) - - def room_topic(self, room_id, topic): - # type: (str, str) -> HttpRequest - query_parameters = {"access_token": self.access_token} - - content = {"topic": topic} - - path = ("{api}/rooms/{room}/state/m.room.topic?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - room=quote(room_id), - query_parameters=urlencode(query_parameters)) - - return HttpRequest(RequestType.PUT, self.host, path, content) - - def room_redact(self, room_id, event_id, reason=None): - # type: (str, str, str) -> HttpRequest - query_parameters = {"access_token": self.access_token} - content = {} - - if reason: - content["reason"] = reason - - path = ("{api}/rooms/{room}/redact/{event_id}/{tx_id}?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - room=quote(room_id), - event_id=quote(event_id), - tx_id=quote(str(self._get_txn_id())), - query_parameters=urlencode(query_parameters)) - - return HttpRequest(RequestType.PUT, self.host, path, content) - - def room_get_messages(self, - room_id, - start_token, - end_token="", - limit=10, - direction='b'): - query_parameters = { - "access_token": self.access_token, - "from": start_token, - "dir": direction, - "limit": str(limit) - } - - if end_token: - query_parameters["to"] = end_token - - path = ("{api}/rooms/{room}/messages?{query_parameters}").format( - api=MATRIX_API_PATH, - room=quote(room_id), - query_parameters=urlencode(query_parameters)) - - return HttpRequest(RequestType.GET, self.host, path) - - def room_join(self, room_id): - query_parameters = {"access_token": self.access_token} - - path = ("{api}/join/{room_id}?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - room_id=quote(room_id), - query_parameters=urlencode(query_parameters)) - - return HttpRequest(RequestType.POST, self.host, path) - - def room_leave(self, room_id): - query_parameters = {"access_token": self.access_token} - - path = ("{api}/rooms/{room_id}/leave?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - room_id=quote(room_id), - query_parameters=urlencode(query_parameters)) - - return HttpRequest(RequestType.POST, self.host, path) - - def room_invite(self, room_id, user_id): - query_parameters = {"access_token": self.access_token} - - content = {"user_id": user_id} - - path = ("{api}/rooms/{room_id}/invite?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - room_id=quote(room_id), - query_parameters=urlencode(query_parameters)) - - return HttpRequest(RequestType.POST, self.host, path, content) - - def room_kick(self, room_id, user_id, reason=None): - query_parameters = {"access_token": self.access_token} - - content = {"user_id": user_id} - - if reason: - content["reason"] = reason - - path = ("{api}/rooms/{room_id}/kick?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - room_id=quote(room_id), - query_parameters=urlencode(query_parameters)) - - h = HttpRequest(RequestType.POST, self.host, path, content) - return h - - def keys_upload(self, user_id, device_id, olm, keys=None, - one_time_keys=None): - query_parameters = {"access_token": self.access_token} - - path = ("{api}/keys/upload?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - query_parameters=urlencode(query_parameters)) - - content = {} - - if keys: - device_keys = { - "algorithms": [ - "m.olm.v1.curve25519-aes-sha2", - "m.megolm.v1.aes-sha2" - ], - "device_id": device_id, - "user_id": user_id, - "keys": { - "curve25519:" + device_id: keys["curve25519"], - "ed25519:" + device_id: keys["ed25519"] - } - } - - signature = olm.sign_json(device_keys) - - device_keys["signatures"] = { - user_id: { - "ed25519:" + device_id: signature - } - } - - content["device_keys"] = device_keys - - if one_time_keys: - one_time_key_dict = {} - - for key_id, key in one_time_keys.items(): - key_dict = {"key": key} - signature = olm.sign_json(key_dict) - - one_time_key_dict["signed_curve25519:" + key_id] = { - "key": key_dict.pop("key"), - "signatures": { - user_id: { - "ed25519:" + device_id: signature - } - } - } - - content["one_time_keys"] = one_time_key_dict - - return HttpRequest(RequestType.POST, self.host, path, content) - - def keys_query(self, users): - query_parameters = {"access_token": self.access_token} - - path = ("{api}/keys/query?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - query_parameters=urlencode(query_parameters)) - - content = { - "device_keys": {user: {} for user in users} - } - - return HttpRequest(RequestType.POST, self.host, path, content) - - def keys_claim(self, key_dict): - query_parameters = {"access_token": self.access_token} - - path = ("{api}/keys/claim?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - query_parameters=urlencode(query_parameters)) - - content = { - "one_time_keys": key_dict, - "timeout": 10000 - } - - return HttpRequest(RequestType.POST, self.host, path, content) - - def to_device(self, event_type, content): - query_parameters = {"access_token": self.access_token} - - path = ("{api}/sendToDevice/{event_type}/{tx_id}?" - "{query_parameters}").format( - api=MATRIX_API_PATH, - event_type=event_type, - tx_id=quote(str(self._get_txn_id())), - query_parameters=urlencode(query_parameters)) - - return HttpRequest(RequestType.PUT, self.host, path, content) - - @staticmethod - def mxc_to_http(mxc): - # type: (str) -> str - url = urlparse(mxc) - - if url.scheme != "mxc": - return None - - if not url.netloc or not url.path: - return None - - return "https://{}/_matrix/media/r0/download/{}{}".format( - url.netloc, - url.netloc, - url.path - ) - - -class MatrixMessage(): - - def __init__( - self, - request_func, # type: Callable[[...], HttpRequest] - func_args, - ): - # type: (...) -> None - # yapf: disable - - self.request = None # type: HttpRequest - self.response = None # type: HttpResponse - self.decoded_response = None # type: Dict[Any, Any] - - self.creation_time = time.time() # type: float - self.send_time = None # type: float - self.receive_time = None # type: float - self.event = None - - self.request = request_func(**func_args) - # yapf: enable - - def decode_body(self, server): - try: - self.decoded_response = json.loads( - self.response.body, encoding='utf-8') - return (True, None) - except Exception as error: - return (False, error) - - def _decode(self, server, object_hook): - try: - parsed_dict = json.loads( - self.response.body, - encoding='utf-8', - ) - - self.event = object_hook(parsed_dict) - - return (True, None) - - except JSONDecodeError as error: - return (False, error) - - -class MatrixLoginMessage(MatrixMessage): - - def __init__(self, client, user, password, device_name, device_id=None): - data = {"user": user, "password": password, "device_name": device_name} - - if device_id: - data["device_id"] = device_id - - MatrixMessage.__init__(self, client.login, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixLoginEvent.from_dict, server) - - return self._decode(server, object_hook) - - -class MatrixSyncMessage(MatrixMessage): - - def __init__(self, client, next_batch=None, limit=None): - data = {} - - if next_batch: - data["next_batch"] = next_batch - - if limit: - data["sync_filter"] = {"room": {"timeline": {"limit": limit}}} - - MatrixMessage.__init__(self, client.sync, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixSyncEvent.from_dict, server) - - return self._decode(server, object_hook) - - -class MatrixSendMessage(MatrixMessage): - - def __init__(self, - client, - room_id, - formatted_message, - message_type="m.text"): - self.room_id = room_id - self.formatted_message = formatted_message - - data = { - "room_id": self.room_id, - "message_type": message_type, - "content": self.formatted_message.to_plain() - } - - if self.formatted_message.is_formatted(): - data["formatted_content"] = self.formatted_message.to_html() - - MatrixMessage.__init__(self, client.room_send_message, data) - - def decode_body(self, server): - object_hook = partial( - MatrixEvents.MatrixSendEvent.from_dict, - server, - self.room_id, - self.formatted_message, - ) - - return self._decode(server, object_hook) - - -class MatrixEmoteMessage(MatrixSendMessage): - - def __init__(self, client, room_id, formatted_message): - MatrixSendMessage.__init__(self, client, room_id, formatted_message, - "m.emote") - - def decode_body(self, server): - object_hook = partial( - MatrixEvents.MatrixEmoteEvent.from_dict, - server, - self.room_id, - self.formatted_message, - ) - - return self._decode(server, object_hook) - - -class MatrixTopicMessage(MatrixMessage): - - def __init__(self, client, room_id, topic): - self.room_id = room_id - self.topic = topic - - data = {"room_id": self.room_id, "topic": self.topic} - - MatrixMessage.__init__(self, client.room_topic, data) - - def decode_body(self, server): - object_hook = partial( - MatrixEvents.MatrixTopicEvent.from_dict, - server, - self.room_id, - self.topic, - ) - - return self._decode(server, object_hook) - - -class MatrixRedactMessage(MatrixMessage): - - def __init__(self, client, room_id, event_id, reason=None): - self.room_id = room_id - self.event_id = event_id - self.reason = reason - - data = {"room_id": self.room_id, "event_id": self.event_id} - - if reason: - data["reason"] = reason - - MatrixMessage.__init__(self, client.room_redact, data) - - def decode_body(self, server): - object_hook = partial( - MatrixEvents.MatrixRedactEvent.from_dict, - server, - self.room_id, - self.reason, - ) - - return self._decode(server, object_hook) - - -class MatrixBacklogMessage(MatrixMessage): - - def __init__(self, client, room_id, token, limit): - self.room_id = room_id - - data = { - "room_id": self.room_id, - "start_token": token, - "direction": "b", - "limit": limit - } - - MatrixMessage.__init__(self, client.room_get_messages, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixBacklogEvent.from_dict, server, - self.room_id) - - return self._decode(server, object_hook) - - -class MatrixJoinMessage(MatrixMessage): - - def __init__(self, client, room_id): - self.room_id = room_id - - data = {"room_id": self.room_id} - - MatrixMessage.__init__(self, client.room_join, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixJoinEvent.from_dict, server, - self.room_id) - - return self._decode(server, object_hook) - - -class MatrixPartMessage(MatrixMessage): - - def __init__(self, client, room_id): - self.room_id = room_id - - data = {"room_id": self.room_id} - - MatrixMessage.__init__(self, client.room_leave, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixPartEvent.from_dict, server, - self.room_id) - - return self._decode(server, object_hook) - - -class MatrixInviteMessage(MatrixMessage): - - def __init__(self, client, room_id, user_id): - self.room_id = room_id - self.user_id = user_id - - data = {"room_id": self.room_id, "user_id": self.user_id} - - MatrixMessage.__init__(self, client.room_invite, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixInviteEvent.from_dict, server, - self.room_id, self.user_id) - - return self._decode(server, object_hook) - - -class MatrixKickMessage(MatrixMessage): - - def __init__(self, client, room_id, user_id, reason=None): - self.room_id = room_id - self.user_id = user_id - self.reason = reason - - data = {"room_id": self.room_id, - "user_id": self.user_id, - "reason": reason} - - MatrixMessage.__init__(self, client.room_kick, data) - - def decode_body(self, server): - object_hook = partial( - MatrixEvents.MatrixKickEvent.from_dict, - server, - self.room_id, - self.user_id, - self.reason) - - return self._decode(server, object_hook) - - -class MatrixKeyUploadMessage(MatrixMessage): - - def __init__(self, client, user_id, device_id, olm, keys=None, - one_time_keys=None): - data = { - "device_id": device_id, - "user_id": user_id, - "olm": olm, - "keys": keys, - "one_time_keys": one_time_keys - } - - self.device_keys = True if keys else False - - MatrixMessage.__init__(self, client.keys_upload, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixKeyUploadEvent.from_dict, - server, self.device_keys) - - return self._decode(server, object_hook) - - -class MatrixKeyQueryMessage(MatrixMessage): - - def __init__(self, client, users): - data = { - "users": users, - } - - MatrixMessage.__init__(self, client.keys_query, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixKeyQueryEvent.from_dict, - server) - - return self._decode(server, object_hook) - - -class MatrixKeyClaimMessage(MatrixMessage): - - def __init__(self, client, room_id, key_dict): - self.room_id = room_id - data = { - "key_dict": key_dict, - } - - MatrixMessage.__init__(self, client.keys_claim, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixKeyClaimEvent.from_dict, - server, self.room_id) - - return self._decode(server, object_hook) - - -class MatrixToDeviceMessage(MatrixMessage): - def __init__(self, client, to_device_dict): - data = { - "content": to_device_dict, - "event_type": "m.room.encrypted" - } - - MatrixMessage.__init__(self, client.to_device, data) - - def decode_body(self, server): - object_hook = partial(MatrixEvents.MatrixToDeviceEvent.from_dict, - server) - - return self._decode(server, object_hook) - - -class MatrixEncryptedMessage(MatrixMessage): - - def __init__(self, - client, - room_id, - formatted_message, - content): - self.room_id = room_id - self.formatted_message = formatted_message - - data = { - "room_id": self.room_id, - "content": content - } - - MatrixMessage.__init__(self, client.room_encrypted_message, data) - - def decode_body(self, server): - object_hook = partial( - MatrixEvents.MatrixSendEvent.from_dict, - server, - self.room_id, - self.formatted_message, - ) - - return self._decode(server, object_hook) diff --git a/matrix/encryption.py b/matrix/encryption.py deleted file mode 100644 index 851baf7..0000000 --- a/matrix/encryption.py +++ /dev/null @@ -1,913 +0,0 @@ -# -*- coding: utf-8 -*- - -# Weechat Matrix Protocol Script -# Copyright © 2018 Damir Jelić -# -# Permission to use, copy, modify, and/or distribute this software for -# any purpose with or without fee is hereby granted, provided that the -# above copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -from __future__ import unicode_literals - -import os -import json -import sqlite3 -import pprint -import argparse - -# pylint: disable=redefined-builtin -from builtins import str, bytes - -from collections import defaultdict -from itertools import chain -from functools import wraps -from future.moves.itertools import zip_longest - -try: - FileNotFoundError -except NameError: - FileNotFoundError = IOError - -import matrix.globals - -try: - from olm import ( - Account, - OlmAccountError, - Session, - InboundSession, - OutboundSession, - OlmSessionError, - OlmPreKeyMessage, - InboundGroupSession, - OutboundGroupSession, - OlmGroupSessionError - ) -except ImportError: - matrix.globals.ENCRYPTION = False - -from matrix.globals import W, SERVERS -from matrix.utils import sanitize_id -from matrix.utf import utf8_decode - - -class ParseError(Exception): - pass - - -class OlmTrustError(Exception): - pass - - -class WeechatArgParse(argparse.ArgumentParser): - def print_usage(self, file): - pass - - def error(self, message): - m = ("{prefix}Error: {message} for command {command} " - "(see /help {command})").format(prefix=W.prefix("error"), - message=message, - command=self.prog) - W.prnt("", m) - raise ParseError - - -def own_buffer_or_error(f): - - @wraps(f) - def wrapper(data, buffer, *args, **kwargs): - - for server in SERVERS.values(): - if buffer in server.buffers.values(): - return f(server.name, buffer, *args, **kwargs) - elif buffer == server.server_buffer: - return f(server.name, buffer, *args, **kwargs) - - W.prnt("", "{prefix}matrix: command \"olm\" must be executed on a " - "matrix buffer (server or channel)".format( - prefix=W.prefix("error"))) - - return W.WEECHAT_RC_OK - - return wrapper - - -def encrypt_enabled(f): - @wraps(f) - def wrapper(*args, **kwds): - if matrix.globals.ENCRYPTION: - return f(*args, **kwds) - return None - - return wrapper - - -@encrypt_enabled -def matrix_hook_olm_command(): - W.hook_command( - # Command name and short description - "olm", - "Matrix olm encryption command", - # Synopsis - ("info all|blacklisted|private|unverified|verified ||" - "blacklist ||" - "unverify ||" - "verify "), - # Description - (" info: show info about known devices and their keys\n" - "blacklist: blacklist a device\n" - " unverify: unverify a device\n" - " verify: verify a device\n\n" - "Examples:\n"), - # Completions - ('info all|blacklisted|private|unverified|verified ||' - 'blacklist %(device_ids) ||' - 'unverify %(user_ids) %(device_ids) ||' - 'verify %(olm_user_ids) %(olm_devices)'), - # Function name - 'matrix_olm_command_cb', - '') - - -def olm_cmd_parse_args(args): - parser = WeechatArgParse(prog="olm") - subparsers = parser.add_subparsers(dest="subcommand") - - info_parser = subparsers.add_parser("info") - info_parser.add_argument( - "category", nargs="?", default="private", - choices=[ - "all", - "blacklisted", - "private", - "unverified", - "verified" - ]) - info_parser.add_argument("filter", nargs="?") - - verify_parser = subparsers.add_parser("verify") - verify_parser.add_argument("user_filter") - verify_parser.add_argument("device_filter", nargs="?") - - try: - parsed_args = parser.parse_args(args.split()) - return parsed_args - except ParseError: - return None - - -def grouper(iterable, n, fillvalue=None): - "Collect data into fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" - args = [iter(iterable)] * n - return zip_longest(*args, fillvalue=fillvalue) - - -def partition_key(key): - groups = grouper(key, 4, " ") - return ' '.join(''.join(g) for g in groups) - - -def olm_info_command(server, args): - olm = server.olm - - if args.category == "private": - device_msg = (" - Device ID: {}\n".format(server.device_id) - if server.device_id else "") - id_key = partition_key(olm.account.identity_keys["curve25519"]) - fp_key = partition_key(olm.account.identity_keys["ed25519"]) - message = ("{prefix}matrix: Identity keys:\n" - " - User: {user}\n" - "{device_msg}" - " - Identity key: {id_key}\n" - " - Fingerprint key: {fp_key}\n").format( - prefix=W.prefix("network"), - user=server.user, - device_msg=device_msg, - id_key=id_key, - fp_key=fp_key) - W.prnt(server.server_buffer, message) - elif args.category == "all": - for user, keys in olm.device_keys.items(): - message = ("{prefix}matrix: Identity keys:\n" - " - User: {user}\n").format( - prefix=W.prefix("network"), - user=user) - W.prnt(server.server_buffer, message) - - for key in keys: - id_key = partition_key(key.keys["curve25519"]) - fp_key = partition_key(key.keys["ed25519"]) - device_msg = (" - Device ID: {}\n".format( - key.device_id) if key.device_id else "") - message = ("{device_msg}" - " - Identity key: {id_key}\n" - " - Fingerprint key: {fp_key}\n\n").format( - device_msg=device_msg, - id_key=id_key, - fp_key=fp_key) - W.prnt(server.server_buffer, message) - - -def olm_verify_command(server, args): - olm = server.olm - devices = olm.device_keys - filtered_devices = [] - - if args.user_filter == "*": - filtered_devices = devices.values() - else: - device_keys = filter(lambda x: args.user_filter in x, devices) - filtered_devices = [devices[x] for x in device_keys] - - filtered_devices = chain.from_iterable(filtered_devices) - - if args.device_filter and args.device_filter != "*": - filtered_devices = filter( - lambda x: args.device_filter in x.device_id, - filtered_devices - ) - - key_list = [] - - for device in list(filtered_devices): - if olm.verify_device(device): - key_list.append(str(device)) - - if not key_list: - message = ("{prefix}matrix: No matching unverified devices " - "found.").format(prefix=W.prefix("error")) - W.prnt(server.server_buffer, message) - return - - key_str = "\n - ".join(key_list) - message = ("{prefix}matrix: Verified keys:\n" - " - {key_str}").format(prefix=W.prefix("join"), - key_str=key_str) - - W.prnt(server.server_buffer, message) - - -@own_buffer_or_error -@utf8_decode -def matrix_olm_command_cb(server_name, buffer, args): - server = SERVERS[server_name] - parsed_args = olm_cmd_parse_args(args) - - if not parsed_args: - return W.WEECHAT_RC_OK - - if not parsed_args.subcommand or parsed_args.subcommand == "info": - olm_info_command(server, parsed_args) - elif parsed_args.subcommand == "verify": - olm_verify_command(server, parsed_args) - else: - message = ("{prefix}matrix: Command not implemented.".format( - prefix=W.prefix("error"))) - W.prnt(server.server_buffer, message) - - return W.WEECHAT_RC_OK - - -class EncryptionError(Exception): - pass - - -class DeviceStore(object): - def __init__(self, filename): - # type: (str) -> None - self._entries = [] - self._filename = filename - - self._load(filename) - - def _load(self, filename): - # type: (str) -> None - try: - with open(filename, "r") as f: - for line in f: - line = line.strip() - - if not line or line.startswith("#"): - continue - - entry = StoreEntry.from_line(line) - - if not entry: - continue - - self._entries.append(entry) - except FileNotFoundError: - pass - - def _save_store(f): - @wraps(f) - def decorated(*args, **kwargs): - self = args[0] - ret = f(*args, **kwargs) - self._save() - return ret - - return decorated - - def _save(self): - # type: (str) -> None - with open(self._filename, "w") as f: - for entry in self._entries: - line = entry.to_line() - f.write(line) - - @_save_store - def add(self, device): - # type: (OlmDeviceKey) -> None - new_entries = StoreEntry.from_olmdevice(device) - self._entries += new_entries - - # Remove duplicate entries - self._entries = list(set(self._entries)) - - self._save() - - @_save_store - def remove(self, device): - # type: (OlmDeviceKey) -> int - removed = 0 - entries = StoreEntry.from_olmdevice(device) - - for entry in entries: - if entry in self._entries: - self._entries.remove(entry) - removed += 1 - - self._save() - - return removed - - def check(self, device): - # type: (OlmDeviceKey) -> bool - entries = StoreEntry.from_olmdevice(device) - result = map(lambda entry: entry in self._entries, entries) - - if False in result: - return False - - return True - - -class StoreEntry(object): - def __init__(self, user_id, device_id, key_type, key): - # type: (str, str, str, str) -> None - self.user_id = user_id - self.device_id = device_id - self.key_type = key_type - self.key = key - - @classmethod - def from_line(cls, line): - # type: (str) -> StoreEntry - fields = line.split(' ') - - if len(fields) < 4: - return None - - user_id, device_id, key_type, key = fields[:4] - - if key_type == "matrix-ed25519": - return cls(user_id, device_id, "ed25519", key) - else: - return None - - @classmethod - def from_olmdevice(cls, device_key): - # type: (OlmDeviceKey) -> [StoreEntry] - entries = [] - - user_id = device_key.user_id - device_id = device_key.device_id - - for key_type, key in device_key.keys.items(): - if key_type == "ed25519": - entries.append(cls(user_id, device_id, "ed25519", key)) - - return entries - - def to_line(self): - # type: () -> str - key_type = "matrix-{}".format(self.key_type) - line = "{} {} {} {}\n".format( - self.user_id, - self.device_id, - key_type, - self.key - ) - return line - - def __hash__(self): - # type: () -> int - return hash(str(self)) - - def __str__(self): - # type: () -> str - key_type = "matrix-{}".format(self.key_type) - line = "{} {} {} {}".format( - self.user_id, - self.device_id, - key_type, - self.key - ) - return line - - def __eq__(self, value): - # type: (StoreEntry) -> bool - if (self.user_id == value.user_id - and self.device_id == value.device_id - and self.key_type == value.key_type and self.key == value.key): - return True - - return False - - -class OlmDeviceKey(): - def __init__(self, user_id, device_id, key_dict): - # type: (str, str, Dict[str, str]) - self.user_id = user_id - self.device_id = device_id - self.keys = key_dict - - def __str__(self): - # type: () -> str - return "{} {} {}".format( - self.user_id, self.device_id, self.keys["ed25519"]) - - def __repr__(self): - # type: () -> str - return str(self) - - -class OneTimeKey(): - def __init__(self, user_id, device_id, key): - # type: (str, str, str) -> None - self.user_id = user_id - self.device_id = device_id - self.key = key - - -class Olm(): - - @encrypt_enabled - def __init__( - self, - user, - device_id, - session_path, - database=None, - account=None, - sessions=None, - inbound_group_sessions=None - ): - # type: (str, str, str, Account, Dict[str, List[Session]) -> None - self.user = user - self.device_id = device_id - self.session_path = session_path - self.database = database - self.device_keys = {} - self.shared_sessions = [] - - if not database: - db_file = "{}_{}.db".format(user, device_id) - db_path = os.path.join(session_path, db_file) - self.database = sqlite3.connect(db_path) - Olm._check_db_tables(self.database) - - if account: - self.account = account - - else: - self.account = Account() - self._insert_acc_to_db() - - if not sessions: - sessions = defaultdict(lambda: defaultdict(list)) - - if not inbound_group_sessions: - inbound_group_sessions = defaultdict(dict) - - self.sessions = sessions - self.inbound_group_sessions = inbound_group_sessions - self.outbound_group_sessions = {} - - trust_file_path = "{}_{}.trusted_devices".format(user, device_id) - self.trust_db = DeviceStore(os.path.join(session_path, trust_file_path)) - - def _create_session(self, sender, sender_key, message): - W.prnt("", "matrix: Creating session for {}".format(sender)) - session = InboundSession(self.account, message, sender_key) - W.prnt("", "matrix: Created session for {}".format(sender)) - self.account.remove_one_time_keys(session) - self._update_acc_in_db() - - return session - - def verify_device(self, device): - if self.trust_db.check(device): - return False - - self.trust_db.add(device) - return True - - def unverify_device(self, device): - self.trust_db.remove(device) - - def create_session(self, user_id, device_id, one_time_key): - W.prnt("", "matrix: Creating session for {}".format(user_id)) - id_key = None - - for user, keys in self.device_keys.items(): - if user != user_id: - continue - - for key in keys: - if key.device_id == device_id: - id_key = key.keys["curve25519"] - break - - if not id_key: - W.prnt("", "ERRR not found ID key") - W.prnt("", "Found id key {}".format(id_key)) - session = OutboundSession(self.account, id_key, one_time_key) - self._update_acc_in_db() - self.sessions[user_id][device_id].append(session) - self._store_session(user_id, device_id, session) - W.prnt("", "matrix: Created session for {}".format(user_id)) - - def create_group_session(self, room_id, session_id, session_key): - W.prnt("", "matrix: Creating group session for {}".format(room_id)) - session = InboundGroupSession(session_key) - self.inbound_group_sessions[room_id][session_id] = session - self._store_inbound_group_session(room_id, session) - - def create_outbound_group_session(self, room_id): - session = OutboundGroupSession() - self.outbound_group_sessions[room_id] = session - self.create_group_session(room_id, session.id, session.session_key) - - @encrypt_enabled - def get_missing_sessions(self, users): - # type: (List[str]) -> Dict[str, Dict[str, str]] - missing = {} - - for user in users: - devices = [] - - for key in self.device_keys[user]: - # we don't need a session for our own device, skip it - if key.device_id == self.device_id: - continue - - if not self.sessions[user][key.device_id]: - W.prnt("", "Missing session for device {}".format(key.device_id)) - devices.append(key.device_id) - - if devices: - missing[user] = {device: "signed_curve25519" for - device in devices} - - return missing - - @encrypt_enabled - def decrypt(self, sender, sender_key, message): - plaintext = None - - for device_id, session_list in self.sessions[sender].items(): - for session in session_list: - W.prnt("", "Trying session for device {}".format(device_id)) - try: - if isinstance(message, OlmPreKeyMessage): - if not session.matches(message): - continue - - W.prnt("", "Decrypting using existing session") - plaintext = session.decrypt(message) - parsed_plaintext = json.loads(plaintext, encoding='utf-8') - W.prnt("", "Decrypted using existing session") - return parsed_plaintext - except OlmSessionError: - pass - - try: - session = self._create_session(sender, sender_key, message) - except OlmSessionError: - return None - - try: - plaintext = session.decrypt(message) - parsed_plaintext = json.loads(plaintext, encoding='utf-8') - - device_id = sanitize_id(parsed_plaintext["sender_device"]) - self.sessions[sender][device_id].append(session) - self._store_session(sender, device_id, session) - return parsed_plaintext - except OlmSessionError: - return None - - def group_encrypt(self, room_id, plaintext_dict, own_id, users): - # type: (str, Dict[str, str]) -> Dict[str, str], Optional[Dict[Any, Any]] - plaintext_dict["room_id"] = room_id - to_device_dict = None - - if room_id not in self.outbound_group_sessions: - self.create_outbound_group_session(room_id) - - if self.outbound_group_sessions[room_id].id not in self.shared_sessions: - to_device_dict = self.share_group_session(room_id, own_id, users) - self.shared_sessions.append( - self.outbound_group_sessions[room_id].id - ) - - session = self.outbound_group_sessions[room_id] - - ciphertext = session.encrypt(Olm._to_json(plaintext_dict)) - - payload_dict = { - "algorithm": "m.megolm.v1.aes-sha2", - "sender_key": self.account.identity_keys["curve25519"], - "ciphertext": ciphertext, - "session_id": session.id, - "device_id": self.device_id - } - - return payload_dict, to_device_dict - - @encrypt_enabled - def group_decrypt(self, room_id, session_id, ciphertext): - if session_id not in self.inbound_group_sessions[room_id]: - return None, None - - session = self.inbound_group_sessions[room_id][session_id] - - try: - return session.decrypt(ciphertext) - except OlmGroupSessionError: - pass - - return None, None - - def share_group_session(self, room_id, own_id, users): - group_session = self.outbound_group_sessions[room_id] - - key_content = { - "algorithm": "m.megolm.v1.aes-sha2", - "room_id": room_id, - "session_id": group_session.id, - "session_key": group_session.session_key, - "chain_index": group_session.message_index - } - - payload_dict = { - "type": "m.room_key", - "content": key_content, - # TODO we don't have the user_id in the Olm class - "sender": own_id, - "sender_device": self.device_id, - "keys": { - "ed25519": self.account.identity_keys["ed25519"] - } - } - - to_device_dict = { - "messages": {} - } - - for user in users: - if user not in self.device_keys: - continue - - for key in self.device_keys[user]: - if key.device_id == self.device_id: - continue - - if not self.sessions[user][key.device_id]: - continue - - if not self.trust_db.check(key): - raise OlmTrustError - - device_payload_dict = payload_dict.copy() - # TODO sort the sessions - session = self.sessions[user][key.device_id][0] - device_payload_dict["recipient"] = user - device_payload_dict["recipient_keys"] = { - "ed25519": key.keys["ed25519"] - } - - olm_message = session.encrypt( - Olm._to_json(device_payload_dict) - ) - - olm_dict = { - "algorithm": "m.olm.v1.curve25519-aes-sha2", - "sender_key": self.account.identity_keys["curve25519"], - "ciphertext": { - key.keys["curve25519"]: { - "type": (0 if isinstance( - olm_message, - OlmPreKeyMessage - ) else 1), - "body": olm_message.ciphertext - } - } - } - - if user not in to_device_dict["messages"]: - to_device_dict["messages"][user] = {} - - to_device_dict["messages"][user][key.device_id] = olm_dict - - # W.prnt("", pprint.pformat(to_device_dict)) - return to_device_dict - - @classmethod - @encrypt_enabled - def from_session_dir(cls, user, device_id, session_path): - # type: (Server) -> Olm - db_file = "{}_{}.db".format(user, device_id) - db_path = os.path.join(session_path, db_file) - database = sqlite3.connect(db_path) - Olm._check_db_tables(database) - - cursor = database.cursor() - - cursor.execute("select pickle from olmaccount where user = ?", (user,)) - row = cursor.fetchone() - account_pickle = row[0] - - cursor.execute("select user, device_id, pickle from olmsessions") - db_sessions = cursor.fetchall() - - cursor.execute("select room_id, pickle from inbound_group_sessions") - db_inbound_group_sessions = cursor.fetchall() - - cursor.close() - - sessions = defaultdict(lambda: defaultdict(list)) - inbound_group_sessions = defaultdict(dict) - - try: - try: - account_pickle = bytes(account_pickle, "utf-8") - except TypeError: - pass - - account = Account.from_pickle(account_pickle) - - for db_session in db_sessions: - session_pickle = db_session[2] - try: - session_pickle = bytes(session_pickle, "utf-8") - except TypeError: - pass - - sessions[db_session[0]][db_session[1]].append( - Session.from_pickle(session_pickle)) - - for db_session in db_inbound_group_sessions: - session_pickle = db_session[1] - try: - session_pickle = bytes(session_pickle, "utf-8") - except TypeError: - pass - - session = InboundGroupSession.from_pickle(session_pickle) - inbound_group_sessions[db_session[0]][session.id] = session - - return cls(user, device_id, session_path, database, account, - sessions, inbound_group_sessions) - except (OlmAccountError, OlmSessionError) as error: - raise EncryptionError(error) - - def _update_acc_in_db(self): - cursor = self.database.cursor() - cursor.execute("update olmaccount set pickle=? where user = ?", - (self.account.pickle(), self.user)) - self.database.commit() - cursor.close() - - def _update_sessions_in_db(self): - cursor = self.database.cursor() - - for user, session_dict in self.sessions.items(): - for device_id, session_list in session_dict.items(): - for session in session_list: - cursor.execute("""update olmsessions set pickle=? - where user = ? and session_id = ? and - device_id = ?""", - (session.pickle(), user, session.id, - device_id)) - self.database.commit() - - cursor.close() - - def _update_inbound_group_sessions(self): - cursor = self.database.cursor() - - for room_id, session_dict in self.inbound_group_sessions.items(): - for session in session_dict.values(): - cursor.execute("""update inbound_group_sessions set pickle=? - where room_id = ? and session_id = ?""", - (session.pickle(), room_id, session.id)) - self.database.commit() - - cursor.close() - - def _store_session(self, user, device_id, session): - cursor = self.database.cursor() - - cursor.execute("insert into olmsessions values(?,?,?,?)", - (user, device_id, session.id, session.pickle())) - - self.database.commit() - - cursor.close() - - def _store_inbound_group_session(self, room_id, session): - cursor = self.database.cursor() - - cursor.execute("insert into inbound_group_sessions values(?,?,?)", - (room_id, session.id, session.pickle())) - - self.database.commit() - - cursor.close() - - def _insert_acc_to_db(self): - cursor = self.database.cursor() - cursor.execute("insert into olmaccount values (?,?)", - (self.user, self.account.pickle())) - self.database.commit() - cursor.close() - - @staticmethod - def _check_db_tables(database): - cursor = database.cursor() - cursor.execute("""select name from sqlite_master where type='table' - and name='olmaccount'""") - if not cursor.fetchone(): - cursor.execute("create table olmaccount (user text, pickle text)") - database.commit() - - cursor.execute("""select name from sqlite_master where type='table' - and name='olmsessions'""") - if not cursor.fetchone(): - cursor.execute("""create table olmsessions (user text, - device_id text, session_id text, pickle text)""") - database.commit() - - cursor.execute("""select name from sqlite_master where type='table' - and name='inbound_group_sessions'""") - if not cursor.fetchone(): - cursor.execute("""create table inbound_group_sessions - (room_id text, session_id text, pickle text)""") - database.commit() - - cursor.close() - - @encrypt_enabled - def to_session_dir(self): - # type: (Server) -> None - try: - self._update_acc_in_db() - self._update_sessions_in_db() - except OlmAccountError as error: - raise EncryptionError(error) - - def sign_json(self, json_dict): - signature = self.account.sign(json.dumps( - json_dict, - ensure_ascii=False, - separators=(',', ':'), - sort_keys=True, - )) - - return signature - - @staticmethod - def _to_json(json_dict): - # type: (Dict[Any, Any]) -> str - return json.dumps( - json_dict, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True - ) - - @encrypt_enabled - def mark_keys_as_published(self): - self.account.mark_keys_as_published() diff --git a/matrix/events.py b/matrix/events.py deleted file mode 100644 index 3221516..0000000 --- a/matrix/events.py +++ /dev/null @@ -1,386 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2018 Damir Jelić -# -# Permission to use, copy, modify, and/or distribute this software for -# any purpose with or without fee is hereby granted, provided that the -# above copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -from __future__ import unicode_literals -from builtins import str - -import json -import pprint - -from collections import deque, defaultdict -from functools import partial -from operator import itemgetter - -from matrix.globals import W -from matrix.utils import (tags_for_message, sanitize_id, sanitize_token, - sanitize_text, tags_from_line_data) - -from matrix.encryption import OlmDeviceKey, OneTimeKey -from .buffer import RoomUser, OwnMessage, OwnAction - -try: - from olm.session import OlmMessage, OlmPreKeyMessage -except ImportError: - pass - - -class MatrixEvent(): - - def __init__(self, server): - self.server = server - - def execute(self): - pass - - -class MatrixErrorEvent(MatrixEvent): - - def __init__(self, server, error_message, fatal=False): - self.error_message = error_message - self.fatal = fatal - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, error_prefix, fatal, parsed_dict): - try: - message = "{prefix}: {error}".format( - prefix=error_prefix, error=sanitize_text(parsed_dict["error"])) - return cls(server, message, fatal=fatal) - except KeyError: - return cls( - server, ("{prefix}: Invalid JSON response " - "from server.").format(prefix=error_prefix), - fatal=fatal) - - -class MatrixKeyUploadEvent(MatrixEvent): - - def __init__(self, server, device_keys): - self.device_keys = device_keys - MatrixEvent.__init__(self, server) - - def execute(self): - self.server.olm.mark_keys_as_published() - self.server.store_olm() - - if not self.device_keys: - return - - message = "{prefix}matrix: Uploaded Olm device keys.".format( - prefix=W.prefix("network")) - - W.prnt(self.server.server_buffer, message) - - @classmethod - def from_dict(cls, server, device_keys, parsed_dict): - try: - return cls(server, device_keys) - except (KeyError, TypeError, ValueError): - return MatrixErrorEvent.from_dict(server, "Error uploading device" - "keys", False, parsed_dict) - - -class MatrixSendEvent(MatrixEvent): - - def __init__(self, server, room_id, message): - self.room_id = room_id - self.message = message - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, room_id, message, parsed_dict): - try: - event_id = sanitize_id(parsed_dict["event_id"]) - sender = server.user_id - age = 0 - formatted_message = message - - message = OwnMessage(sender, age, event_id, formatted_message) - - return cls(server, room_id, message) - except (KeyError, TypeError, ValueError): - return MatrixErrorEvent.from_dict(server, "Error sending message", - False, parsed_dict) - - -class MatrixEmoteEvent(MatrixSendEvent): - - @classmethod - def from_dict(cls, server, room_id, message, parsed_dict): - try: - event_id = sanitize_id(parsed_dict["event_id"]) - sender = server.user_id - age = 0 - formatted_message = message - - message = OwnAction(sender, age, event_id, formatted_message) - - return cls(server, room_id, message) - except (KeyError, TypeError, ValueError): - return MatrixErrorEvent.from_dict(server, "Error sending message", - False, parsed_dict) - - -class MatrixTopicEvent(MatrixEvent): - - def __init__(self, server, room_id, event_id, topic): - self.room_id = room_id - self.topic = topic - self.event_id = event_id - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, room_id, topic, parsed_dict): - try: - return cls(server, room_id, sanitize_id(parsed_dict["event_id"]), - topic) - except (KeyError, TypeError, ValueError): - return MatrixErrorEvent.from_dict(server, "Error setting topic", - False, parsed_dict) - - -class MatrixRedactEvent(MatrixEvent): - - def __init__(self, server, room_id, event_id, reason): - self.room_id = room_id - self.topic = reason - self.event_id = event_id - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, room_id, reason, parsed_dict): - try: - return cls(server, room_id, sanitize_id(parsed_dict["event_id"]), - reason) - except (KeyError, TypeError, ValueError): - return MatrixErrorEvent.from_dict(server, "Error redacting message", - False, parsed_dict) - - -class MatrixJoinEvent(MatrixEvent): - - def __init__(self, server, room, room_id): - self.room = room - self.room_id = room_id - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, room_id, parsed_dict): - try: - return cls( - server, - room_id, - sanitize_id(parsed_dict["room_id"]), - ) - except (KeyError, TypeError, ValueError): - return MatrixErrorEvent.from_dict(server, "Error joining room", - False, parsed_dict) - - -class MatrixPartEvent(MatrixEvent): - - def __init__(self, server, room_id): - self.room_id = room_id - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, room_id, parsed_dict): - try: - if parsed_dict == {}: - return cls(server, room_id) - - raise KeyError - except KeyError: - return MatrixErrorEvent.from_dict(server, "Error leaving room", - False, parsed_dict) - - -class MatrixInviteEvent(MatrixEvent): - - def __init__(self, server, room_id, user_id): - self.room_id = room_id - self.user_id = user_id - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, room_id, user_id, parsed_dict): - try: - if parsed_dict == {}: - return cls(server, room_id, user_id) - - raise KeyError - except KeyError: - return MatrixErrorEvent.from_dict(server, "Error inviting user", - False, parsed_dict) - - -class MatrixKickEvent(MatrixEvent): - - def __init__(self, server, room_id, user_id, reason): - self.room_id = room_id - self.user_id = user_id - self.reason = reason - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, room_id, user_id, reason, parsed_dict): - try: - if parsed_dict == {}: - return cls(server, room_id, user_id, reason) - - raise KeyError - except KeyError: - return MatrixErrorEvent.from_dict(server, "Error kicking user", - False, parsed_dict) - - -class MatrixKeyQueryEvent(MatrixEvent): - - def __init__(self, server, keys): - self.keys = keys - MatrixEvent.__init__(self, server) - - @staticmethod - def _get_keys(key_dict): - keys = {} - - for key_type, key in key_dict.items(): - key_type, _ = key_type.split(":") - keys[key_type] = key - - return keys - - @classmethod - def from_dict(cls, server, parsed_dict): - keys = defaultdict(list) - try: - for user_id, device_dict in parsed_dict["device_keys"].items(): - for device_id, key_dict in device_dict.items(): - device_keys = MatrixKeyQueryEvent._get_keys( - key_dict.pop("keys")) - keys[user_id].append(OlmDeviceKey(user_id, device_id, - device_keys)) - return cls(server, keys) - except KeyError: - # TODO error message - return MatrixErrorEvent.from_dict(server, "Error kicking user", - False, parsed_dict) - - def execute(self): - # TODO move this logic into an Olm method - olm = self.server.olm - - if olm.device_keys == self.keys: - return - - olm.device_keys = self.keys - # TODO invalidate megolm sessions for rooms that got new devices - - -class MatrixKeyClaimEvent(MatrixEvent): - - def __init__(self, server, room_id, keys): - self.keys = keys - self.room_id = room_id - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, room_id, parsed_dict): - W.prnt("", pprint.pformat(parsed_dict)) - keys = [] - try: - for user_id, user_dict in parsed_dict["one_time_keys"].items(): - for device_id, device_dict in user_dict.items(): - for key_dict in device_dict.values(): - # TODO check the signature of the key - key = OneTimeKey(user_id, device_id, key_dict["key"]) - keys.append(key) - - return cls(server, room_id, keys) - except KeyError: - return MatrixErrorEvent.from_dict( - server, ("Error claiming onetime keys."), False, parsed_dict) - - def execute(self): - server = self.server - olm = server.olm - - for key in self.keys: - olm.create_session(key.user_id, key.device_id, key.key) - - while server.encryption_queue[self.room_id]: - formatted_message = server.encryption_queue[self.room_id].popleft() - room, _ = server.find_room_from_id(self.room_id) - server.send_room_message(room, formatted_message, True) - - -class MatrixToDeviceEvent(MatrixEvent): - - def __init__(self, server): - MatrixEvent.__init__(self, server) - - @classmethod - def from_dict(cls, server, parsed_dict): - try: - if parsed_dict == {}: - return cls(server) - - raise KeyError - except KeyError: - return MatrixErrorEvent.from_dict(server, ("Error sending to " - "device message"), - False, parsed_dict) - - -class MatrixBacklogEvent(MatrixEvent): - - def __init__(self, server, room_id, end_token, events): - self.room_id = room_id - self.end_token = end_token - self.events = events - MatrixEvent.__init__(self, server) - - @staticmethod - def _room_event_from_dict(room_id, event_dict): - if room_id != event_dict["room_id"]: - raise ValueError - - if "redacted_by" in event_dict["unsigned"]: - return RoomRedactedMessageEvent.from_dict(event_dict) - - return RoomMessageEvent.from_dict(event_dict) - - @classmethod - def from_dict(cls, server, room_id, parsed_dict): - try: - end_token = sanitize_id(parsed_dict["end"]) - - if not parsed_dict["chunk"]: - return cls(server, room_id, end_token, []) - - event_func = partial(MatrixBacklogEvent._room_event_from_dict, - room_id) - - message_events = list( - filter(lambda event: event["type"] == "m.room.message", - parsed_dict["chunk"])) - - events = [event_func(m) for m in message_events] - - return cls(server, room_id, end_token, events) - except (KeyError, ValueError, TypeError): - return MatrixErrorEvent.from_dict(server, "Error fetching backlog", - False, parsed_dict) diff --git a/matrix/http.py b/matrix/http.py deleted file mode 100644 index 8ad1ef4..0000000 --- a/matrix/http.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2018 Damir Jelić -# -# Permission to use, copy, modify, and/or distribute this software for -# any purpose with or without fee is hereby granted, provided that the -# above copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -from __future__ import unicode_literals -from builtins import str - -import json -from enum import Enum, unique - - -@unique -class RequestType(Enum): - GET = 0 - POST = 1 - PUT = 2 - - -class HttpResponse: - - def __init__(self, status, headers, body): - self.status = status # type: int - self.headers = headers # type: Dict[str, str] - self.body = body # type: bytes - - -# yapf: disable -class HttpRequest: - def __init__( - self, - request_type, # type: RequestType - host, # type: str - location, # type: str - data=None, # type: Dict[str, Any] - user_agent='weechat-matrix/{version}'.format( - version="0.1") # type: str - ): - # type: (...) -> None - user_agent = 'User-Agent: {agent}'.format(agent=user_agent) - host_header = 'Host: {host}'.format(host=host) - keepalive = "Connection: keep-alive" - request_list = [] # type: List[str] - accept_header = 'Accept: */*' # type: str - end_separator = '\r\n' # type: str - payload = "" # type: str - # yapf: enable - - if request_type == RequestType.GET: - get = 'GET {location} HTTP/1.1'.format(location=location) - request_list = [ - get, host_header, user_agent, keepalive, accept_header, - end_separator - ] - - elif (request_type == RequestType.POST or - request_type == RequestType.PUT): - - json_data = json.dumps(data, separators=(',', ':')) - - if request_type == RequestType.POST: - method = "POST" - else: - method = "PUT" - - request_line = '{method} {location} HTTP/1.1'.format( - method=method, location=location) - - type_header = 'Content-Type: application/x-www-form-urlencoded' - length_header = 'Content-Length: {length}'.format( - length=len(json_data)) - - request_list = [ - request_line, host_header, user_agent, keepalive, - accept_header, length_header, type_header, end_separator - ] - payload = json_data - - request = '\r\n'.join(request_list) - - self.request = request - self.payload = payload diff --git a/matrix/server.py b/matrix/server.py index eb8ca05..0c28206 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -15,55 +15,23 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import unicode_literals -from builtins import str, bytes import os import ssl import socket import time -import datetime import pprint -import json from collections import deque, defaultdict -from http_parser.pyparser import HttpParser -from nio import Client, LoginResponse, SyncRepsponse +from nio import HttpClient, LoginResponse, SyncRepsponse from matrix.plugin_options import Option, DebugType from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, - create_server_buffer, tags_for_message, - server_ts_to_weechat, shorten_sender) + create_server_buffer) from matrix.utf import utf8_decode -from matrix.globals import W, SERVERS, OPTIONS -import matrix.api as API +from matrix.globals import W, SERVERS from .buffer import RoomBuffer, OwnMessage, OwnAction -from matrix.api import ( - MatrixClient, - MatrixSyncMessage, - MatrixLoginMessage, - MatrixKeyUploadMessage, - MatrixKeyQueryMessage, - MatrixToDeviceMessage, - MatrixSendMessage, - MatrixEncryptedMessage, - MatrixKeyClaimMessage -) - -from .events import ( - MatrixSendEvent, - MatrixBacklogEvent, - MatrixErrorEvent, - MatrixEmoteEvent, - MatrixJoinEvent -) - -from matrix.encryption import ( - Olm, - EncryptionError, - OlmTrustError, - encrypt_enabled -) try: FileNotFoundError @@ -108,7 +76,6 @@ class MatrixServer: self.ssl_context = ssl.create_default_context() # type: ssl.SSLContext self.client = None - self.nio_client = Client() # type: Option[Client] self.access_token = None # type: str self.next_batch = None # type: str self.transaction_id = 0 # type: int @@ -117,18 +84,8 @@ class MatrixServer: self.send_fd_hook = None # type: weechat.hook self.send_buffer = b"" # type: bytes - self.current_message = None # type: MatrixMessage self.device_check_timestamp = None - self.http_parser = HttpParser() # type: HttpParser - self.http_buffer = [] # type: List[bytes] - - # Queue of messages we need to send off. - self.send_queue = deque() # type: Deque[MatrixMessage] - - # Queue of messages we send off and are waiting a response for - self.receive_queue = deque() # type: Deque[MatrixMessage] - self.event_queue_timer = None self.event_queue = deque() # type: Deque[RoomInfo] @@ -166,48 +123,6 @@ class MatrixServer: with open(path, 'w') as f: f.write(self.device_id) - def _load_olm(self): - try: - self.olm = Olm.from_session_dir( - self.user, - self.device_id, - self.get_session_path() - ) - message = ("{prefix}matrix: Loaded Olm account for {user} (device:" - "{device})").format(prefix=W.prefix("network"), - user=self.user, - device=self.device_id) - W.prnt("", message) - - except FileNotFoundError: - pass - except EncryptionError as error: - message = ("{prefix}matrix: Error loading Olm" - "account: {error}.").format( - prefix=W.prefix("error"), error=error) - W.prnt("", message) - - @encrypt_enabled - def create_olm(self): - message = ("{prefix}matrix: Creating new Olm identity for " - "{self_color}{user}{ncolor}" - " on {server_color}{server}{ncolor} for device " - "{device}.").format( - prefix=W.prefix("network"), - self_color=W.color("chat_nick_self"), - ncolor=W.color("reset"), - user=self.user_id, - server_color=W.color("chat_server"), - server=self.name, - device=self.device_id) - W.prnt(self.server_buffer, message) - self.olm = Olm(self.user, self.device_id, self.get_session_path()) - - @encrypt_enabled - def store_olm(self): - if self.olm: - self.olm.to_session_dir() - def _create_options(self, config_file): options = [ Option('autoconnect', 'boolean', '', 0, 0, 'off', @@ -243,15 +158,9 @@ class MatrixServer: option.max, option.value, option.value, 0, "", "", "matrix_config_server_change_cb", self.name, "", "") - def reset_parser(self): - self.http_parser = HttpParser() - self.http_buffer = [] - def _change_client(self): host = ':'.join([self.address, str(self.port)]) - user_agent = 'weechat-matrix/{version}'.format(version="0.1") - self.client = MatrixClient(host, user_agent=user_agent) - # self.nio_client = Client() + self.client = HttpClient(host, self.user) def update_option(self, option, option_name): if option_name == "address": @@ -280,13 +189,9 @@ class MatrixServer: value = W.config_string(option) self.user = value self.access_token = "" - self.nio_client.user = value - self.nio_client.access_token = "" - self._load_device_id() - - # if self.device_id: - # self._load_olm() + if self.client: + self.client.user = value elif option_name == "password": value = W.config_string(option) @@ -305,7 +210,6 @@ class MatrixServer: "Adding to queue").format( prefix=W.prefix("error"), t=message.__class__.__name__)) - self.send_queue.append(message) def try_send(self, message): # type: (MatrixServer, bytes) -> bool @@ -362,37 +266,22 @@ class MatrixServer: return True def _abort_send(self): - self.send_queue.appendleft(self.current_message) self.current_message = None self.send_buffer = "" def _finalize_send(self): # type: (MatrixServer) -> None - self.current_message.send_time = time.time() - self.receive_queue.append(self.current_message) - self.send_buffer = b"" - self.current_message = None - def send(self, message): - # type: (MatrixServer, MatrixMessage) -> bool - if self.current_message: - return False - - self.current_message = message - - request = message.request.request - payload = message.request.payload - - bytes_message = bytes(request, 'utf-8') + bytes(payload, 'utf-8') - - self.try_send(bytes_message) + def send(self, data): + # type: (bytes) -> bool + self.try_send(data) return True def reconnect(self): message = ("{prefix}matrix: reconnecting to server..." - ).format(prefix=W.prefix("network")) + ).format(prefix=W.prefix("network")) server_buffer_prnt(self, message) @@ -437,8 +326,6 @@ class MatrixServer: self.socket = None self.connected = False self.access_token = "" - self.send_queue.clear() - self.receive_queue.clear() self.send_buffer = b"" self.current_message = None @@ -500,124 +387,13 @@ class MatrixServer: return True def sync(self): - limit = None if self.next_batch else OPTIONS.sync_limit - message = MatrixSyncMessage(self.client, self.next_batch, limit) - self.send_queue.append(message) - - def _send_unencrypted_message(self, room_id, formatted_data): - message = MatrixSendMessage( - self.client, room_id=room_id, formatted_message=formatted_data) - self.send_or_queue(message) - - def send_room_message( - self, - room, - formatted_data, - already_claimed=False - ): - # type: (str, Formatted) -> None - if not room.encrypted: - self._send_unencrypted_message(room.room_id, formatted_data) - return - - # TODO don't send messages unless all the devices are verified - missing = self.olm.get_missing_sessions(room.users.keys()) - - if missing and not already_claimed: - W.prnt("", "{prefix}matrix: Olm session missing for room, can't" - " encrypt message.") - W.prnt("", pprint.pformat(missing)) - self.encryption_queue[room.room_id].append(formatted_data) - message = MatrixKeyClaimMessage(self.client, room.room_id, missing) - self.send_or_queue(message) - return - - body = {"msgtype": "m.text", "body": formatted_data.to_plain()} - - if formatted_data.is_formatted(): - body["format"] = "org.matrix.custom.html" - body["formatted_body"] = formatted_data.to_html() - - plaintext_dict = { - "type": "m.room.message", - "content": body - } - - W.prnt("", "matrix: Encrypting message") - - try: - payload_dict, to_device_dict = self.olm.group_encrypt( - room.room_id, - plaintext_dict, - self.user_id, - room.users.keys() - ) - - if to_device_dict: - W.prnt("", "matrix: Megolm session missing for room.") - message = MatrixToDeviceMessage(self.client, to_device_dict) - self.send_queue.append(message) - - message = MatrixEncryptedMessage( - self.client, - room.room_id, - formatted_data, - payload_dict - ) - - self.send_queue.append(message) - - except OlmTrustError: - m = ("{prefix}matrix: Untrusted devices found in room, " - "verification is needed before sending a message").format( - prefix=W.prefix("error")) - W.prnt(self.server_buffer, m) - return - - @encrypt_enabled - def upload_keys(self, device_keys=False, one_time_keys=False): - keys = self.olm.account.identity_keys if device_keys else None - - one_time_keys = (self.olm.account.one_time_keys["curve25519"] if - one_time_keys else None) - - message = MatrixKeyUploadMessage(self.client, self.user_id, - self.device_id, self.olm, - keys, one_time_keys) - self.send_queue.append(message) - - @encrypt_enabled - def check_one_time_keys(self, key_count): - max_keys = self.olm.account.max_one_time_keys - - key_count = (max_keys / 2) - key_count - - if key_count <= 0: - return - - self.olm.account.generate_one_time_keys(key_count) - self.upload_keys(device_keys=False, one_time_keys=True) - - @encrypt_enabled - def query_keys(self): - users = [] - - for room_buffer in self.room_buffers.values(): - if not room_buffer.room.encrypted: - continue - users += list(room_buffer.room.users) - - if not users: - return - - message = MatrixKeyQueryMessage(self.client, users) - self.send_queue.append(message) + request = self.client.sync() + self.send_or_queue(request) def login(self): - # type: (MatrixServer) -> None - message = MatrixLoginMessage(self.client, self.user, self.password, - self.device_name, self.device_id) - self.send_or_queue(message) + # type: () -> None + request = self.client.login(self.password) + self.send_or_queue(request) msg = "{prefix}matrix: Logging in...".format(prefix=W.prefix("network")) @@ -698,92 +474,18 @@ class MatrixServer: # self.check_one_time_keys(response.one_time_key_count) # self.handle_events() - def handle_matrix_response(self, response): - if isinstance(response, MatrixSendEvent): - room_buffer = self.find_room_from_id(response.room_id) - self.handle_own_messages(room_buffer, response.message) - - elif isinstance(response, MatrixBacklogEvent): - room_buffer = self.find_room_from_id(response.room_id) - room_buffer.handle_backlog(response.events) - W.bar_item_update("buffer_modes") - - elif isinstance(response, MatrixErrorEvent): - self._handle_erorr_response(response) - - def nio_receive(self): - response = self.nio_client.next_response() + def handle_response(self, response): + # type: (MatrixMessage) -> None if isinstance(response, LoginResponse): self._handle_login(response) elif isinstance(response, SyncRepsponse): self._handle_sync(response) - def nio_parse_response(self, response): - if isinstance(response, MatrixLoginMessage): - self.nio_client.receive("login", response.response.body) - elif isinstance(response, MatrixSyncMessage): - self.nio_client.receive("sync", response.response.body) - - self.nio_receive() - - return - - def handle_response(self, message): - # type: (MatrixMessage) -> None - - assert message.response - - if ('content-type' in message.response.headers and - message.response.headers['content-type'] == 'application/json'): - - if isinstance(message, (MatrixLoginMessage, MatrixSyncMessage)): - self.nio_parse_response(message) - - else: - ret, error = message.decode_body(self) - - if not ret: - message = ("{prefix}matrix: Error decoding json response" - " from server: {error}").format( - prefix=W.prefix("error"), error=error) - W.prnt(self.server_buffer, message) - return - - event = message.event - self.handle_matrix_response(event) - else: - status_code = message.response.status - if status_code == 504: - if isinstance(message, API.MatrixSyncMessage): - self.sync() - else: - self._print_message_error(message) - else: - self._print_message_error(message) - - creation_date = datetime.datetime.fromtimestamp(message.creation_time) - done_time = time.time() - info_message = ( - "Message of type {t} created at {c}." - "\nMessage lifetime information:" - "\n Send delay: {s} ms" - "\n Receive delay: {r} ms" - "\n Handling time: {h} ms" - "\n Total time: {total} ms").format( - t=message.__class__.__name__, - c=creation_date, - s=(message.send_time - message.creation_time) * 1000, - r=(message.receive_time - message.send_time) * 1000, - h=(done_time - message.receive_time) * 1000, - total=(done_time - message.creation_time) * 1000, - ) - prnt_debug(DebugType.TIMING, self, info_message) - return def create_room_buffer(self, room_id): - room = self.nio_client.rooms[room_id] + room = self.client.rooms[room_id] buf = RoomBuffer(room, self.name) # TODO this should turned into a propper class self.room_buffers[room_id] = buf @@ -867,31 +569,31 @@ def matrix_timer_cb(server_name, remaining_calls): if not server.connected: return W.WEECHAT_RC_OK - # check lag, disconnect if it's too big - if server.receive_queue: - message = server.receive_queue.popleft() - server.lag = (current_time - message.send_time) * 1000 - server.receive_queue.appendleft(message) - server.lag_done = False - W.bar_item_update("lag") + # # check lag, disconnect if it's too big + # if server.receive_queue: + # message = server.receive_queue.popleft() + # server.lag = (current_time - message.send_time) * 1000 + # server.receive_queue.appendleft(message) + # server.lag_done = False + # W.bar_item_update("lag") - # TODO print out message, make timeout configurable - if server.lag > 300000: - server.disconnect() - return W.WEECHAT_RC_OK + # # TODO print out message, make timeout configurable + # if server.lag > 300000: + # server.disconnect() + # return W.WEECHAT_RC_OK - while server.send_queue: - message = server.send_queue.popleft() - prnt_debug( - DebugType.MESSAGING, - server, ("Timer hook found message of type {t} in queue. Sending " - "out.".format(t=message.__class__.__name__))) + # while server.send_queue: + # message = server.send_queue.popleft() + # prnt_debug( + # DebugType.MESSAGING, + # server, ("Timer hook found message of type {t} in queue. Sending " + # "out.".format(t=message.__class__.__name__))) - if not server.send(message): - # We got an error while sending the last message return the message - # to the queue and exit the loop - server.send_queue.appendleft(message) - break + # if not server.send(message): + # # We got an error while sending the last message return the message + # # to the queue and exit the loop + # server.send_queue.appendleft(message) + # break if not server.next_batch: return W.WEECHAT_RC_OK @@ -904,7 +606,6 @@ def matrix_timer_cb(server_name, remaining_calls): "{prefix}matrix: Querying user devices.".format( prefix=W.prefix("networ"))) - server.query_keys() server.device_check_timestamp = current_time return W.WEECHAT_RC_OK From 1ff0818d462515f319c99f310a61b6e807a0f342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 20 Jul 2018 19:14:32 +0200 Subject: [PATCH 065/269] matrix: Switch to the nio http client. --- main.py | 4 ++-- matrix/server.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 781d643..c855d84 100644 --- a/main.py +++ b/main.py @@ -386,8 +386,8 @@ def room_close_cb(data, buffer): def matrix_unload_cb(): matrix_config_free(matrix.globals.CONFIG) - for server in SERVERS.values(): - server.store_olm() + # for server in SERVERS.values(): + # server.store_olm() W.prnt("", "unloading") return W.WEECHAT_RC_OK diff --git a/matrix/server.py b/matrix/server.py index 0c28206..642baac 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -329,7 +329,7 @@ class MatrixServer: self.send_buffer = b"" self.current_message = None - self.reset_parser() + self.client.disconnect() self.lag = 0 W.bar_item_update("lag") From 7a12f047f74e3567f0435d2596080d9311181fd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 21 Jul 2018 15:15:48 +0200 Subject: [PATCH 066/269] server: Add http2 support. --- main.py | 26 ++++++++++++++++++--- matrix/server.py | 61 +++++++++++++++++++++++++++--------------------- 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/main.py b/main.py index c855d84..98905c2 100644 --- a/main.py +++ b/main.py @@ -272,7 +272,7 @@ def receive_cb(server_name, file_descriptor): server_buffer_prnt( server, ("{prefix}matrix: disconnecting from server..." - ).format(prefix=W.prefix("network"))) + ).format(prefix=W.prefix("network"))) server.disconnect() @@ -285,7 +285,7 @@ def receive_cb(server_name, file_descriptor): prefix=W.prefix("network"))) server_buffer_prnt( server, ("{prefix}matrix: disconnecting from server..." - ).format(prefix=W.prefix("network"))) + ).format(prefix=W.prefix("network"))) server.disconnect() break @@ -294,6 +294,12 @@ def receive_cb(server_name, file_descriptor): response = server.client.next_response() + # Check if we need to send some data back + data_to_send = server.client.data_to_send() + + if data_to_send: + server.send(data_to_send) + if response: server.handle_response(response) break @@ -314,7 +320,21 @@ def finalize_connection(server): server.fd_hook = hook server.connected = True server.connecting = False - server.client.connect(TransportType.HTTP) + + negotiated_protocol = server.socket.selected_alpn_protocol() + + if negotiated_protocol is None: + negotiated_protocol = server.socket.selected_npn_protocol() + + transport_type = None + + if negotiated_protocol == "http/1.1": + transport_type = TransportType.HTTP + elif negotiated_protocol == "h2": + transport_type = TransportType.HTTP2 + + data = server.client.connect(transport_type) + server.send(data) server.login() diff --git a/matrix/server.py b/matrix/server.py index 642baac..18b0927 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -75,6 +75,14 @@ class MatrixServer: self.socket = None # type: ssl.SSLSocket self.ssl_context = ssl.create_default_context() # type: ssl.SSLContext + # Enable http2 negotiation on the ssl context. + self.ssl_context.set_alpn_protocols(["h2", "http/1.1"]) + + try: + self.ssl_context.set_npn_protocols(["h2", "http/1.1"]) + except NotImplementedError: + pass + self.client = None self.access_token = None # type: str self.next_batch = None # type: str @@ -86,6 +94,8 @@ class MatrixServer: self.send_buffer = b"" # type: bytes self.device_check_timestamp = None + self.send_queue = deque() + self.event_queue_timer = None self.event_queue = deque() # type: Deque[RoomInfo] @@ -202,14 +212,10 @@ class MatrixServer: else: pass - def send_or_queue(self, message): - # type: (MatrixServer, MatrixMessage) -> None - if not self.send(message): - prnt_debug(DebugType.MESSAGING, self, - ("{prefix} Failed sending message of type {t}. " - "Adding to queue").format( - prefix=W.prefix("error"), - t=message.__class__.__name__)) + def send_or_queue(self, request): + # type: (bytes) -> None + if not self.send(request): + self.send_queue.append(request) def try_send(self, message): # type: (MatrixServer, bytes) -> bool @@ -365,7 +371,7 @@ class MatrixServer: create_server_buffer(self) if not self.timer_hook: - self.timer_hook = W.hook_timer(1 * 1000, 0, 0, "matrix_timer_cb", + self.timer_hook = W.hook_timer(2 * 1000, 0, 0, "matrix_timer_cb", self.name) ssl_message = " (SSL)" if self.ssl_context.check_hostname else "" @@ -388,7 +394,7 @@ class MatrixServer: def sync(self): request = self.client.sync() - self.send_or_queue(request) + self.send_queue.append(request) def login(self): # type: () -> None @@ -473,6 +479,7 @@ class MatrixServer: self.next_batch = response.next_batch # self.check_one_time_keys(response.one_time_key_count) # self.handle_events() + self.sync() def handle_response(self, response): # type: (MatrixMessage) -> None @@ -582,31 +589,31 @@ def matrix_timer_cb(server_name, remaining_calls): # server.disconnect() # return W.WEECHAT_RC_OK - # while server.send_queue: - # message = server.send_queue.popleft() - # prnt_debug( - # DebugType.MESSAGING, - # server, ("Timer hook found message of type {t} in queue. Sending " - # "out.".format(t=message.__class__.__name__))) + while server.send_queue: + message = server.send_queue.popleft() + prnt_debug( + DebugType.MESSAGING, + server, ("Timer hook found message of type {t} in queue. Sending " + "out.".format(t=message.__class__.__name__))) - # if not server.send(message): - # # We got an error while sending the last message return the message - # # to the queue and exit the loop - # server.send_queue.appendleft(message) - # break + if not server.send(message): + # We got an error while sending the last message return the message + # to the queue and exit the loop + server.send_queue.appendleft(message) + break if not server.next_batch: return W.WEECHAT_RC_OK # check for new devices by users in encrypted rooms periodically - if (not server.device_check_timestamp or - current_time - server.device_check_timestamp > 600): + # if (not server.device_check_timestamp or + # current_time - server.device_check_timestamp > 600): - W.prnt(server.server_buffer, - "{prefix}matrix: Querying user devices.".format( - prefix=W.prefix("networ"))) + # W.prnt(server.server_buffer, + # "{prefix}matrix: Querying user devices.".format( + # prefix=W.prefix("networ"))) - server.device_check_timestamp = current_time + # server.device_check_timestamp = current_time return W.WEECHAT_RC_OK From 5990704c0d6fe7e2e439e1642a6f0ffe8a35a888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 21 Jul 2018 15:16:02 +0200 Subject: [PATCH 067/269] buffer: Handle the case where a user left the room in the timeline. --- matrix/buffer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 6aa2973..4f90886 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -698,7 +698,12 @@ class RoomBuffer(object): def handle_membership_events(self, event, is_state): def join(event, date, is_state): - user = self.room.users[event.sender] + try: + user = self.room.users[event.sender] + except KeyError: + # No user found, he must have left already in an event that is + # yet to come, so do nothing + return short_name = shorten_sender(user.user_id) From 3785cd0beec742951bfd0aebf89f354174b968cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 21 Jul 2018 19:54:36 +0200 Subject: [PATCH 068/269] matrix: Enable nio logger. --- main.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/main.py b/main.py index 98905c2..7cf457f 100644 --- a/main.py +++ b/main.py @@ -31,7 +31,10 @@ from future.utils import bytes_to_native_str as n # pylint: disable=unused-import from typing import (List, Set, Dict, Tuple, Text, Optional, AnyStr, Deque, Any) +import logbook +from logbook import Logger, StderrHandler, StreamHandler +import nio from nio import TransportType from matrix.colors import Formatted @@ -92,6 +95,9 @@ WEECHAT_SCRIPT_LICENSE = "ISC" # type: str # yapf: enable +logger = Logger("matrix-cli") + + def print_certificate_info(buff, sock, cert): cert_pem = ssl.DER_cert_to_PEM_cert(sock.getpeercert(True)) @@ -419,6 +425,23 @@ def autoconnect(servers): server.connect() +class WeechatHandler(StreamHandler): + def __init__(self, level=logbook.NOTSET, format_string=None, filter=None, + bubble=False): + StreamHandler.__init__( + self, + object(), + level, + format_string, + None, + filter, + bubble + ) + + def write(self, item): + W.prnt("", item) + + if __name__ == "__main__": if W.register(WEECHAT_SCRIPT_NAME, WEECHAT_SCRIPT_AUTHOR, WEECHAT_SCRIPT_VERSION, WEECHAT_SCRIPT_LICENSE, @@ -429,6 +452,11 @@ if __name__ == "__main__": "directory").format(prefix=W.prefix("error")) W.prnt("", message) + handler = WeechatHandler() + handler.format_string = "{record.channel}: {record.message}" + handler.push_application() + nio.http.logger.level = logbook.DEBUG + # TODO if this fails we should abort and unload the script. matrix.globals.CONFIG = W.config_new("matrix", "matrix_config_reload_cb", "") From 59b034737e575d3d1c7c1d65d2572a94cf0a8433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 21 Jul 2018 19:55:03 +0200 Subject: [PATCH 069/269] buffer: Handle an empty user part of an user id. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 4f90886..51b739a 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -708,7 +708,7 @@ class RoomBuffer(object): short_name = shorten_sender(user.user_id) # TODO make this configurable - if short_name in self.displayed_nicks.values(): + if not short_name or short_name in self.displayed_nicks.values(): # Use the full user id, but don't include the @ nick = event.sender[1:] else: From 4b67dc015b2f032023f61bb8226ce79938649e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 24 Jul 2018 11:05:03 +0200 Subject: [PATCH 070/269] matrix: Make logging configurable. --- main.py | 1 - matrix/config.py | 56 +++++++++++++++++++++++++++++++++++++--- matrix/plugin_options.py | 10 +++++-- 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 7cf457f..1b9066d 100644 --- a/main.py +++ b/main.py @@ -455,7 +455,6 @@ if __name__ == "__main__": handler = WeechatHandler() handler.format_string = "{record.channel}: {record.message}" handler.push_application() - nio.http.logger.level = logbook.DEBUG # TODO if this fails we should abort and unload the script. matrix.globals.CONFIG = W.config_new("matrix", diff --git a/matrix/config.py b/matrix/config.py index 4bc8850..e259d75 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -16,6 +16,9 @@ from __future__ import unicode_literals +import nio +import logbook + from matrix.plugin_options import (Option, RedactType, ServerBufferType) import matrix.globals @@ -30,6 +33,19 @@ def matrix_config_reload_cb(data, config_file): return W.WEECHAT_RC_OK +def change_log_level(category, level): + if category == "all": + nio.logger_group.level = level + elif category == "http": + nio.http.logger.level = level + elif category == "client": + nio.client.logger.level = level + elif category == "events": + nio.events.logger.level = level + elif category == "responses": + nio.responses.logger.level = level + + @utf8_decode def matrix_config_change_cb(data, option): option_name = key_from_value(OPTIONS.options, option) @@ -49,6 +65,36 @@ def matrix_config_change_cb(data, option): elif option_name == "max_backlog_sync_events": OPTIONS.backlog_limit = W.config_integer(option) + elif option_name == "debug_level": + value = W.config_integer(option) + if value == 0: + OPTIONS.debug_level = logbook.ERROR + elif value == 1: + OPTIONS.debug_level = logbook.WARNING + elif value == 2: + OPTIONS.debug_level = logbook.INFO + elif value == 3: + OPTIONS.debug_level = logbook.DEBUG + + change_log_level(OPTIONS.debug_category, OPTIONS.debug_level) + + elif option_name == "debug_category": + value = W.config_integer(option) + change_log_level(OPTIONS.debug_category, logbook.ERROR) + + if value == 0: + OPTIONS.debug_category = "all" + elif value == 1: + OPTIONS.debug_category = "http" + elif value == 2: + OPTIONS.debug_category = "client" + elif value == 3: + OPTIONS.debug_category = "events" + elif value == 4: + OPTIONS.debug_category = "responses" + + change_log_level(OPTIONS.debug_category, OPTIONS.debug_level) + elif option_name == "fetch_backlog_on_pgup": OPTIONS.enable_backlog = W.config_boolean(option) @@ -80,7 +126,11 @@ def matrix_config_init(config_file): Option("max_backlog_sync_events", "integer", "", 1, 100, "10", ("How many events to fetch during backlog fetching")), Option("fetch_backlog_on_pgup", "boolean", "", 0, 0, "on", - ("Fetch messages in the backlog on a window page up event")) + ("Fetch messages in the backlog on a window page up event")), + Option("debug_level", "integer", "error|warn|info|debug", 0, 0, + "off", "Enable network protocol debugging."), + Option("debug_category", "integer", "all|http|client|events|responses", + 0, 0, "all", "Debugging category") ] def add_global_options(section, options): @@ -101,8 +151,8 @@ def matrix_config_init(config_file): add_global_options(section, look_options) - section = W.config_new_section(config_file, "network", 0, 0, "", "", "", "", - "", "", "", "", "", "") + section = W.config_new_section(config_file, "network", 0, 0, "", "", "", + "", "", "", "", "", "", "") add_global_options(section, network_options) diff --git a/matrix/plugin_options.py b/matrix/plugin_options.py index fc0ecf5..a872a85 100644 --- a/matrix/plugin_options.py +++ b/matrix/plugin_options.py @@ -15,6 +15,9 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import unicode_literals + +import logbook + from collections import namedtuple from enum import Enum, unique @@ -49,7 +52,8 @@ class PluginOptions: def __init__(self): self.redaction_type = RedactType.STRIKETHROUGH # type: RedactType - self.look_server_buf = ServerBufferType.MERGE_CORE # type: ServerBufferType + self.look_server_buf = ServerBufferType.MERGE_CORE \ + # type: ServerBufferType self.sync_limit = 30 # type: int self.backlog_limit = 10 # type: int @@ -59,4 +63,6 @@ class PluginOptions: self.redaction_comp_len = 50 # type: int self.options = dict() # type: Dict[str, weechat.config_option] - self.debug = [] # type: List[DebugType] + self.debug = [] + self.debug_level = logbook.ERROR + self.debug_category = "all" From 32df9230ea15ce62eb43e3e575c2eabbba1d695b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 24 Jul 2018 11:05:29 +0200 Subject: [PATCH 071/269] server: Handle malformed server responses. --- main.py | 8 ++++++-- matrix/server.py | 10 +++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 1b9066d..381bfd1 100644 --- a/main.py +++ b/main.py @@ -35,7 +35,7 @@ import logbook from logbook import Logger, StderrHandler, StreamHandler import nio -from nio import TransportType +from nio import TransportType, RemoteTransportError from matrix.colors import Formatted from matrix.utf import utf8_decode @@ -296,7 +296,11 @@ def receive_cb(server_name, file_descriptor): server.disconnect() break - server.client.receive(data) + try: + server.client.receive(data) + except RemoteTransportError as e: + server.error(str(e)) + server.disconnect() response = server.client.next_response() diff --git a/matrix/server.py b/matrix/server.py index 18b0927..dd45754 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -30,7 +30,7 @@ from matrix.plugin_options import Option, DebugType from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, create_server_buffer) from matrix.utf import utf8_decode -from matrix.globals import W, SERVERS +from matrix.globals import W, SERVERS, SCRIPT_NAME from .buffer import RoomBuffer, OwnMessage, OwnAction try: @@ -279,6 +279,14 @@ class MatrixServer: # type: (MatrixServer) -> None self.send_buffer = b"" + def error(self, message): + buf = "" + if self.server_buffer: + buf = self.server_buffer + + msg = "{prefix}{}: {}".format(SCRIPT_NAME, message) + W.prnt(buf, msg) + def send(self, data): # type: (bytes) -> bool self.try_send(data) From df61dba644466ed8988acb1c8f872c8daa54acf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 24 Jul 2018 11:39:05 +0200 Subject: [PATCH 072/269] matrix: Add option to log into a separate debug buffer. --- main.py | 18 ++++++++++++++++-- matrix/config.py | 7 ++++++- matrix/plugin_options.py | 2 ++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 381bfd1..1b0aa1a 100644 --- a/main.py +++ b/main.py @@ -84,7 +84,7 @@ from matrix.config import (matrix_config_init, matrix_config_read, import matrix.globals -from matrix.globals import W, SERVERS, SCRIPT_NAME +from matrix.globals import W, SERVERS, SCRIPT_NAME, OPTIONS # yapf: disable WEECHAT_SCRIPT_NAME = SCRIPT_NAME @@ -429,6 +429,11 @@ def autoconnect(servers): server.connect() +def debug_buffer_close_cb(data, buffer): + OPTIONS.debug_buffer_ptr = "" + return W.WEECHAT_RC_OK + + class WeechatHandler(StreamHandler): def __init__(self, level=logbook.NOTSET, format_string=None, filter=None, bubble=False): @@ -443,7 +448,16 @@ class WeechatHandler(StreamHandler): ) def write(self, item): - W.prnt("", item) + buf = "" + + if OPTIONS.debug_buffer: + if not OPTIONS.debug_buffer_ptr: + OPTIONS.debug_buffer_ptr = W.buffer_new( + "Matrix Debug", "", "", "debug_buffer_close_cb", "") + + buf = OPTIONS.debug_buffer_ptr + + W.prnt(buf, item) if __name__ == "__main__": diff --git a/matrix/config.py b/matrix/config.py index e259d75..7c49e66 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -95,6 +95,9 @@ def matrix_config_change_cb(data, option): change_log_level(OPTIONS.debug_category, OPTIONS.debug_level) + elif option_name == "debug_buffer": + OPTIONS.debug_buffer = W.config_boolean(option) + elif option_name == "fetch_backlog_on_pgup": OPTIONS.enable_backlog = W.config_boolean(option) @@ -130,7 +133,9 @@ def matrix_config_init(config_file): Option("debug_level", "integer", "error|warn|info|debug", 0, 0, "off", "Enable network protocol debugging."), Option("debug_category", "integer", "all|http|client|events|responses", - 0, 0, "all", "Debugging category") + 0, 0, "all", "Debugging category"), + Option("debug_buffer", "boolean", "", 0, 0, "off", + ("Use a separate buffer for debug logs.")), ] def add_global_options(section, options): diff --git a/matrix/plugin_options.py b/matrix/plugin_options.py index a872a85..7fbd9a6 100644 --- a/matrix/plugin_options.py +++ b/matrix/plugin_options.py @@ -66,3 +66,5 @@ class PluginOptions: self.debug = [] self.debug_level = logbook.ERROR self.debug_category = "all" + self.debug_buffer = False + self.debug_buffer_ptr = "" From 9e6bd0c246ffef6fd68557aa357d774bdb17893e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 26 Jul 2018 11:33:05 +0200 Subject: [PATCH 073/269] server: Handle http error responses. --- main.py | 4 ++-- matrix/server.py | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 1b0aa1a..c8482da 100644 --- a/main.py +++ b/main.py @@ -35,7 +35,7 @@ import logbook from logbook import Logger, StderrHandler, StreamHandler import nio -from nio import TransportType, RemoteTransportError +from nio import TransportType, RemoteTransportError, RemoteProtocolError from matrix.colors import Formatted from matrix.utf import utf8_decode @@ -298,7 +298,7 @@ def receive_cb(server_name, file_descriptor): try: server.client.receive(data) - except RemoteTransportError as e: + except (RemoteTransportError, RemoteProtocolError) as e: server.error(str(e)) server.disconnect() diff --git a/matrix/server.py b/matrix/server.py index dd45754..d6c4ed6 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -24,7 +24,13 @@ import pprint from collections import deque, defaultdict -from nio import HttpClient, LoginResponse, SyncRepsponse +from nio import ( + HttpClient, + LoginResponse, + SyncRepsponse, + TransportResponse, + LocalProtocolError +) from matrix.plugin_options import Option, DebugType from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, @@ -284,7 +290,7 @@ class MatrixServer: if self.server_buffer: buf = self.server_buffer - msg = "{prefix}{}: {}".format(SCRIPT_NAME, message) + msg = "{}{}: {}".format(W.prefix("network"), SCRIPT_NAME, message) W.prnt(buf, msg) def send(self, data): @@ -343,7 +349,11 @@ class MatrixServer: self.send_buffer = b"" self.current_message = None - self.client.disconnect() + + try: + self.client.disconnect() + except LocalProtocolError: + pass self.lag = 0 W.bar_item_update("lag") @@ -491,8 +501,12 @@ class MatrixServer: def handle_response(self, response): # type: (MatrixMessage) -> None + if isinstance(response, TransportResponse): + self.error("Error in response, code: {}".format( + response.status_code)) + self.disconnect() - if isinstance(response, LoginResponse): + elif isinstance(response, LoginResponse): self._handle_login(response) elif isinstance(response, SyncRepsponse): self._handle_sync(response) From 35e69786228a2d5776b4764e956f7d9dfb3e46fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 31 Jul 2018 11:57:27 +0200 Subject: [PATCH 074/269] main: Handle socket errors during the SSL handshake. --- main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index c8482da..8759f1a 100644 --- a/main.py +++ b/main.py @@ -239,8 +239,11 @@ def try_ssl_handshake(server): return False - except ssl.SSLError as error: - str_error = error.reason if error.reason else "Unknown error" + except (ssl.SSLError, socket.error) as error: + try: + str_error = error.reason if error.reason else "Unknown error" + except AttributeError: + str_error = str(error) message = ("{prefix}Error while doing SSL handshake" ": {error}").format( @@ -250,7 +253,7 @@ def try_ssl_handshake(server): server_buffer_prnt( server, ("{prefix}matrix: disconnecting from server..." - ).format(prefix=W.prefix("network"))) + ).format(prefix=W.prefix("network"))) server.disconnect() return False From 6cb9be55784b23f22cc50e2b60b51f6a1f4c289c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 31 Jul 2018 12:02:48 +0200 Subject: [PATCH 075/269] server: Add the ability to send messages. --- matrix/buffer.py | 9 ++++--- matrix/server.py | 65 ++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 51b739a..0e529af 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -50,6 +50,7 @@ OwnMessage = NamedTuple("OwnMessage", [ ("sender", str), ("age", int), ("event_id", str), + ("room_id", str), ("formatted_message", Formatted) ]) @@ -73,7 +74,7 @@ def room_buffer_input_cb(server_name, buffer, input_data): formatted_data = Formatted.from_input_line(input_data) - server.send_room_message(room_buffer.room, formatted_data) + server.room_send_text(room_buffer, formatted_data) return W.WEECHAT_RC_OK @@ -398,9 +399,9 @@ class WeechatChannelBuffer(object): # type: (str) -> None """ Print an error to the room buffer """ message = "{prefix}{script}: {message}".format( - W.prefix("error"), - SCRIPT_NAME, - string + prefix=W.prefix("error"), + script=SCRIPT_NAME, + message=string ) self._print(message) diff --git a/matrix/server.py b/matrix/server.py index d6c4ed6..f450af3 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -28,6 +28,7 @@ from nio import ( HttpClient, LoginResponse, SyncRepsponse, + RoomSendResponse, TransportResponse, LocalProtocolError ) @@ -72,13 +73,14 @@ class MatrixServer: self.timer_hook = None # type: weechat.hook self.numeric_address = "" # type: str - self.autoconnect = False # type: bool - self.connected = False # type: bool - self.connecting = False # type: bool - self.proxy = None # type: str - self.reconnect_delay = 0 # type: int - self.reconnect_time = None # type: float - self.socket = None # type: ssl.SSLSocket + self.autoconnect = False # type: bool + self.connected = False # type: bool + self.connecting = False # type: bool + self.proxy = None # type: str + self.reconnect_delay = 0 # type: int + self.reconnect_time = None # type: float + self.sync_time = None # type: Optional[float] + self.socket = None # type: ssl.SSLSocket self.ssl_context = ssl.create_default_context() # type: ssl.SSLContext # Enable http2 negotiation on the ssl context. @@ -101,6 +103,7 @@ class MatrixServer: self.device_check_timestamp = None self.send_queue = deque() + self.own_message_queue = dict() # type: Dict[OwnMessage] self.event_queue_timer = None self.event_queue = deque() # type: Deque[RoomInfo] @@ -254,7 +257,7 @@ class MatrixServer: server_buffer_prnt(self, error_message) server_buffer_prnt( self, ("{prefix}matrix: disconnecting from server..." - ).format(prefix=W.prefix("network"))) + ).format(prefix=W.prefix("network"))) self.disconnect() return False @@ -410,19 +413,50 @@ class MatrixServer: return True + def schedule_sync(self): + self.sync_time = time.time() + def sync(self): - request = self.client.sync() - self.send_queue.append(request) + self.sync_time = None + _, request = self.client.sync() + self.send_or_queue(request) def login(self): # type: () -> None - request = self.client.login(self.password) + _, request = self.client.login(self.password) self.send_or_queue(request) - msg = "{prefix}matrix: Logging in...".format(prefix=W.prefix("network")) + msg = "{prefix}matrix: Logging in...".format( + prefix=W.prefix("network") + ) W.prnt(self.server_buffer, msg) + def room_send_text(self, room_buffer, formatted): + # type: (RoomBuffer, Formatted) -> None + own_message = OwnMessage( + self.user_id, + 0, + "", + room_buffer.room.room_id, + formatted + ) + + body = {"msgtype": "m.text", "body": formatted.to_plain()} + + if formatted.is_formatted(): + body["format"] = "org.matrix.custom.html" + body["formatted_body"] = formatted.to_html() + + uuid, request = self.client.room_send( + room_buffer.room.room_id, + "m.room.message", + body + ) + + self.own_message_queue[uuid] = own_message + self.send_or_queue(request) + def _print_message_error(self, message): server_buffer_prnt(self, ("{prefix}Unhandled {status_code} error, please " @@ -434,7 +468,10 @@ class MatrixServer: server_buffer_prnt(self, pprint.pformat(message.request.payload)) server_buffer_prnt(self, pprint.pformat(message.response.body)) - def handle_own_messages(self, room_buffer, message): + def handle_own_messages(self, response): + message = self.own_message_queue.pop(response.uuid) + room_buffer = self.room_buffers[message.room_id] + if isinstance(message, OwnAction): room_buffer.self_action(message) return @@ -510,6 +547,8 @@ class MatrixServer: self._handle_login(response) elif isinstance(response, SyncRepsponse): self._handle_sync(response) + elif isinstance(response, RoomSendResponse): + self.handle_own_messages(response) return From 0c8e732ad028265aaeffe12250551762e8d824b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 31 Jul 2018 12:03:05 +0200 Subject: [PATCH 076/269] buffer: Fix room join event handling. --- matrix/buffer.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 0e529af..d3c91c7 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -700,7 +700,7 @@ class RoomBuffer(object): def handle_membership_events(self, event, is_state): def join(event, date, is_state): try: - user = self.room.users[event.sender] + user = self.room.users[event.state_key] except KeyError: # No user found, he must have left already in an event that is # yet to come, so do nothing @@ -731,16 +731,11 @@ class RoomBuffer(object): date = server_ts_to_weechat(event.server_timestamp) if event.content["membership"] == "join": - if event.prev_content and "membership" in event.prev_content: - if (event.prev_content["membership"] != "join"): - join(event, date, is_state) - else: - # TODO print out profile changes - return - else: - # No previous content for this user in this room, so he just - # joined. + if event.state_key not in self.displayed_nicks: join(event, date, is_state) + else: + # TODO print out profile changes + return elif event.content["membership"] == "leave": nick = self.find_nick(event.state_key) From 7a8ce17d8ae2a345ba6e0473c6e393f73937ff2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 31 Jul 2018 12:03:23 +0200 Subject: [PATCH 077/269] server: Change the way syncs are scheduled. --- matrix/server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index f450af3..100c81b 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -526,15 +526,12 @@ class MatrixServer: def _handle_sync(self, response): # we got the same batch again, nothing to do if self.next_batch == response.next_batch: - self.sync() + self.schedule_sync() return self._handle_room_info(response) - # self._queue_joined_info(response) self.next_batch = response.next_batch - # self.check_one_time_keys(response.one_time_key_count) - # self.handle_events() - self.sync() + self.schedule_sync() def handle_response(self, response): # type: (MatrixMessage) -> None @@ -650,6 +647,9 @@ def matrix_timer_cb(server_name, remaining_calls): # server.disconnect() # return W.WEECHAT_RC_OK + if server.sync_time and current_time > (server.sync_time + 2): + server.sync() + while server.send_queue: message = server.send_queue.popleft() prnt_debug( From 2e28e03e0c2e40ca686170ede08e6d4f733948f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 31 Jul 2018 17:12:16 +0200 Subject: [PATCH 078/269] server: Enable lag functionality again. --- matrix/server.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 100c81b..db867e6 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -392,7 +392,7 @@ class MatrixServer: create_server_buffer(self) if not self.timer_hook: - self.timer_hook = W.hook_timer(2 * 1000, 0, 0, "matrix_timer_cb", + self.timer_hook = W.hook_timer(1 * 1000, 0, 0, "matrix_timer_cb", self.name) ssl_message = " (SSL)" if self.ssl_context.check_hostname else "" @@ -535,6 +535,10 @@ class MatrixServer: def handle_response(self, response): # type: (MatrixMessage) -> None + self.lag = response.elapsed * 1000 + self.lag_done = True + W.bar_item_update("lag") + if isinstance(response, TransportResponse): self.error("Error in response, code: {}".format( response.status_code)) @@ -634,18 +638,15 @@ def matrix_timer_cb(server_name, remaining_calls): if not server.connected: return W.WEECHAT_RC_OK - # # check lag, disconnect if it's too big - # if server.receive_queue: - # message = server.receive_queue.popleft() - # server.lag = (current_time - message.send_time) * 1000 - # server.receive_queue.appendleft(message) - # server.lag_done = False - # W.bar_item_update("lag") + # check lag, disconnect if it's too big + server.lag = server.client.lag * 1000 + server.lag_done = False + W.bar_item_update("lag") - # # TODO print out message, make timeout configurable - # if server.lag > 300000: - # server.disconnect() - # return W.WEECHAT_RC_OK + # TODO print out message, make timeout configurable + if server.lag > 300000: + server.disconnect() + return W.WEECHAT_RC_OK if server.sync_time and current_time > (server.sync_time + 2): server.sync() From 3b01483cc4706b3720143e50f22b5314c25be942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 3 Aug 2018 12:25:17 +0200 Subject: [PATCH 079/269] server: Use a timeout for syncs. --- matrix/server.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index db867e6..b294e1f 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -416,9 +416,10 @@ class MatrixServer: def schedule_sync(self): self.sync_time = time.time() - def sync(self): + def sync(self, timeout=None): + # type: Optional[int] -> None self.sync_time = None - _, request = self.client.sync() + _, request = self.client.sync(timeout) self.send_or_queue(request) def login(self): @@ -536,6 +537,13 @@ class MatrixServer: def handle_response(self, response): # type: (MatrixMessage) -> None self.lag = response.elapsed * 1000 + + # If the response was a sync response and contained a timeout the + # timeout is expected and should be removed from the lag. + # TODO the timeout isn't a constant + if isinstance(response, SyncRepsponse): + self.lag = max(0, self.lag - (30000)) + self.lag_done = True W.bar_item_update("lag") @@ -649,7 +657,7 @@ def matrix_timer_cb(server_name, remaining_calls): return W.WEECHAT_RC_OK if server.sync_time and current_time > (server.sync_time + 2): - server.sync() + server.sync(30000) while server.send_queue: message = server.send_queue.popleft() From 2a4ec66742720eade8544838350d36a4c5e9e21a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 3 Aug 2018 14:06:27 +0200 Subject: [PATCH 080/269] buffer: Enable handling of unknown events. --- matrix/buffer.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index d3c91c7..c4d9e69 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -42,7 +42,8 @@ from nio import ( RoomAliasEvent, RoomTopicEvent, RoomMessageEmote, - RoomNameEvent + RoomNameEvent, + RoomMessageUnknown ) @@ -896,7 +897,6 @@ class RoomBuffer(object): self.get_event_tags(event) ) - elif isinstance(event, RoomMessageText): nick = self.find_nick(event.sender) formatted = None @@ -942,19 +942,19 @@ class RoomBuffer(object): # self.get_event_tags(event) # ) - # elif isinstance(event, RoomMessageUnknown): - # nick = self.find_nick(event.sender) - # date = server_ts_to_weechat(event.server_timestamp) - # data = ("Unknown message of type {t}, body: {body}").format( - # t=event.message_type, - # body=event.message - # ) - # self.weechat_buffer.message( - # nick, - # data, - # date, - # self.get_event_tags(event) - # ) + elif isinstance(event, RoomMessageUnknown): + nick = self.find_nick(event.sender) + date = server_ts_to_weechat(event.server_timestamp) + data = ("Unknown message of type {t}, body: {body}").format( + t=event.message_type, + body=event.message + ) + self.weechat_buffer.message( + nick, + data, + date, + self.get_event_tags(event) + ) # elif isinstance(event, RoomRedactionEvent): # self._redact_line(event) From 3bd4be2258d2f8cc36bb5951ef1652ecfd3156d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 3 Aug 2018 14:50:23 +0200 Subject: [PATCH 081/269] buffer: Reenable redaction event handling. --- matrix/buffer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index c4d9e69..1cb211d 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -29,7 +29,6 @@ from .utils import ( shorten_sender, server_ts_to_weechat, string_strikethrough, - mxc_to_http ) from .plugin_options import RedactType @@ -43,7 +42,8 @@ from nio import ( RoomTopicEvent, RoomMessageEmote, RoomNameEvent, - RoomMessageUnknown + RoomMessageUnknown, + RedactionEvent ) @@ -774,7 +774,7 @@ class RoomBuffer(object): return False lines = self.weechat_buffer.find_lines( - partial(predicate, event.redaction_id) + partial(predicate, event.redacts) ) # No line to redact, return early @@ -956,8 +956,8 @@ class RoomBuffer(object): self.get_event_tags(event) ) - # elif isinstance(event, RoomRedactionEvent): - # self._redact_line(event) + elif isinstance(event, RedactionEvent): + self._redact_line(event) elif isinstance(event, RedactedEvent): self._handle_redacted_message(event) From 4b4d02b62037dce4524443c55be2ae95247127fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 3 Aug 2018 15:09:15 +0200 Subject: [PATCH 082/269] buffer: Add the event id to the tags for our own messages. --- matrix/buffer.py | 20 +++++++++++--------- matrix/server.py | 1 + 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 1cb211d..7430c25 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -647,14 +647,14 @@ class WeechatChannelBuffer(object): self.topic_author = nick self.topic_date = date - def self_message(self, nick, message, date): + def self_message(self, nick, message, date, tags=None): user = self._get_user(nick) - tags = self._message_tags(user, "self_message") + tags = self._message_tags(user, "self_message") + (tags or []) self._print_message(user, message, date, tags) - def self_action(self, nick, message, date): + def self_action(self, nick, message, date, tags=None): user = self._get_user(nick) - tags = self._message_tags(user, "self_message") + tags = self._message_tags(user, "self_message") + (tags or []) tags.append(SCRIPT_NAME + "_action") self._print_action(user, message, date, tags) @@ -994,20 +994,22 @@ class RoomBuffer(object): # type: (OwnMessage) -> None nick = self.find_nick(self.room.own_user_id) data = message.formatted_message.to_weechat() - - # TODO event_id tag is missing + tags = self.get_event_tags(message) date = message.age - self.weechat_buffer.self_message(nick, data, date) + + self.weechat_buffer.self_message(nick, data, date, tags) def self_action(self, message): # type: (OwnMessage) -> None nick = self.find_nick(self.room.own_user_id) date = message.age - # TODO event_id tag is missing + tags = self.get_event_tags(message) + self.weechat_buffer.self_action( nick, message.formatted_message.to_weechat(), - date + date, + tags ) def old_redacted(self, event): diff --git a/matrix/server.py b/matrix/server.py index b294e1f..916d6ad 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -472,6 +472,7 @@ class MatrixServer: def handle_own_messages(self, response): message = self.own_message_queue.pop(response.uuid) room_buffer = self.room_buffers[message.room_id] + message = message._replace(event_id=response.event_id) if isinstance(message, OwnAction): room_buffer.self_action(message) From 5e676f168f9cffe94bacacb4074290cdaea09bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 3 Aug 2018 15:27:11 +0200 Subject: [PATCH 083/269] buffer: Fix printing of unknown messages. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 7430c25..ffc6511 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -946,7 +946,7 @@ class RoomBuffer(object): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.server_timestamp) data = ("Unknown message of type {t}, body: {body}").format( - t=event.message_type, + t=event.type, body=event.message ) self.weechat_buffer.message( From a513b824187b769799b93a260986af4f609381a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 3 Aug 2018 17:43:34 +0200 Subject: [PATCH 084/269] server: Introduce a server config class. --- main.py | 4 +- matrix/commands.py | 39 ++++++--- matrix/server.py | 209 +++++++++++++++++++++++++++------------------ matrix/utils.py | 8 +- 4 files changed, 162 insertions(+), 98 deletions(-) diff --git a/main.py b/main.py index 8759f1a..8f9515e 100644 --- a/main.py +++ b/main.py @@ -181,7 +181,7 @@ def wrap_socket(server, file_descriptor): ssl_socket = server.ssl_context.wrap_socket( sock, do_handshake_on_connect=False, - server_hostname=server.address) # type: ssl.SSLSocket + server_hostname=server.config.address) # type: ssl.SSLSocket server.socket = ssl_socket @@ -428,7 +428,7 @@ def matrix_unload_cb(): def autoconnect(servers): for server in servers.values(): - if server.autoconnect: + if server.config.autoconnect: server.connect() diff --git a/matrix/commands.py b/matrix/commands.py index eb4baff..8387603 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -620,7 +620,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.options["autoconnect"] + option = server.config.options["autoconnect"] default_value = W.config_string_default(option) value = W.config_string(option) @@ -629,7 +629,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.options["address"] + option = server.config.options["address"] default_value = W.config_string_default(option) value = W.config_string(option) @@ -638,7 +638,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.options["port"] + option = server.config.options["port"] default_value = str(W.config_integer_default(option)) value = str(W.config_integer(option)) @@ -647,7 +647,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.options["username"] + option = server.config.options["username"] default_value = W.config_string_default(option) value = W.config_string(option) @@ -656,7 +656,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.options["password"] + option = server.config.options["password"] value = W.config_string(option) if value: @@ -691,7 +691,7 @@ def matrix_server_command_delete(args): if server.server_buffer: W.buffer_close(server.server_buffer) - for option in server.options.values(): + for option in server.config.options.values(): W.config_option_free(option) message = ("matrix: server {color}{server}{ncolor} has been " @@ -721,7 +721,7 @@ def matrix_server_command_add(args): return def remove_server(server): - for option in server.options.values(): + for option in server.config.options.values(): W.config_option_free(option) del SERVERS[server.name] @@ -746,7 +746,11 @@ def matrix_server_command_add(args): except ValueError: host, port = args[1], None - return_code = W.config_option_set(server.options["address"], host, 1) + return_code = W.config_option_set( + server.config.options["address"], + host, + 1 + ) if return_code == W.WEECHAT_CONFIG_OPTION_SET_ERROR: remove_server(server) @@ -763,7 +767,11 @@ def matrix_server_command_add(args): return if port: - return_code = W.config_option_set(server.options["port"], port, 1) + return_code = W.config_option_set( + server.config.options["port"], + port, + 1 + ) if return_code == W.WEECHAT_CONFIG_OPTION_SET_ERROR: remove_server(server) message = ("{prefix}Failed to set port for server " @@ -780,7 +788,11 @@ def matrix_server_command_add(args): if len(args) >= 3: user = args[2] - return_code = W.config_option_set(server.options["username"], user, 1) + return_code = W.config_option_set( + server.config.options["username"], + user, + 1 + ) if return_code == W.WEECHAT_CONFIG_OPTION_SET_ERROR: remove_server(server) @@ -799,8 +811,11 @@ def matrix_server_command_add(args): if len(args) == 4: password = args[3] - return_code = W.config_option_set(server.options["password"], password, - 1) + return_code = W.config_option_set( + server.config.options["password"], + password, + 1 + ) if return_code == W.WEECHAT_CONFIG_OPTION_SET_ERROR: remove_server(server) message = ("{prefix}Failed to set password for server " diff --git a/matrix/server.py b/matrix/server.py index 916d6ad..e653314 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -46,25 +46,118 @@ except NameError: FileNotFoundError = IOError -class MatrixServer: +class ServerConfig(object): + _section_name = "{}.{}".format(SCRIPT_NAME, "server") + + def __init__(self, server_name, config_ptr): + # type: (str, str) -> None + self._server_name = server_name + self._ptr = config_ptr + self.options = {} + + options = [ + Option('autoconnect', 'boolean', '', 0, 0, 'off', + ("automatically connect to the matrix server when weechat " + "is starting")), + Option('address', 'string', '', 0, 0, '', + "Hostname or IP address for the server"), + Option('port', 'integer', '', 0, 65535, '443', + "Port for the server"), + Option('proxy', 'string', '', 0, 0, '', + ("Name of weechat proxy to use (see /help proxy)")), + Option('ssl_verify', 'boolean', '', 0, 0, 'on', + ("Check that the SSL connection is fully trusted")), + Option('username', 'string', '', 0, 0, '', + "Username to use on server"), + Option( + 'password', 'string', '', 0, 0, '', + ("Password for server (note: content is evaluated, see /help " + "eval)")), + Option('device_name', 'string', '', 0, 0, 'Weechat Matrix', + "Device name to use while logging in to the matrix server"), + ] + + section = W.config_search_section(config_ptr, 'server') + + for option in options: + option_name = "{server}.{option}".format( + server=self._server_name, option=option.name) + + self.options[option.name] = W.config_new_option( + config_ptr, section, option_name, option.type, + option.description, option.string_values, option.min, + option.max, option.value, option.value, 0, "", "", + "matrix_config_server_change_cb", self._server_name, "", "") + + def _get_str_option(self, option_name): + return W.config_string(self.options[option_name]) + + def _get_bool_option(self, option_name): + return bool(W.config_boolean(self.options[option_name])) + + @property + def config_section(self): + # type: () -> str + return "{}.{}".format(self._server_name, self._server_name) + + @property + def autoconnect(self): + # type: () -> bool + return self._get_bool_option("autoconnect") + + @property + def address(self): + # type: () -> str + return self._get_str_option("address") + + @property + def port(self): + # type: () -> int + return W.config_integer(self.options["port"]) + + @property + def proxy(self): + # type: () -> str + return self._get_str_option("proxy") + + @property + def ssl_verify(self): + # type: () -> bool + return self._get_bool_option("ssl_verify") + + @property + def username(self): + # type: () -> str + return self._get_str_option("username") + + @property + def password(self): + # type: () -> str + return W.string_eval_expression( + self._get_str_option("password"), + {}, + {}, + {} + ) + + @property + def device_name(self): + # type: () -> str + return self._get_str_option("device_name") + + +class MatrixServer(object): # pylint: disable=too-many-instance-attributes def __init__(self, name, config_file): # type: (str, weechat.config) -> None # yapf: disable self.name = name # type: str self.user_id = "" - self.address = "" # type: str - self.port = 8448 # type: int - self.options = dict() # type: Dict[str, weechat.config] - self.device_name = "Weechat Matrix" # type: str self.device_id = "" # type: str self.olm = None # type: Olm self.encryption_queue = defaultdict(deque) - self.user = "" # type: str - self.password = "" # type: str - self.room_buffers = dict() # type: Dict[str, WeechatChannelBuffer] self.buffers = dict() # type: Dict[str, weechat.buffer] self.server_buffer = None # type: weechat.buffer @@ -73,10 +166,8 @@ class MatrixServer: self.timer_hook = None # type: weechat.hook self.numeric_address = "" # type: str - self.autoconnect = False # type: bool self.connected = False # type: bool self.connecting = False # type: bool - self.proxy = None # type: str self.reconnect_delay = 0 # type: int self.reconnect_time = None # type: float self.sync_time = None # type: Optional[float] @@ -108,7 +199,8 @@ class MatrixServer: self.event_queue_timer = None self.event_queue = deque() # type: Deque[RoomInfo] - self._create_options(config_file) + # self._create_options(config_file) + self.config = ServerConfig(self.name, config_file) self._create_session_dir() # yapf: enable @@ -124,7 +216,7 @@ class MatrixServer: return os.path.join(home_dir, "matrix", self.name) def _load_device_id(self): - file_name = "{}{}".format(self.user, ".device_id") + file_name = "{}{}".format(self.config.username, ".device_id") path = os.path.join(self.get_session_path(), file_name) if not os.path.isfile(path): @@ -136,66 +228,21 @@ class MatrixServer: self.device_id = device_id def save_device_id(self): - file_name = "{}{}".format(self.user, ".device_id") + file_name = "{}{}".format(self.config.username, ".device_id") path = os.path.join(self.get_session_path(), file_name) with open(path, 'w') as f: f.write(self.device_id) - def _create_options(self, config_file): - options = [ - Option('autoconnect', 'boolean', '', 0, 0, 'off', - ("automatically connect to the matrix server when weechat " - "is starting")), - Option('address', 'string', '', 0, 0, '', - "Hostname or IP address for the server"), - Option('port', 'integer', '', 0, 65535, '8448', - "Port for the server"), - Option('proxy', 'string', '', 0, 0, '', - ("Name of weechat proxy to use (see /help proxy)")), - Option('ssl_verify', 'boolean', '', 0, 0, 'on', - ("Check that the SSL connection is fully trusted")), - Option('username', 'string', '', 0, 0, '', - "Username to use on server"), - Option( - 'password', 'string', '', 0, 0, '', - ("Password for server (note: content is evaluated, see /help " - "eval)")), - Option('device_name', 'string', '', 0, 0, 'Weechat Matrix', - "Device name to use while logging in to the matrix server"), - ] - - section = W.config_search_section(config_file, 'server') - - for option in options: - option_name = "{server}.{option}".format( - server=self.name, option=option.name) - - self.options[option.name] = W.config_new_option( - config_file, section, option_name, option.type, - option.description, option.string_values, option.min, - option.max, option.value, option.value, 0, "", "", - "matrix_config_server_change_cb", self.name, "", "") - def _change_client(self): - host = ':'.join([self.address, str(self.port)]) - self.client = HttpClient(host, self.user) + host = ':'.join([self.config.address, str(self.config.port)]) + self.client = HttpClient(host, self.config.username) def update_option(self, option, option_name): if option_name == "address": - value = W.config_string(option) - self.address = value self._change_client() - elif option_name == "autoconnect": - value = W.config_boolean(option) - self.autoconnect = value elif option_name == "port": - value = W.config_integer(option) - self.port = value self._change_client() - elif option_name == "proxy": - value = W.config_string(option) - self.proxy = value elif option_name == "ssl_verify": value = W.config_boolean(option) if value: @@ -206,18 +253,10 @@ class MatrixServer: self.ssl_context.verify_mode = ssl.CERT_NONE elif option_name == "username": value = W.config_string(option) - self.user = value self.access_token = "" if self.client: self.client.user = value - - elif option_name == "password": - value = W.config_string(option) - self.password = W.string_eval_expression(value, {}, {}, {}) - elif option_name == "device_name": - value = W.config_string(option) - self.device_name = value else: pass @@ -271,7 +310,7 @@ class MatrixServer: prefix=W.prefix("network"))) server_buffer_prnt( self, ("{prefix}matrix: disconnecting from server..." - ).format(prefix=W.prefix("network"))) + ).format(prefix=W.prefix("network"))) self.disconnect() return False @@ -373,13 +412,14 @@ class MatrixServer: def connect(self): # type: (MatrixServer) -> int - if not self.address or not self.port: + if not self.config.address or not self.config.port: + W.prnt("", self.config.address) message = "{prefix}Server address or port not set".format( prefix=W.prefix("error")) W.prnt("", message) return False - if not self.user or not self.password: + if not self.config.username or not self.config.password: message = "{prefix}User or password not set".format( prefix=W.prefix("error")) W.prnt("", message) @@ -400,14 +440,14 @@ class MatrixServer: message = ("{prefix}matrix: Connecting to " "{server}:{port}{ssl}...").format( prefix=W.prefix("network"), - server=self.address, - port=self.port, + server=self.config.address, + port=self.config.port, ssl=ssl_message) W.prnt(self.server_buffer, message) - W.hook_connect(self.proxy if self.proxy else "", - self.address, self.port, + W.hook_connect(self.config.proxy, + self.config.address, self.config.port, 1, 0, "", "connect_cb", self.name) @@ -424,7 +464,7 @@ class MatrixServer: def login(self): # type: () -> None - _, request = self.client.login(self.password) + _, request = self.client.login(self.config.password) self.send_or_queue(request) msg = "{prefix}matrix: Logging in...".format( @@ -597,8 +637,12 @@ def matrix_config_server_read_cb(data, config_file, section, option_name, SERVERS[server.name] = server # Ignore invalid options - if option in server.options: - return_code = W.config_option_set(server.options[option], value, 1) + if option in server.config.options: + return_code = W.config_option_set( + server.config.options[option], + value, + 1 + ) # TODO print out error message in case of erroneous return_code @@ -611,7 +655,7 @@ def matrix_config_server_write_cb(data, config_file, section_name): return W.WECHAT_CONFIG_WRITE_ERROR for server in SERVERS.values(): - for option in server.options.values(): + for option in server.config.options.values(): if not W.config_write_option(config_file, option): return W.WECHAT_CONFIG_WRITE_ERROR @@ -627,7 +671,7 @@ def matrix_config_server_change_cb(server_name, option): # The function config_option_get_string() is used to get differing # properties from a config option, sadly it's only available in the plugin # API of weechat. - option_name = key_from_value(server.options, option) + option_name = key_from_value(server.config.options, option) server.update_option(option, option_name) return 1 @@ -690,10 +734,11 @@ def matrix_timer_cb(server_name, remaining_calls): def create_default_server(config_file): - server = MatrixServer('matrix.org', config_file) + server = MatrixServer('matrix_org', config_file) SERVERS[server.name] = server - W.config_option_set(server.options["address"], "matrix.org", 1) + option = W.config_get(SCRIPT_NAME + ".server." + server.name + ".address") + W.config_option_set(option, "matrix.org", 1) return True diff --git a/matrix/utils.py b/matrix/utils.py index 15b3e6f..eea0a6d 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -69,7 +69,11 @@ def create_server_buffer(server): server_buffer_set_title(server) W.buffer_set(server.server_buffer, "localvar_set_type", 'server') - W.buffer_set(server.server_buffer, "localvar_set_nick", server.user) + W.buffer_set( + server.server_buffer, + "localvar_set_nick", + server.config.username + ) W.buffer_set(server.server_buffer, "localvar_set_server", server.name) W.buffer_set(server.server_buffer, "localvar_set_channel", server.name) @@ -106,7 +110,7 @@ def server_buffer_set_title(server): ip_string = "" title = ("Matrix: {address}:{port}{ip}").format( - address=server.address, port=server.port, ip=ip_string) + address=server.config.address, port=server.config.port, ip=ip_string) W.buffer_set(server.server_buffer, "title", title) From fb5a889cca8e3b679f5240bc6d9963e55503b6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 7 Aug 2018 11:25:48 +0200 Subject: [PATCH 085/269] main: Break from the receive loop if there was a transport exception. --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 8f9515e..583d1ad 100644 --- a/main.py +++ b/main.py @@ -304,6 +304,7 @@ def receive_cb(server_name, file_descriptor): except (RemoteTransportError, RemoteProtocolError) as e: server.error(str(e)) server.disconnect() + break response = server.client.next_response() From 99ca1b50ce1e9c0e3bdd1781934ea1aeb0d38e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 7 Aug 2018 11:26:31 +0200 Subject: [PATCH 086/269] server: Use a sync timeout only on HTTP2. --- main.py | 8 +++----- matrix/server.py | 6 +++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 583d1ad..73efd48 100644 --- a/main.py +++ b/main.py @@ -340,14 +340,12 @@ def finalize_connection(server): if negotiated_protocol is None: negotiated_protocol = server.socket.selected_npn_protocol() - transport_type = None - if negotiated_protocol == "http/1.1": - transport_type = TransportType.HTTP + server.transport_type = TransportType.HTTP elif negotiated_protocol == "h2": - transport_type = TransportType.HTTP2 + server.transport_type = TransportType.HTTP2 - data = server.client.connect(transport_type) + data = server.client.connect(server.transport_type) server.send(data) server.login() diff --git a/matrix/server.py b/matrix/server.py index e653314..7809a84 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -30,6 +30,7 @@ from nio import ( SyncRepsponse, RoomSendResponse, TransportResponse, + TransportType, LocalProtocolError ) @@ -173,6 +174,7 @@ class MatrixServer(object): self.sync_time = None # type: Optional[float] self.socket = None # type: ssl.SSLSocket self.ssl_context = ssl.create_default_context() # type: ssl.SSLContext + self.transport_type = None # type: Optional[nio.TransportType] # Enable http2 negotiation on the ssl context. self.ssl_context.set_alpn_protocols(["h2", "http/1.1"]) @@ -391,6 +393,7 @@ class MatrixServer(object): self.send_buffer = b"" self.current_message = None + self.transport_type = None try: self.client.disconnect() @@ -702,7 +705,8 @@ def matrix_timer_cb(server_name, remaining_calls): return W.WEECHAT_RC_OK if server.sync_time and current_time > (server.sync_time + 2): - server.sync(30000) + timeout = 0 if server.transport_type == TransportType.HTTP else 30000 + server.sync(timeout) while server.send_queue: message = server.send_queue.popleft() From 4d299b19ebe5b577dd995436bb50f4f04a76a98a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 7 Aug 2018 11:26:45 +0200 Subject: [PATCH 087/269] buffer: Unknown messages don't contain a message attribute anymore. --- matrix/buffer.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index ffc6511..4b13614 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -945,10 +945,7 @@ class RoomBuffer(object): elif isinstance(event, RoomMessageUnknown): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.server_timestamp) - data = ("Unknown message of type {t}, body: {body}").format( - t=event.type, - body=event.message - ) + data = ("Unknown message of type {t}").format(t=event.type) self.weechat_buffer.message( nick, data, From 47b303e97511dbc1c33142fb507b9e8d17196955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 7 Aug 2018 11:44:30 +0200 Subject: [PATCH 088/269] server: Add sync filters on initial sync and subsequents syncs. --- matrix/server.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 7809a84..3751f54 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -38,7 +38,7 @@ from matrix.plugin_options import Option, DebugType from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, create_server_buffer) from matrix.utf import utf8_decode -from matrix.globals import W, SERVERS, SCRIPT_NAME +from matrix.globals import W, SERVERS, SCRIPT_NAME, OPTIONS from .buffer import RoomBuffer, OwnMessage, OwnAction try: @@ -459,10 +459,10 @@ class MatrixServer(object): def schedule_sync(self): self.sync_time = time.time() - def sync(self, timeout=None): + def sync(self, timeout=None, filter=None): # type: Optional[int] -> None self.sync_time = None - _, request = self.client.sync(timeout) + _, request = self.client.sync(timeout, filter) self.send_or_queue(request) def login(self): @@ -553,7 +553,8 @@ class MatrixServer(object): # self.store_olm() # self.upload_keys(device_keys=True, one_time_keys=False) - self.sync() + sync_filter = {"room": {"timeline": {"limit": OPTIONS.sync_limit}}} + self.sync(timeout=0, filter=sync_filter) def _handle_room_info(self, response): for room_id, join_info in response.rooms.join.items(): @@ -706,7 +707,8 @@ def matrix_timer_cb(server_name, remaining_calls): if server.sync_time and current_time > (server.sync_time + 2): timeout = 0 if server.transport_type == TransportType.HTTP else 30000 - server.sync(timeout) + sync_filter = {"room": {"timeline": {"limit": 5000}}} + server.sync(timeout, sync_filter) while server.send_queue: message = server.send_queue.popleft() From 173e3450602c21126f17e27d56ea9d77e0415984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 7 Aug 2018 16:48:18 +0200 Subject: [PATCH 089/269] commands: Big command cleanup. --- main.py | 8 +- matrix/buffer.py | 2 +- matrix/commands.py | 263 ++++++++++++++++----------------------------- matrix/server.py | 20 ++-- 4 files changed, 109 insertions(+), 184 deletions(-) diff --git a/main.py b/main.py index 73efd48..af8cf80 100644 --- a/main.py +++ b/main.py @@ -39,15 +39,12 @@ from nio import TransportType, RemoteTransportError, RemoteProtocolError from matrix.colors import Formatted from matrix.utf import utf8_decode -from matrix.http import HttpResponse -from matrix.api import MatrixSendMessage -from matrix.encryption import matrix_olm_command_cb # Weechat searches for the registered callbacks in the scope of the main script # file, import the callbacks here so weechat can find them. from matrix.commands import (hook_commands, hook_page_up, matrix_command_cb, - matrix_command_join_cb, matrix_command_part_cb, - matrix_command_invite_cb, matrix_command_topic_cb, + matrix_topic_command_cb, matrix_command_join_cb, + matrix_command_part_cb, matrix_command_invite_cb, matrix_command_pgup_cb, matrix_redact_command_cb, matrix_command_buf_clear_cb, matrix_me_command_cb, matrix_command_kick_cb) @@ -421,7 +418,6 @@ def matrix_unload_cb(): # for server in SERVERS.values(): # server.store_olm() - W.prnt("", "unloading") return W.WEECHAT_RC_OK diff --git a/matrix/buffer.py b/matrix/buffer.py index 4b13614..0e53c06 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -75,7 +75,7 @@ def room_buffer_input_cb(server_name, buffer, input_data): formatted_data = Formatted.from_input_line(input_data) - server.room_send_text(room_buffer, formatted_data) + server.room_send_message(room_buffer, formatted_data, "m.text") return W.WEECHAT_RC_OK diff --git a/matrix/commands.py b/matrix/commands.py index 8387603..7fc1ec8 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -19,20 +19,47 @@ from builtins import str import re import time +import argparse import matrix.globals from matrix.globals import W, OPTIONS, SERVERS from matrix.utf import utf8_decode -from matrix.api import (MatrixTopicMessage, MatrixRedactMessage, - MatrixBacklogMessage, MatrixJoinMessage, - MatrixPartMessage, MatrixInviteMessage, - MatrixEmoteMessage, MatrixKickMessage) from matrix.utils import key_from_value, tags_from_line_data -from matrix.plugin_options import DebugType from matrix.server import MatrixServer from matrix.colors import Formatted -from matrix.encryption import matrix_hook_olm_command + + +class ParseError(Exception): + pass + + +class WeechatArgParse(argparse.ArgumentParser): + def print_usage(self, file): + pass + + def error(self, message): + m = ("{prefix}Error: {message} for command {command} " + "(see /help {command})").format(prefix=W.prefix("error"), + message=message, + command=self.prog) + W.prnt("", m) + raise ParseError + + +class WeechatCommandParser(object): + @staticmethod + def topic(args): + parser = WeechatArgParse(prog="topic") + + parser.add_argument("-delete", action="store_true") + parser.add_argument("topic", nargs="*") + + try: + parsed_args = parser.parse_args(args.split()) + return parsed_args + except ParseError: + return None def hook_commands(): @@ -46,7 +73,6 @@ def hook_commands(): 'connect ||' 'disconnect ||' 'reconnect ||' - 'debug ||' 'help '), # Description (' server: list, add, or remove Matrix servers\n' @@ -54,14 +80,12 @@ def hook_commands(): 'disconnect: disconnect from one or all Matrix servers\n' ' reconnect: reconnect to server(s)\n\n' ' help: show detailed command help\n\n' - ' debug: enable or disable debugging\n\n' 'Use /matrix help [command] to find out more.\n'), # Completions ('server %(matrix_server_commands)|%* ||' 'connect %(matrix_servers) ||' 'disconnect %(matrix_servers) ||' 'reconnect %(matrix_servers) ||' - 'debug %(matrix_debug_types) ||' 'help %(matrix_commands)'), # Function name 'matrix_command_cb', @@ -85,6 +109,21 @@ def hook_commands(): # 'matrix_redact_command_cb', # '') + W.hook_command( + # Command name and short description + "topic", + "get/set the room topic", + # Synopsis + ("[|-delete]"), + # Description + (" topic: topic to set\n" + "-delete: delete room topic"), + # Completions + "", + # Callback + "matrix_topic_command_cb", + "") + W.hook_command( # Command name and short description "me", @@ -95,19 +134,18 @@ def hook_commands(): ("message: message to send"), # Completions "", - # Function name + # Callback "matrix_me_command_cb", "") - matrix_hook_olm_command() - - # W.hook_command_run('/topic', 'matrix_command_topic_cb', '') - W.hook_command_run('/buffer clear', 'matrix_command_buf_clear_cb', '') + # TODO those should be hook_command() calls W.hook_command_run('/join', 'matrix_command_join_cb', '') W.hook_command_run('/part', 'matrix_command_part_cb', '') W.hook_command_run('/invite', 'matrix_command_invite_cb', '') W.hook_command_run('/kick', 'matrix_command_kick_cb', '') + W.hook_command_run('/buffer clear', 'matrix_command_buf_clear_cb', '') + # if OPTIONS.enable_backlog: # hook_page_up() @@ -129,16 +167,8 @@ def matrix_me_command_cb(data, buffer, args): return W.WEECHAT_RC_OK formatted_data = Formatted.from_input_line(args) - message = MatrixEmoteMessage( - server.client, - room_id=room_buffer.room.room_id, - formatted_message=formatted_data) - - if room_buffer.room.encrypted: - return W.WEECHAT_RC_OK - - server.send_or_queue(message) + server.room_send_message(room_buffer, formatted_data, "m.emote") return W.WEECHAT_RC_OK elif buffer == server.server_buffer: @@ -149,6 +179,36 @@ def matrix_me_command_cb(data, buffer, args): return W.WEECHAT_RC_OK +@utf8_decode +def matrix_topic_command_cb(data, buffer, args): + parsed_args = WeechatCommandParser.topic(args) + if not parsed_args: + return W.WEECHAT_RC_OK + + for server in SERVERS.values(): + if buffer == server.server_buffer: + server.error("command \"topic\" must be " + "executed on a Matrix room buffer") + return W.WEECHAT_RC_OK + + room = server.find_room_from_ptr(buffer) + if not room: + continue + + if not parsed_args.topic and not parsed_args.delete: + # TODO print the current topic + return W.WEECHAT_RC_OK + + if parsed_args.delete and parsed_args.topic: + # TODO error message + return W.WEECHAT_RC_OK + + topic = "" if parsed_args.delete else " ".join(parsed_args.topic) + + # TODO set the new topic + return W.WEECHAT_RC_OK + + def matrix_fetch_old_messages(server, room_id): room_buffer = server.find_room_from_id(room_id) room = room_buffer.room @@ -161,17 +221,7 @@ def matrix_fetch_old_messages(server, room_id): if not prev_batch: return - message = MatrixBacklogMessage( - server.client, - room_id=room_id, - token=prev_batch, - limit=OPTIONS.backlog_limit) - room_buffer.backlog_pending = True - W.bar_item_update("buffer_modes") - - server.send_or_queue(message) - - return + raise NotImplementedError def check_server_existence(server_name, servers): @@ -237,8 +287,8 @@ def matrix_command_join_cb(data, buffer, command): return _, room_id = split_args - message = MatrixJoinMessage(server.client, room_id=room_id) - server.send_or_queue(message) + + raise NotImplementedError for server in SERVERS.values(): if buffer in server.buffers.values(): @@ -273,9 +323,7 @@ def matrix_command_part_cb(data, buffer, command): _, rooms = split_args rooms = rooms.split(" ") - for room_id in rooms: - message = MatrixPartMessage(server.client, room_id=room_id) - server.send_or_queue(message) + raise NotImplementedError for server in SERVERS.values(): if buffer in server.buffers.values(): @@ -305,9 +353,7 @@ def matrix_command_invite_cb(data, buffer, command): _, invitee = split_args room_id = key_from_value(server.buffers, buf) - message = MatrixInviteMessage( - server.client, room_id=room_id, user_id=invitee) - server.send_or_queue(message) + raise NotImplementedError for server in SERVERS.values(): if buffer in server.buffers.values(): @@ -341,9 +387,7 @@ def matrix_command_kick_cb(data, buffer, command): kicked_user = split_args[0] reason = split_args[1:] or None - message = MatrixKickMessage( - server.client, room_id=room_id, user_id=kicked_user, reason=reason) - server.send_or_queue(message) + raise NotImplementedError for server in SERVERS.values(): if buffer in server.buffers.values(): @@ -411,66 +455,25 @@ def matrix_redact_command_cb(data, buffer, args): W.prnt("", message) return W.WEECHAT_RC_OK - message = MatrixRedactMessage( - server.client, - room_id=room_id, - event_id=event_id, - reason=reason) - server.send_or_queue(message) + raise NotImplementedError return W.WEECHAT_RC_OK elif buffer == server.server_buffer: message = ("{prefix}matrix: command \"redact\" must be " "executed on a Matrix channel buffer" - ).format(prefix=W.prefix("error")) + ).format(prefix=W.prefix("error")) W.prnt("", message) return W.WEECHAT_RC_OK return W.WEECHAT_RC_OK -def matrix_command_debug(args): - if not args: - message = ("{prefix}matrix: Too few arguments for command " - "\"/matrix debug\" (see /matrix help debug)" - ).format(prefix=W.prefix("error")) - W.prnt("", message) - return - - def toggle_debug(debug_type): - if debug_type in OPTIONS.debug: - message = ("{prefix}matrix: Disabling matrix {t} " - "debugging.").format( - prefix=W.prefix("error"), t=debug_type) - W.prnt("", message) - OPTIONS.debug.remove(debug_type) - else: - message = ("{prefix}matrix: Enabling matrix {t} " - "debugging.").format( - prefix=W.prefix("error"), t=debug_type) - W.prnt("", message) - OPTIONS.debug.append(debug_type) - - for command in args: - if command == "network": - toggle_debug(DebugType.NETWORK) - elif command == "messaging": - toggle_debug(DebugType.MESSAGING) - elif command == "timing": - toggle_debug(DebugType.TIMING) - else: - message = ("{prefix}matrix: Unknown matrix debug " - "type \"{t}\".").format( - prefix=W.prefix("error"), t=command) - W.prnt("", message) - - def matrix_command_help(args): if not args: message = ("{prefix}matrix: Too few arguments for command " "\"/matrix help\" (see /matrix help help)" - ).format(prefix=W.prefix("error")) + ).format(prefix=W.prefix("error")) W.prnt("", message) return @@ -556,19 +559,6 @@ def matrix_command_help(args): cmd_color=W.color("chat_buffer"), ncolor=W.color("reset")) - elif command == "debug": - message = ("{delimiter_color}[{ncolor}matrix{delimiter_color}] " - "{ncolor}{cmd_color}/debug{ncolor} " - " [...]" - "\n\n" - "enable/disable degugging for a Matrix subsystem" - "\n\n" - "debug-type: a Matrix debug type, one of messaging, " - "timing, network").format( - delimiter_color=W.color("chat_delimiters"), - cmd_color=W.color("chat_buffer"), - ncolor=W.color("reset")) - else: message = ("{prefix}matrix: No help available, \"{command}\" " "is not a matrix command").format( @@ -912,9 +902,6 @@ def matrix_command_cb(data, buffer, args): elif command == 'help': matrix_command_help(args) - elif command == 'debug': - matrix_command_debug(args) - else: message = ("{prefix}matrix: Error: unknown matrix command, " "\"{command}\" (type /help matrix for help)").format( @@ -922,69 +909,3 @@ def matrix_command_cb(data, buffer, args): W.prnt("", message) return W.WEECHAT_RC_OK - - -@utf8_decode -def matrix_command_topic_cb(data, buffer, command): - for server in SERVERS.values(): - if buffer in server.buffers.values(): - topic = None - room_buffer = server.find_room_from_ptr(buffer) - split_command = command.split(' ', 1) - - if len(split_command) == 2: - topic = split_command[1] - - if not topic: - room_buffer = server.find_room_from_ptr(buffer) - room = room_buffer.room - if not room.topic: - return W.WEECHAT_RC_OK - - if room.is_named(): - message = ('{prefix}Topic for {color}{room}{ncolor} is ' - '"{topic}"').format( - prefix=W.prefix("network"), - color=W.color("chat_buffer"), - ncolor=W.color("reset"), - room=room.named_room_name(), - topic=room.topic) - else: - message = ('{prefix}Topic is "{topic}"').format( - prefix=W.prefix("network"), - topic=room.topic) - - date = int(time.time()) - topic_date = room_buffer.weechat_buffer.topic_date.strftime( - "%a, %d %b %Y %H:%M:%S") - - tags = "matrix_topic,log1" - W.prnt_date_tags(buffer, date, tags, message) - - # TODO the nick should be colored - - # TODO we should use the display name as well as - # the user name here - message = ("{prefix}Topic set by {author} on " - "{date}").format( - prefix=W.prefix("network"), - author=room.topic_author, - date=topic_date) - W.prnt_date_tags(buffer, date, tags, message) - - return W.WEECHAT_RC_OK_EAT - - message = MatrixTopicMessage( - server.client, room_id=room.room_id, topic=topic) - server.send_or_queue(message) - - return W.WEECHAT_RC_OK_EAT - - elif buffer == server.server_buffer: - message = ("{prefix}matrix: command \"topic\" must be " - "executed on a Matrix channel buffer" - ).format(prefix=W.prefix("error")) - W.prnt(buffer, message) - return W.WEECHAT_RC_OK_EAT - - return W.WEECHAT_RC_OK diff --git a/matrix/server.py b/matrix/server.py index 3751f54..95e4299 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -476,9 +476,14 @@ class MatrixServer(object): W.prnt(self.server_buffer, msg) - def room_send_text(self, room_buffer, formatted): + def room_send_message(self, room_buffer, formatted, msgtype="m.text"): # type: (RoomBuffer, Formatted) -> None - own_message = OwnMessage( + if msgtype == "m.emote": + message_class = OwnAction + else: + message_class = OwnMessage + + own_message = message_class( self.user_id, 0, "", @@ -486,7 +491,7 @@ class MatrixServer(object): formatted ) - body = {"msgtype": "m.text", "body": formatted.to_plain()} + body = {"msgtype": msgtype, "body": formatted.to_plain()} if formatted.is_formatted(): body["format"] = "org.matrix.custom.html" @@ -614,10 +619,13 @@ class MatrixServer(object): self.buffers[room_id] = buf.weechat_buffer._ptr def find_room_from_ptr(self, pointer): - room_id = key_from_value(self.buffers, pointer) - room_buffer = self.room_buffers[room_id] + try: + room_id = key_from_value(self.buffers, pointer) + room_buffer = self.room_buffers[room_id] - return room_buffer + return room_buffer + except (ValueError, KeyError): + return None def find_room_from_id(self, room_id): room_buffer = self.room_buffers[room_id] From f1e87fe9b0a5c42c44112eec9a88eecc0382652f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 7 Aug 2018 19:07:54 +0200 Subject: [PATCH 090/269] server: Add topic setting. --- matrix/commands.py | 3 ++- matrix/server.py | 27 ++++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index 7fc1ec8..a601602 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -204,8 +204,9 @@ def matrix_topic_command_cb(data, buffer, args): return W.WEECHAT_RC_OK topic = "" if parsed_args.delete else " ".join(parsed_args.topic) + content = {"topic": topic} + server.room_send_state(room, content, "m.room.topic") - # TODO set the new topic return W.WEECHAT_RC_OK diff --git a/matrix/server.py b/matrix/server.py index 95e4299..323e3c7 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -29,6 +29,7 @@ from nio import ( LoginResponse, SyncRepsponse, RoomSendResponse, + RoomPutStateResponse, TransportResponse, TransportType, LocalProtocolError @@ -476,6 +477,14 @@ class MatrixServer(object): W.prnt(self.server_buffer, msg) + def room_send_state(self, room_buffer, body, event_type): + _, request = self.client.room_put_state( + room_buffer.room.room_id, + event_type, + body + ) + self.send_or_queue(request) + def room_send_message(self, room_buffer, formatted, msgtype="m.text"): # type: (RoomBuffer, Formatted) -> None if msgtype == "m.emote": @@ -584,6 +593,15 @@ class MatrixServer(object): self.next_batch = response.next_batch self.schedule_sync() + def handle_transport_response(self, response): + self.error(("Error with response of type type: {}, " + "error code {}").format( + response.request_info.type, response.status_code)) + + # TODO better error handling. + if response.request_info.type == "sync": + self.disconnect() + def handle_response(self, response): # type: (MatrixMessage) -> None self.lag = response.elapsed * 1000 @@ -598,17 +616,20 @@ class MatrixServer(object): W.bar_item_update("lag") if isinstance(response, TransportResponse): - self.error("Error in response, code: {}".format( - response.status_code)) - self.disconnect() + self.handle_transport_response(response) elif isinstance(response, LoginResponse): self._handle_login(response) + elif isinstance(response, SyncRepsponse): self._handle_sync(response) + elif isinstance(response, RoomSendResponse): self.handle_own_messages(response) + elif isinstance(response, RoomPutStateResponse): + pass + return def create_room_buffer(self, room_id): From 458f6580fbc5fe828ba021ba3ad804487f0201a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 8 Aug 2018 15:04:19 +0200 Subject: [PATCH 091/269] buffer: Enable support for media messages. --- matrix/buffer.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 0e53c06..1c2a9ba 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -33,6 +33,7 @@ from .utils import ( from .plugin_options import RedactType from nio import ( + Api, RoomMessageText, RoomMemberEvent, PowerLevelsEvent, @@ -42,6 +43,7 @@ from nio import ( RoomTopicEvent, RoomMessageEmote, RoomNameEvent, + RoomMessageMedia, RoomMessageUnknown, RedactionEvent ) @@ -925,22 +927,22 @@ class RoomBuffer(object): # self.get_event_tags(event) # ) - # elif isinstance(event, RoomMessageMedia): - # nick = self.find_nick(event.sender) - # date = server_ts_to_weechat(event.server_timestamp) - # http_url = mxc_to_http(event.url) - # url = http_url if http_url else event.url + elif isinstance(event, RoomMessageMedia): + nick = self.find_nick(event.sender) + date = server_ts_to_weechat(event.server_timestamp) + http_url = Api.mxc_to_http(event.url) + url = http_url if http_url else event.url - # description = ("/{}".format(event.description) - # if event.description else "") - # data = "{url}{desc}".format(url=url, desc=description) + description = ("/{}".format(event.body) + if event.body else "") + data = "{url}{desc}".format(url=url, desc=description) - # self.weechat_buffer.message( - # nick, - # data, - # date, - # self.get_event_tags(event) - # ) + self.weechat_buffer.message( + nick, + data, + date, + self.get_event_tags(event) + ) elif isinstance(event, RoomMessageUnknown): nick = self.find_nick(event.sender) From 8d4415f873fdd78e67a0c6b9bb186df949b549b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 8 Aug 2018 15:05:15 +0200 Subject: [PATCH 092/269] buffer: Handle encrypted rooms in the state dict as well. --- matrix/buffer.py | 10 ++++++---- matrix/server.py | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 1c2a9ba..c94723a 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -22,7 +22,7 @@ from builtins import super from functools import partial from typing import NamedTuple -from .globals import W, SERVERS, OPTIONS, SCRIPT_NAME, ENCRYPTION +from .globals import W, SERVERS, OPTIONS, SCRIPT_NAME from .utf import utf8_decode from .colors import Formatted from .utils import ( @@ -875,6 +875,11 @@ class RoomBuffer(object): self._handle_topic(event, True) elif isinstance(event, PowerLevelsEvent): self._handle_power_level(event) + elif isinstance(event, RoomEncryptionEvent): + message = ("This room is encrypted, encryption is " + "currently unsuported. Message sending is disabled for " + "this room.") + self.weechat_buffer.error(message) def handle_timeline_event(self, event): if isinstance(event, RoomMemberEvent): @@ -962,9 +967,6 @@ class RoomBuffer(object): self._handle_redacted_message(event) elif isinstance(event, RoomEncryptionEvent): - if ENCRYPTION: - return - message = ("This room is encrypted, encryption is " "currently unsuported. Message sending is disabled for " "this room.") diff --git a/matrix/server.py b/matrix/server.py index 323e3c7..caa9272 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -478,6 +478,9 @@ class MatrixServer(object): W.prnt(self.server_buffer, msg) def room_send_state(self, room_buffer, body, event_type): + if room_buffer.room.encrypted: + return + _, request = self.client.room_put_state( room_buffer.room.room_id, event_type, @@ -487,6 +490,9 @@ class MatrixServer(object): def room_send_message(self, room_buffer, formatted, msgtype="m.text"): # type: (RoomBuffer, Formatted) -> None + if room_buffer.room.encrypted: + return + if msgtype == "m.emote": message_class = OwnAction else: From 995d9215df36063e9c03af2c70e64bac523f3975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 8 Aug 2018 15:43:00 +0200 Subject: [PATCH 093/269] buffer: Add support for notices. --- matrix/buffer.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index c94723a..c366b01 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -44,6 +44,7 @@ from nio import ( RoomMessageEmote, RoomNameEvent, RoomMessageMedia, + RoomMessageNotice, RoomMessageUnknown, RedactionEvent ) @@ -922,15 +923,15 @@ class RoomBuffer(object): self.get_event_tags(event) ) - # elif isinstance(event, RoomMessageNotice): - # nick = self.find_nick(event.sender) - # date = server_ts_to_weechat(event.server_timestamp) - # self.weechat_buffer.notice( - # nick, - # event.message, - # date, - # self.get_event_tags(event) - # ) + elif isinstance(event, RoomMessageNotice): + nick = self.find_nick(event.sender) + date = server_ts_to_weechat(event.server_timestamp) + self.weechat_buffer.notice( + nick, + event.body, + date, + self.get_event_tags(event) + ) elif isinstance(event, RoomMessageMedia): nick = self.find_nick(event.sender) From 20b903ec52634ccf9dc075e9ecb5c43da3ca67d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 8 Aug 2018 16:27:49 +0200 Subject: [PATCH 094/269] buffer: Change the way buffer names are created. This mimics how the irc plugin creates buffer names. --- matrix/buffer.py | 4 +++- matrix/utils.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index c366b01..595b070 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -683,12 +683,14 @@ class RoomBuffer(object): self.room = room self.backlog_pending = False + buffer_name = "{}.{}".format(room.room_id, server_name) + # This dict remembers the connection from a user_id to the name we # displayed in the buffer self.displayed_nicks = {} user = shorten_sender(self.room.own_user_id) self.weechat_buffer = WeechatChannelBuffer( - room.room_id, + buffer_name, server_name, user ) diff --git a/matrix/utils.py b/matrix/utils.py index eea0a6d..3f3662b 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -64,10 +64,12 @@ def tags_from_line_data(line_data): def create_server_buffer(server): # type: (MatrixServer) -> None - server.server_buffer = W.buffer_new(server.name, "server_buffer_cb", + buffer_name = "server.{}".format(server.name) + server.server_buffer = W.buffer_new(buffer_name, "server_buffer_cb", server.name, "", "") server_buffer_set_title(server) + W.buffer_set(server.server_buffer, "short_name", server.name) W.buffer_set(server.server_buffer, "localvar_set_type", 'server') W.buffer_set( server.server_buffer, From cea4b7846d01296bb1b7fc4ef3f4ac128964f097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 8 Aug 2018 17:07:29 +0200 Subject: [PATCH 095/269] server: Reuse the stored device id. --- matrix/server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index caa9272..8fcbf0a 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -239,7 +239,7 @@ class MatrixServer(object): def _change_client(self): host = ':'.join([self.config.address, str(self.config.port)]) - self.client = HttpClient(host, self.config.username) + self.client = HttpClient(host, self.config.username, self.device_id) def update_option(self, option, option_name): if option_name == "address": @@ -258,8 +258,12 @@ class MatrixServer(object): value = W.config_string(option) self.access_token = "" + self._load_device_id() + if self.client: self.client.user = value + if self.device_id: + self.client.device_id = self.device_id else: pass From 67b44a51a3675355b478bccb0ab0bb479f5b3707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 8 Aug 2018 17:37:08 +0200 Subject: [PATCH 096/269] server: Don't login every time we have a reconnect. --- matrix/server.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index 8fcbf0a..1a46e0d 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -472,7 +472,23 @@ class MatrixServer(object): def login(self): # type: () -> None - _, request = self.client.login(self.config.password) + if self.client.logged_in: + msg = ("{prefix}{script_name}: Already logged in, " + "syncing...").format( + prefix=W.prefix("network"), + script_name=SCRIPT_NAME + ) + W.prnt(self.server_buffer, msg) + timeout = (0 if self.transport_type == TransportType.HTTP + else 30000) + sync_filter = {"room": {"timeline": {"limit": 5000}}} + self.sync(timeout, sync_filter) + return + + _, request = self.client.login( + self.config.password, + self.config.device_name + ) self.send_or_queue(request) msg = "{prefix}matrix: Logging in...".format( From 8597dbbf520406c04aaedacbb86a09af8df2a69f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 8 Aug 2018 20:42:04 +0200 Subject: [PATCH 097/269] buffer: Allow extra tags for notices. --- matrix/buffer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 595b070..053ae7b 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -456,20 +456,20 @@ class WeechatChannelBuffer(object): self.print_date_tags(data, date, tags) - def message(self, nick, message, date, extra_tags=[]): + def message(self, nick, message, date, extra_tags=None): # type: (str, str, int, str) -> None user = self._get_user(nick) - tags = self._message_tags(user, "message") + extra_tags + tags = self._message_tags(user, "message") + (extra_tags or []) self._print_message(user, message, date, tags) - def notice(self, nick, message, date): + def notice(self, nick, message, date, extra_tags=None): # type: (str, str, int) -> None data = "{color}{message}{ncolor}".format( color=W.color("irc.color.notice"), message=message, ncolor=W.color("reset")) - self.message(nick, data, date) + self.message(nick, data, date, extra_tags) def _print_action(self, user, message, date, tags): nick_prefix = ("" if not user.prefix else "{}{}{}".format( From 94330922eb49afa6f367073cacdb4000cb8edf8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 8 Aug 2018 20:42:16 +0200 Subject: [PATCH 098/269] colors: Handle the

tag. --- matrix/colors.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/matrix/colors.py b/matrix/colors.py index 18a2809..dec9cab 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -389,6 +389,12 @@ class MatrixHtmlParser(HTMLParser): self._toggle_attribute("strikethrough") elif tag == "blockquote": self._toggle_attribute("quote") + elif tag == "p": + if self.text: + self.add_substring(self.text, self.attributes.copy()) + self.text = "\n" + self.add_substring(self.text, DEFAULT_ATRIBUTES.copy()) + self.text = "" elif tag == "font": if self.text: self.add_substring(self.text, self.attributes.copy()) From 1475572c06ce61ef3b8e11b36b1edc54718bb850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 9 Aug 2018 13:58:55 +0200 Subject: [PATCH 099/269] buffer: Fix incorrect number of arguments for notices. --- matrix/buffer.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 053ae7b..908476d 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -151,6 +151,11 @@ class WeechatChannelBuffer(object): "notify_message", "log1", ], + "notice": [ + SCRIPT_NAME + "_notice", + "notify_message", + "log1", + ], "old_message": [ SCRIPT_NAME + "_message", "notify_message", @@ -288,7 +293,7 @@ class WeechatChannelBuffer(object): W.hdata_update(self._hdata, self._ptr, new_data) def __init__(self, name, server_name, user): - # type: (str, str, str) + # type: (str, str, str) -> None self._ptr = W.buffer_new( name, "room_buffer_input_cb", @@ -427,7 +432,7 @@ class WeechatChannelBuffer(object): color = self._color_for_tags(user.color) - if message_type != "action": + if message_type != "action" and message_type != "notice": tags.append("prefix_nick_{color}".format(color=color)) return tags @@ -463,13 +468,32 @@ class WeechatChannelBuffer(object): self._print_message(user, message, date, tags) def notice(self, nick, message, date, extra_tags=None): - # type: (str, str, int) -> None - data = "{color}{message}{ncolor}".format( - color=W.color("irc.color.notice"), - message=message, - ncolor=W.color("reset")) + # type: (str, str, int, Optional[List[str]]) -> None + user = self._get_user(nick) + user_prefix = ("" if not user.prefix else "{}{}{}".format( + W.color(self._get_prefix_color(user.prefix)), + user.prefix, + W.color("reset") + )) - self.message(nick, data, date, extra_tags) + user_string = "{}{}{}{}".format( + user_prefix, + user.color, + user.nick, + W.color("reset") + ) + + data = ("{prefix}\t{color}Notice" + "{del_color}({ncolor}{user}{del_color}){ncolor}" + ": {message}").format(prefix=W.prefix("network"), + color=W.color("irc.color.notice"), + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + user=user_string, + message=message) + + tags = self._message_tags(user, "notice") + (extra_tags or []) + self.print_date_tags(data, date, tags) def _print_action(self, user, message, date, tags): nick_prefix = ("" if not user.prefix else "{}{}{}".format( @@ -490,7 +514,7 @@ class WeechatChannelBuffer(object): self.print_date_tags(data, date, tags) def action(self, nick, message, date, extra_tags=[]): - # type: (str, str, int) -> None + # type: (str, str, int, Optional[List[str]]) -> None user = self._get_user(nick) tags = self._message_tags(user, "action") + extra_tags self._print_action(user, message, date, tags) From 0e66455cce02feb330514408dffdbc5d145d419c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 9 Aug 2018 17:51:32 +0200 Subject: [PATCH 100/269] matrix: Mypy fixes. --- matrix/bar_items.py | 2 -- matrix/server.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 5ae77b2..5a45547 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -15,10 +15,8 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import unicode_literals -from builtins import str from matrix.utf import utf8_decode -from matrix.utils import key_from_value from matrix.globals import W, SERVERS diff --git a/matrix/server.py b/matrix/server.py index 1a46e0d..c0c24a0 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -465,7 +465,7 @@ class MatrixServer(object): self.sync_time = time.time() def sync(self, timeout=None, filter=None): - # type: Optional[int] -> None + # type: (Optional[int], Optional[Dict[Any, Any]]) -> None self.sync_time = None _, request = self.client.sync(timeout, filter) self.send_or_queue(request) @@ -509,7 +509,7 @@ class MatrixServer(object): self.send_or_queue(request) def room_send_message(self, room_buffer, formatted, msgtype="m.text"): - # type: (RoomBuffer, Formatted) -> None + # type: (RoomBuffer, Formatted, str) -> None if room_buffer.room.encrypted: return From 8fca7d19c77f8cb76dd03b38154e406e3927ba3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 9 Aug 2018 17:52:24 +0200 Subject: [PATCH 101/269] colors: Rework newline adding for blockquotes and

tags. --- matrix/colors.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index dec9cab..2d71fcb 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -301,7 +301,7 @@ class Formatted(): return text weechat_strings = map(format_string, self.substrings) - return "".join(weechat_strings).rstrip("\n") + return "".join(weechat_strings).rstrip("\n").replace("\n\n", "\n") # TODO this should be a typed dict. @@ -357,6 +357,12 @@ class MatrixHtmlParser(HTMLParser): self._toggle_attribute("strikethrough") elif tag == "blockquote": self._toggle_attribute("quote") + elif tag == "p": + if self.text: + self.add_substring(self.text, self.attributes.copy()) + self.text = "\n" + self.add_substring(self.text, DEFAULT_ATRIBUTES.copy()) + self.text = "" elif tag == "br": if self.text: self.add_substring(self.text, self.attributes.copy()) @@ -389,9 +395,6 @@ class MatrixHtmlParser(HTMLParser): self._toggle_attribute("strikethrough") elif tag == "blockquote": self._toggle_attribute("quote") - elif tag == "p": - if self.text: - self.add_substring(self.text, self.attributes.copy()) self.text = "\n" self.add_substring(self.text, DEFAULT_ATRIBUTES.copy()) self.text = "" From c9ef4ac086288ea366557c626bf4022b41404b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 9 Aug 2018 17:52:51 +0200 Subject: [PATCH 102/269] server: Reconnect if there was an error during login. --- matrix/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index c0c24a0..6aaff99 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -625,7 +625,8 @@ class MatrixServer(object): response.request_info.type, response.status_code)) # TODO better error handling. - if response.request_info.type == "sync": + if (response.request_info.type == "sync" or + response.request_info.type == "login"): self.disconnect() def handle_response(self, response): From 7f71bf47a1c2be3850fe653fa2b681d602c017d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 20 Aug 2018 11:11:54 +0200 Subject: [PATCH 103/269] colors: Colorize matrix block quotes. --- matrix/colors.py | 20 ++++++++++++++------ matrix/config.py | 7 ++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 2d71fcb..a2ad6ae 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -21,7 +21,7 @@ from __future__ import unicode_literals # pylint: disable=redefined-builtin from builtins import str from collections import namedtuple -from matrix.globals import W +from matrix.globals import W, OPTIONS from matrix.utils import string_strikethrough import textwrap @@ -33,13 +33,9 @@ except ImportError: from html.parser import HTMLParser import html -from html.entities import name2codepoint FormattedString = namedtuple('FormattedString', ['text', 'attributes']) -quote_wrapper = textwrap.TextWrapper( - initial_indent="> ", subsequent_indent="> ") - class Formatted(): @@ -47,6 +43,17 @@ class Formatted(): # type: (List[FormattedString]) -> None self.substrings = substrings + @property + def textwrapper(self): + return textwrap.TextWrapper( + width=67, + initial_indent="{}> ".format( + W.color(W.config_string(OPTIONS.options["quote"])) + ), + subsequent_indent="{}> ".format( + W.color(W.config_string(OPTIONS.options["quote"])) + )) + def is_formatted(self): # type: (Formatted) -> bool for string in self.substrings: @@ -268,7 +275,8 @@ class Formatted(): return string_strikethrough(string) elif name == "quote" and value: - return quote_wrapper.fill(string.replace("\n", "")) + return self.textwrapper.fill( + W.string_remove_color(string.replace("\n", ""), "")) elif name == "fgcolor" and value: return "{color_on}{text}{color_off}".format( diff --git a/matrix/config.py b/matrix/config.py index 7c49e66..3382049 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -138,6 +138,11 @@ def matrix_config_init(config_file): ("Use a separate buffer for debug logs.")), ] + color_options = [ + Option("quote", "color", "", 0, 0, "lightgreen", + ("Color for matrix style blockquotes")) + ] + def add_global_options(section, options): for option in options: OPTIONS.options[option.name] = W.config_new_option( @@ -149,7 +154,7 @@ def matrix_config_init(config_file): section = W.config_new_section(config_file, "color", 0, 0, "", "", "", "", "", "", "", "", "", "") - # TODO color options + add_global_options(section, color_options) section = W.config_new_section(config_file, "look", 0, 0, "", "", "", "", "", "", "", "", "", "") From 1e016ad49552dd8c6abd7985f02785df4c8f133b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 20 Aug 2018 11:12:28 +0200 Subject: [PATCH 104/269] colors: Colorize code snippets. --- matrix/colors.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/matrix/colors.py b/matrix/colors.py index a2ad6ae..06c6485 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -27,6 +27,11 @@ from matrix.utils import string_strikethrough import textwrap import webcolors +from pygments import highlight +from pygments.lexers import guess_lexer, get_lexer_by_name +from pygments.formatter import Formatter +from pygments.util import ClassNotFound + try: from HTMLParser import HTMLParser except ImportError: @@ -278,6 +283,14 @@ class Formatted(): return self.textwrapper.fill( W.string_remove_color(string.replace("\n", ""), "")) + elif name == "code" and value: + try: + lexer = get_lexer_by_name(value) + except ClassNotFound: + lexer = guess_lexer(string) + + return highlight(string, lexer, WeechatFormatter()) + elif name == "fgcolor" and value: return "{color_on}{text}{color_off}".format( color_on=W.color(value), @@ -319,6 +332,7 @@ DEFAULT_ATRIBUTES = { "underline": False, "strikethrough": False, "quote": False, + "code": None, "fgcolor": None, "bgcolor": None } @@ -365,6 +379,20 @@ class MatrixHtmlParser(HTMLParser): self._toggle_attribute("strikethrough") elif tag == "blockquote": self._toggle_attribute("quote") + elif tag == "code": + lang = None + + for key, value in attrs: + if key == "class": + if value.startswith("language-"): + lang = value.split("-", 1)[1] + + lang = lang or "unknown" + + if self.text: + self.add_substring(self.text, self.attributes.copy()) + self.text = "" + self.attributes["code"] = lang elif tag == "p": if self.text: self.add_substring(self.text, self.attributes.copy()) @@ -401,6 +429,13 @@ class MatrixHtmlParser(HTMLParser): self._toggle_attribute("underline") elif tag == "del": self._toggle_attribute("strikethrough") + elif tag == "code": + if self.text: + self.add_substring(self.text, self.attributes.copy()) + self.text = "\n" + self.add_substring(self.text, DEFAULT_ATRIBUTES.copy()) + self.text = "" + self.attributes["code"] = None elif tag == "blockquote": self._toggle_attribute("quote") self.text = "\n" @@ -932,3 +967,48 @@ def color_weechat_to_html(color): return hex_colors[weechat_basic_colors[color]] else: return hex_colors[color] + + +class WeechatFormatter(Formatter): + def __init__(self, **options): + Formatter.__init__(self, **options) + self.styles = {} + + for token, style in self.style: + start = end = "" + if style["color"]: + start += "{}".format( + W.color(color_html_to_weechat(str(style["color"])))) + end = "{}".format(W.color("resetcolor")) + end + if style["bold"]: + start += W.color("bold") + end = W.color("-bold") + end + if style["italic"]: + start += W.color("italic") + end = W.color("-italic") + end + if style['underline']: + start += W.color("underline") + end = W.color("-underline") + end + self.styles[token] = (start, end) + + def format(self, tokensource, outfile): + lastval = '' + lasttype = None + + for ttype, value in tokensource: + while ttype not in self.styles: + ttype = ttype.parent + + if ttype == lasttype: + lastval += value + else: + if lastval: + stylebegin, styleend = self.styles[lasttype] + outfile.write(stylebegin + lastval + styleend) + # set lastval/lasttype to current values + lastval = value + lasttype = ttype + + if lastval: + stylebegin, styleend = self.styles[lasttype] + outfile.write(stylebegin + lastval + styleend) From 44e065d81802773cc00fc527746d4af59aa85c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 20 Aug 2018 13:06:56 +0200 Subject: [PATCH 105/269] colors: Strip whitespace from both ends of the formatted string. --- matrix/colors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/colors.py b/matrix/colors.py index 06c6485..e1a6eba 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -322,7 +322,7 @@ class Formatted(): return text weechat_strings = map(format_string, self.substrings) - return "".join(weechat_strings).rstrip("\n").replace("\n\n", "\n") + return "".join(weechat_strings).replace("\n\n", "\n").strip() # TODO this should be a typed dict. From 6d299219b055a05cc1668f4a1fe1d126ef37cdfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 21 Aug 2018 12:32:52 +0200 Subject: [PATCH 106/269] buffer: Add initial smart filter functionality. --- matrix/buffer.py | 93 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 908476d..b9f3dd7 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -89,19 +89,41 @@ def room_buffer_close_cb(data, buffer): class WeechatUser(object): - def __init__(self, nick, host=None, prefix=""): + def __init__(self, nick, host=None, prefix="", join_time=None): # type: (str, str, str) -> None self.nick = nick self.host = host self.prefix = prefix self.color = W.info_get("nick_color_name", nick) + self.join_time = join_time or time.time() + self.speaking_time = None # type: int + + def update_speaking_time(self, new_time=None): + self.speaking_time = new_time or time.time() + + @property + def joined_recently(self): + # TODO make the delay configurable + delay = 30 + limit = time.time() - (delay * 60) + return self.join_time < limit + + @property + def spoken_recently(self): + if not self.speaking_time: + return False + + # TODO make the delay configurable + delay = 5 + limit = time.time() - (delay * 60) + return self.speaking_time < limit class RoomUser(WeechatUser): - def __init__(self, nick, user_id=None, power_level=0): + def __init__(self, nick, user_id=None, power_level=0, join_time=None): # type: (str, str, int) -> None prefix = self._get_prefix(power_level) - return super().__init__(nick, user_id, prefix) + return super().__init__(nick, user_id, prefix, join_time) @property def power_level(self): @@ -303,7 +325,8 @@ class WeechatChannelBuffer(object): ) self.name = "" - self.users = {} # type: Dict[str, RoomUser] + self.users = {} # type: Dict[str, WeechatUser] + self.smart_filtered_nicks = set() # type: Set[str] self.topic_author = "" self.topic_date = None @@ -365,6 +388,38 @@ class WeechatChannelBuffer(object): def _hdata(self): return W.hdata_get("buffer") + def add_smart_filtered_nick(self, nick): + self.smart_filtered_nicks.add(nick) + + def remove_smart_filtered_nick(self, nick): + self.smart_filtered_nicks.discard(nick) + + def unmask_smart_filtered_nick(self, nick): + if nick not in self.smart_filtered_nicks: + return + + for line in self.lines: + filtered = False + join = False + tags = line.tags + + if "nick_{}".format(nick) not in tags: + continue + + if SCRIPT_NAME + "_smart_filter" in tags: + filtered = True + elif SCRIPT_NAME + "_join" in tags: + join = True + + if filtered: + tags.remove(SCRIPT_NAME + "_smart_filter") + line.tags = tags + + if join: + break + + self.remove_smart_filtered_nick(nick) + @property def lines(self): own_lines = W.hdata_pointer( @@ -438,12 +493,12 @@ class WeechatChannelBuffer(object): return tags def _get_user(self, nick): - # type: (str) -> RoomUser + # type: (str) -> WeechatUser if nick in self.users: return self.users[nick] # A message from a non joined user - return RoomUser(nick) + return WeechatUser(nick) def _print_message(self, user, message, date, tags): prefix_string = ("" if not user.prefix else "{}{}{}".format( @@ -467,6 +522,9 @@ class WeechatChannelBuffer(object): tags = self._message_tags(user, "message") + (extra_tags or []) self._print_message(user, message, date, tags) + user.update_speaking_time(date) + self.unmask_smart_filtered_nick(nick) + def notice(self, nick, message, date, extra_tags=None): # type: (str, str, int, Optional[List[str]]) -> None user = self._get_user(nick) @@ -495,6 +553,9 @@ class WeechatChannelBuffer(object): tags = self._message_tags(user, "notice") + (extra_tags or []) self.print_date_tags(data, date, tags) + user.update_speaking_time(date) + self.unmask_smart_filtered_nick(nick) + def _print_action(self, user, message, date, tags): nick_prefix = ("" if not user.prefix else "{}{}{}".format( W.color(self._get_prefix_color(user.prefix)), @@ -519,6 +580,9 @@ class WeechatChannelBuffer(object): tags = self._message_tags(user, "action") + extra_tags self._print_action(user, message, date, tags) + user.update_speaking_time(date) + self.unmask_smart_filtered_nick(nick) + @staticmethod def _get_nicklist_group(user): # type: (WeechatUser) -> str @@ -604,7 +668,12 @@ class WeechatChannelBuffer(object): if message: tags = self._message_tags(user, "join") message = self._membership_message(user, "join") + + # TODO add a option to disable smart filters + tags.append(SCRIPT_NAME + "_smart_filter") + self.print_date_tags(message, date, tags) + self.add_smart_filtered_nick(user.nick) def invite(self, nick, date, extra_tags=[]): # type: (str, int, Optional[bool], Optional[List[str]]) -> None @@ -627,8 +696,14 @@ class WeechatChannelBuffer(object): if message: tags = self._message_tags(user, leave_type) + + # TODO make this configurable + if not user.spoken_recently: + tags.append(SCRIPT_NAME + "_smart_filter") + message = self._membership_message(user, leave_type) self.print_date_tags(message, date, tags + extra_tags) + self.remove_smart_filtered_nick(user.nick) if user.nick in self.users: del self.users[user.nick] @@ -657,6 +732,8 @@ class WeechatChannelBuffer(object): ) self.print_date_tags(data, date, tags) + user.update_speaking_time(date) + self.unmask_smart_filtered_nick(nick) @property def topic(self): @@ -745,7 +822,7 @@ class RoomBuffer(object): else: nick = short_name - buffer_user = RoomUser(nick, event.sender, user.power_level) + buffer_user = RoomUser(nick, event.sender, user.power_level, date) self.displayed_nicks[event.sender] = nick if self.room.own_user_id == event.sender: @@ -754,7 +831,7 @@ class RoomBuffer(object): self.weechat_buffer.join( buffer_user, - server_ts_to_weechat(event.server_timestamp), + date, not is_state ) From 166ff3b2e30af5d07210c0af21a8502e283f2ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 21 Aug 2018 12:33:08 +0200 Subject: [PATCH 107/269] colors: Replace multiple newlines with a single one. --- matrix/colors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matrix/colors.py b/matrix/colors.py index e1a6eba..85cbe24 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -24,6 +24,7 @@ from collections import namedtuple from matrix.globals import W, OPTIONS from matrix.utils import string_strikethrough +import re import textwrap import webcolors @@ -322,7 +323,7 @@ class Formatted(): return text weechat_strings = map(format_string, self.substrings) - return "".join(weechat_strings).replace("\n\n", "\n").strip() + return re.sub(r'\n+', '\n', "".join(weechat_strings)).strip() # TODO this should be a typed dict. From 607aad655bb875a26a02a9da12c6eef67d3e6d62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 22 Aug 2018 11:37:33 +0200 Subject: [PATCH 108/269] commands: Add redaction support back. --- matrix/commands.py | 43 ++++++++++++++++++++++--------------------- matrix/server.py | 10 +++++++--- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index a601602..c7ae56f 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -91,23 +91,23 @@ def hook_commands(): 'matrix_command_cb', '') - # W.hook_command( - # # Command name and short description - # 'redact', - # 'redact messages', - # # Synopsis - # ('[:""] []'), - # # Description - # ("message-number: number of message to redact (starting from 1 for\n" - # " the last message received, counting up)\n" - # " message-part: an initial part of the message (ignored, only used\n" - # " as visual feedback when using completion)\n" - # " reason: the redaction reason\n"), - # # Completions - # ('%(matrix_messages)'), - # # Function name - # 'matrix_redact_command_cb', - # '') + W.hook_command( + # Command name and short description + 'redact', + 'redact messages', + # Synopsis + ('[:""] []'), + # Description + ("message-number: number of message to redact (starting from 1 for\n" + " the last message received, counting up)\n" + " message-part: an initial part of the message (ignored, only used\n" + " as visual feedback when using completion)\n" + " reason: the redaction reason\n"), + # Completions + ('%(matrix_messages)'), + # Function name + 'matrix_redact_command_cb', + '') W.hook_command( # Command name and short description @@ -434,13 +434,14 @@ def event_id_from_line(buf, target_number): def matrix_redact_command_cb(data, buffer, args): for server in SERVERS.values(): if buffer in server.buffers.values(): - room_id = key_from_value(server.buffers, buffer) + room_buffer = server.find_room_from_ptr(buffer) matches = re.match(r"(\d+)(:\".*\")? ?(.*)?", args) if not matches: - message = ("{prefix}matrix: Invalid command arguments (see /help redact)" - ).format(prefix=W.prefix("error")) + message = ("{prefix}matrix: Invalid command " + "arguments (see /help redact)" + ).format(prefix=W.prefix("error")) W.prnt("", message) return W.WEECHAT_RC_ERROR @@ -456,7 +457,7 @@ def matrix_redact_command_cb(data, buffer, args): W.prnt("", message) return W.WEECHAT_RC_OK - raise NotImplementedError + server.room_send_redaction(room_buffer, event_id, reason) return W.WEECHAT_RC_OK diff --git a/matrix/server.py b/matrix/server.py index 6aaff99..3db149b 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -508,6 +508,13 @@ class MatrixServer(object): ) self.send_or_queue(request) + def room_send_redaction(self, room_buffer, event_id, reason=None): + _, request = self.client.room_redact( + room_buffer.room.room_id, + event_id, + reason) + self.send_or_queue(request) + def room_send_message(self, room_buffer, formatted, msgtype="m.text"): # type: (RoomBuffer, Formatted, str) -> None if room_buffer.room.encrypted: @@ -654,9 +661,6 @@ class MatrixServer(object): elif isinstance(response, RoomSendResponse): self.handle_own_messages(response) - elif isinstance(response, RoomPutStateResponse): - pass - return def create_room_buffer(self, room_id): From c3ab43b02e3b843489a827f49444f9158854b08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 22 Aug 2018 15:54:48 +0200 Subject: [PATCH 109/269] commands: Add back kick support. --- main.py | 5 ++- matrix/commands.py | 91 ++++++++++++++++++++++++++------------------ matrix/completion.py | 28 ++++++++++++++ matrix/server.py | 7 ++++ 4 files changed, 92 insertions(+), 39 deletions(-) diff --git a/main.py b/main.py index af8cf80..d1a6236 100644 --- a/main.py +++ b/main.py @@ -47,7 +47,7 @@ from matrix.commands import (hook_commands, hook_page_up, matrix_command_cb, matrix_command_part_cb, matrix_command_invite_cb, matrix_command_pgup_cb, matrix_redact_command_cb, matrix_command_buf_clear_cb, matrix_me_command_cb, - matrix_command_kick_cb) + matrix_kick_command_cb) from matrix.buffer import room_buffer_input_cb, room_buffer_close_cb from matrix.server import ( @@ -68,7 +68,8 @@ from matrix.completion import ( init_completion, matrix_command_completion_cb, matrix_server_command_completion_cb, matrix_debug_completion_cb, matrix_message_completion_cb, matrix_server_completion_cb, - matrix_olm_user_completion_cb, matrix_olm_device_completion_cb) + matrix_olm_user_completion_cb, matrix_olm_device_completion_cb, + matrix_user_completion_cb) from matrix.utils import (key_from_value, server_buffer_prnt, prnt_debug, server_buffer_set_title) diff --git a/matrix/commands.py b/matrix/commands.py index c7ae56f..edf2a00 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -18,7 +18,6 @@ from __future__ import unicode_literals from builtins import str import re -import time import argparse import matrix.globals @@ -48,6 +47,14 @@ class WeechatArgParse(argparse.ArgumentParser): class WeechatCommandParser(object): + @staticmethod + def _run_parser(parser, args): + try: + parsed_args = parser.parse_args(args.split()) + return parsed_args + except ParseError: + return None + @staticmethod def topic(args): parser = WeechatArgParse(prog="topic") @@ -55,11 +62,15 @@ class WeechatCommandParser(object): parser.add_argument("-delete", action="store_true") parser.add_argument("topic", nargs="*") - try: - parsed_args = parser.parse_args(args.split()) - return parsed_args - except ParseError: - return None + return WeechatCommandParser._run_parser(parser, args) + + @staticmethod + def kick(args): + parser = WeechatArgParse(prog="kick") + parser.add_argument("user_id") + parser.add_argument("reason", nargs="*") + + return WeechatCommandParser._run_parser(parser, args) def hook_commands(): @@ -138,11 +149,25 @@ def hook_commands(): "matrix_me_command_cb", "") + W.hook_command( + # Command name and short description + "kick", + "kick a user from the current room", + # Synopsis + (" []"), + # Description + ("user-id: user-id to kick\n" + " reason: reason why the user was kicked"), + # Completions + ("%(matrix_users)"), + # Callback + "matrix_kick_command_cb", + "") + # TODO those should be hook_command() calls - W.hook_command_run('/join', 'matrix_command_join_cb', '') - W.hook_command_run('/part', 'matrix_command_part_cb', '') - W.hook_command_run('/invite', 'matrix_command_invite_cb', '') - W.hook_command_run('/kick', 'matrix_command_kick_cb', '') + # W.hook_command_run('/join', 'matrix_command_join_cb', '') + # W.hook_command_run('/part', 'matrix_command_part_cb', '') + # W.hook_command_run('/invite', 'matrix_command_invite_cb', '') W.hook_command_run('/buffer clear', 'matrix_command_buf_clear_cb', '') @@ -365,35 +390,27 @@ def matrix_command_invite_cb(data, buffer, command): @utf8_decode -def matrix_command_kick_cb(data, buffer, command): - - def kick(server, buf, args): - split_args = args.split(" ", 1)[1:] - - if (len(split_args) < 1 or - split_args[0].startswith("#") and len(split_args) < 2): - error_msg = ( - '{prefix}Error with command "/kick" (help on ' - 'command: /help kick)').format(prefix=W.prefix("error")) - W.prnt("", error_msg) - return - - if split_args[0].startswith("#"): - assert len(split_args) >= 2 - room_id = split_args[0] - kicked_user = split_args[1] - reason = split_args[2:] or None - else: - room_id = key_from_value(server.buffers, buf) - kicked_user = split_args[0] - reason = split_args[1:] or None - - raise NotImplementedError +def matrix_kick_command_cb(data, buffer, args): + parsed_args = WeechatCommandParser.kick(args) + if not parsed_args: + return W.WEECHAT_RC_OK for server in SERVERS.values(): - if buffer in server.buffers.values(): - kick(server, buffer, command) - return W.WEECHAT_RC_OK_EAT + if buffer == server.server_buffer: + server.error("command \"kick\" must be " + "executed on a Matrix room buffer") + return W.WEECHAT_RC_OK + + room = server.find_room_from_ptr(buffer) + if not room: + continue + + user_id = parsed_args.user_id + user_id = user_id if user_id.startswith("@") else "@" + user_id + reason = " ".join(parsed_args.reason) if parsed_args.reason else None + + server.room_kick(room, user_id, reason) + break return W.WEECHAT_RC_OK diff --git a/matrix/completion.py b/matrix/completion.py index 454664b..a15d9d2 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -171,6 +171,31 @@ def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): return W.WEECHAT_RC_OK +@utf8_decode +def matrix_user_completion_cb(data, completion_item, buffer, completion): + def add_user(completion, user): + W.hook_completion_list_add(completion, user, 0, + W.WEECHAT_LIST_POS_SORT) + + for server in SERVERS.values(): + if buffer == server.server_buffer: + return W.WEECHAT_RC_OK + + room_buffer = server.find_room_from_ptr(buffer) + + if not room_buffer: + continue + + users = room_buffer.room.users + + users = [user[1:] for user in users] + + for user in users: + add_user(completion, user) + + return W.WEECHAT_RC_OK + + def init_completion(): W.hook_completion("matrix_server_commands", "Matrix server completion", "matrix_server_command_completion_cb", "") @@ -192,3 +217,6 @@ def init_completion(): W.hook_completion("olm_devices", "Matrix olm device id completion", "matrix_olm_device_completion_cb", "") + + W.hook_completion("matrix_users", "Matrix user id completion", + "matrix_user_completion_cb", "") diff --git a/matrix/server.py b/matrix/server.py index 3db149b..1a1ea7b 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -515,6 +515,13 @@ class MatrixServer(object): reason) self.send_or_queue(request) + def room_kick(self, room_buffer, user_id, reason=None): + _, request = self.client.room_kick( + room_buffer.room.room_id, + user_id, + reason) + self.send_or_queue(request) + def room_send_message(self, room_buffer, formatted, msgtype="m.text"): # type: (RoomBuffer, Formatted, str) -> None if room_buffer.room.encrypted: From 53d2152a21f50ece3731d138fc81ab3ac88a8660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 22 Aug 2018 17:13:25 +0200 Subject: [PATCH 110/269] commands: Add invite support. --- main.py | 2 +- matrix/commands.py | 60 ++++++++++++++++++++++++++++++---------------- matrix/server.py | 6 +++++ 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/main.py b/main.py index d1a6236..cc6753b 100644 --- a/main.py +++ b/main.py @@ -44,7 +44,7 @@ from matrix.utf import utf8_decode # file, import the callbacks here so weechat can find them. from matrix.commands import (hook_commands, hook_page_up, matrix_command_cb, matrix_topic_command_cb, matrix_command_join_cb, - matrix_command_part_cb, matrix_command_invite_cb, + matrix_command_part_cb, matrix_invite_command_cb, matrix_command_pgup_cb, matrix_redact_command_cb, matrix_command_buf_clear_cb, matrix_me_command_cb, matrix_kick_command_cb) diff --git a/matrix/commands.py b/matrix/commands.py index edf2a00..021f827 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -72,6 +72,13 @@ class WeechatCommandParser(object): return WeechatCommandParser._run_parser(parser, args) + @staticmethod + def invite(args): + parser = WeechatArgParse(prog="invite") + parser.add_argument("user_id") + + return WeechatCommandParser._run_parser(parser, args) + def hook_commands(): W.hook_command( @@ -164,10 +171,23 @@ def hook_commands(): "matrix_kick_command_cb", "") + W.hook_command( + # Command name and short description + "invite", + "invite a user to the current room", + # Synopsis + (""), + # Description + ("user-id: user-id to invite"), + # Completions + ("%(matrix_users)"), + # Callback + "matrix_invite_command_cb", + "") + # TODO those should be hook_command() calls # W.hook_command_run('/join', 'matrix_command_join_cb', '') # W.hook_command_run('/part', 'matrix_command_part_cb', '') - # W.hook_command_run('/invite', 'matrix_command_invite_cb', '') W.hook_command_run('/buffer clear', 'matrix_command_buf_clear_cb', '') @@ -363,28 +383,26 @@ def matrix_command_part_cb(data, buffer, command): @utf8_decode -def matrix_command_invite_cb(data, buffer, command): - - def invite(server, buf, args): - split_args = args.split(" ", 1) - - # TODO handle join for non public rooms - if len(split_args) != 2: - message = ( - "{prefix}Error with command \"/invite\" (help on " - "command: /help invite)").format(prefix=W.prefix("error")) - W.prnt("", message) - return - - _, invitee = split_args - room_id = key_from_value(server.buffers, buf) - - raise NotImplementedError +def matrix_invite_command_cb(data, buffer, args): + parsed_args = WeechatCommandParser.invite(args) + if not parsed_args: + return W.WEECHAT_RC_OK for server in SERVERS.values(): - if buffer in server.buffers.values(): - invite(server, buffer, command) - return W.WEECHAT_RC_OK_EAT + if buffer == server.server_buffer: + server.error("command \"invite\" must be " + "executed on a Matrix room buffer") + return W.WEECHAT_RC_OK + + room = server.find_room_from_ptr(buffer) + if not room: + continue + + user_id = parsed_args.user_id + user_id = user_id if user_id.startswith("@") else "@" + user_id + + server.room_invite(room, user_id) + break return W.WEECHAT_RC_OK diff --git a/matrix/server.py b/matrix/server.py index 1a1ea7b..0ca08c0 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -522,6 +522,12 @@ class MatrixServer(object): reason) self.send_or_queue(request) + def room_invite(self, room_buffer, user_id): + _, request = self.client.room_invite( + room_buffer.room.room_id, + user_id) + self.send_or_queue(request) + def room_send_message(self, room_buffer, formatted, msgtype="m.text"): # type: (RoomBuffer, Formatted, str) -> None if room_buffer.room.encrypted: From ebcbf454d99f4c5944dd95e4107e5f7fdb9e3d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 26 Aug 2018 20:44:06 +0200 Subject: [PATCH 111/269] commands: Add join/part. --- main.py | 4 +- matrix/buffer.py | 46 ++++++++++++++++ matrix/commands.py | 129 +++++++++++++++++++++++++-------------------- matrix/server.py | 24 ++++++--- 4 files changed, 136 insertions(+), 67 deletions(-) diff --git a/main.py b/main.py index cc6753b..fbcdeaf 100644 --- a/main.py +++ b/main.py @@ -43,8 +43,8 @@ from matrix.utf import utf8_decode # Weechat searches for the registered callbacks in the scope of the main script # file, import the callbacks here so weechat can find them. from matrix.commands import (hook_commands, hook_page_up, matrix_command_cb, - matrix_topic_command_cb, matrix_command_join_cb, - matrix_command_part_cb, matrix_invite_command_cb, + matrix_topic_command_cb, matrix_join_command_cb, + matrix_part_command_cb, matrix_invite_command_cb, matrix_command_pgup_cb, matrix_redact_command_cb, matrix_command_buf_clear_cb, matrix_me_command_cb, matrix_kick_command_cb) diff --git a/matrix/buffer.py b/matrix/buffer.py index b9f3dd7..d346ba4 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -783,6 +783,8 @@ class RoomBuffer(object): def __init__(self, room, server_name): self.room = room self.backlog_pending = False + self.joined = True + self.leave_event_id = None # type: Optional[str] buffer_name = "{}.{}".format(room.room_id, server_name) @@ -854,6 +856,12 @@ class RoomBuffer(object): if event.state_key in self.displayed_nicks: del self.displayed_nicks[event.state_key] + # We left the room, remember the event id of our leave, if we + # rejoin we get events that came before this event as well as + # after our leave, this way we know where to continue + if event.state_key == self.room.own_user_id: + self.leave_event_id = event.event_id + elif event.content["membership"] == "invite": if is_state: return @@ -1200,6 +1208,44 @@ class RoomBuffer(object): self.sort_messages() + def handle_joined_room(self, info): + for event in info.state: + self.handle_state_event(event) + + timeline_events = None + + # This is a rejoin, skip already handled events + if not self.joined: + leave_index = None + + for i, event in enumerate(info.timeline.events): + if event.event_id == self.leave_event_id: + leave_index = i + break + + if leave_index: + timeline_events = info.timeline.events[leave_index:] + else: + timeline_events = info.timeline.events + + # mark that we are now joined + self.joined = True + + else: + timeline_events = info.timeline.events + + for event in timeline_events: + self.handle_timeline_event(event) + + def handle_left_room(self, info): + self.joined = False + + for event in info.state: + self.handle_state_event(event) + + for event in info.timeline.events: + self.handle_timeline_event(event) + def error(self, string): # type: (str) -> None self.weechat_buffer.error(string) diff --git a/matrix/commands.py b/matrix/commands.py index 021f827..51b2707 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -79,6 +79,18 @@ class WeechatCommandParser(object): return WeechatCommandParser._run_parser(parser, args) + @staticmethod + def join(args): + parser = WeechatArgParse(prog="join") + parser.add_argument("room_id") + return WeechatCommandParser._run_parser(parser, args) + + @staticmethod + def part(args): + parser = WeechatArgParse(prog="part") + parser.add_argument("room_id", nargs="?") + return WeechatCommandParser._run_parser(parser, args) + def hook_commands(): W.hook_command( @@ -185,9 +197,34 @@ def hook_commands(): "matrix_invite_command_cb", "") - # TODO those should be hook_command() calls - # W.hook_command_run('/join', 'matrix_command_join_cb', '') - # W.hook_command_run('/part', 'matrix_command_part_cb', '') + W.hook_command( + # Command name and short description + "join", + "join a room", + # Synopsis + ("|"), + # Description + (" room-id: room-id of the room to join\n" + "room-alias: room alias of the room to join"), + # Completions + "", + # Callback + "matrix_join_command_cb", + "") + + W.hook_command( + # Command name and short description + "part", + "leave a room", + # Synopsis + ("[]"), + # Description + (" room-name: room name of the room to leave"), + # Completions + "", + # Callback + "matrix_part_command_cb", + "") W.hook_command_run('/buffer clear', 'matrix_command_buf_clear_cb', '') @@ -320,64 +357,41 @@ def matrix_command_pgup_cb(data, buffer, command): @utf8_decode -def matrix_command_join_cb(data, buffer, command): - - def join(server, args): - split_args = args.split(" ", 1) - - # TODO handle join for non public rooms - if len(split_args) != 2: - message = ("{prefix}Error with command \"/join\" (help on " - "command: /help join)").format(prefix=W.prefix("error")) - W.prnt("", message) - return - - _, room_id = split_args - - raise NotImplementedError +def matrix_join_command_cb(data, buffer, args): + parsed_args = WeechatCommandParser.join(args) + if not parsed_args: + return W.WEECHAT_RC_OK for server in SERVERS.values(): - if buffer in server.buffers.values(): - join(server, command) - return W.WEECHAT_RC_OK_EAT - elif buffer == server.server_buffer: - join(server, command) - return W.WEECHAT_RC_OK_EAT + if buffer in server.buffers.values() or buffer == server.server_buffer: + server.room_join(parsed_args.room_id) + break return W.WEECHAT_RC_OK @utf8_decode -def matrix_command_part_cb(data, buffer, command): - - def part(server, buffer, args): - rooms = [] - - split_args = args.split(" ", 1) - - if len(split_args) == 1: - if buffer == server.server_buffer: - message = ( - "{prefix}Error with command \"/part\" (help on " - "command: /help part)").format(prefix=W.prefix("error")) - W.prnt("", message) - return - - rooms = [key_from_value(server.buffers, buffer)] - - else: - _, rooms = split_args - rooms = rooms.split(" ") - - raise NotImplementedError +def matrix_part_command_cb(data, buffer, args): + parsed_args = WeechatCommandParser.part(args) + if not parsed_args: + return W.WEECHAT_RC_OK for server in SERVERS.values(): - if buffer in server.buffers.values(): - part(server, buffer, command) - return W.WEECHAT_RC_OK_EAT - elif buffer == server.server_buffer: - part(server, buffer, command) - return W.WEECHAT_RC_OK_EAT + if buffer in server.buffers.values() or buffer == server.server_buffer: + room_id = parsed_args.room_id + + if not room_id: + if buffer == server.server_buffer: + server.error("command \"part\" must be " + "executed on a Matrix room buffer or a room " + "name needs to be given") + return W.WEECHAT_RC_OK + + room_buffer = server.find_room_from_ptr(buffer) + room_id = room_buffer.room.room_id + + server.room_leave(room_id) + break return W.WEECHAT_RC_OK @@ -736,14 +750,14 @@ def matrix_server_command_delete(args): def matrix_server_command_add(args): if len(args) < 2: message = ("{prefix}matrix: Too few arguments for command " - "\"/matrix server add\" (see /matrix help server)" - ).format(prefix=W.prefix("error")) + "\"/matrix server add\" (see /matrix help server)").format( + prefix=W.prefix("error")) W.prnt("", message) return elif len(args) > 4: message = ("{prefix}matrix: Too many arguments for command " "\"/matrix server add\" (see /matrix help server)" - ).format(prefix=W.prefix("error")) + ).format(prefix=W.prefix("error")) W.prnt("", message) return @@ -865,7 +879,6 @@ def matrix_server_command_add(args): def matrix_server_command(command, args): - def list_servers(_): if SERVERS: W.prnt("", "\nAll matrix servers:") @@ -892,7 +905,6 @@ def matrix_server_command(command, args): @utf8_decode def matrix_command_cb(data, buffer, args): - def connect_server(args): for server_name in args: if check_server_existence(server_name, SERVERS): @@ -913,7 +925,8 @@ def matrix_command_cb(data, buffer, args): if len(split_args) < 1: message = ("{prefix}matrix: Too few arguments for command " - "\"/matrix\" (see /help matrix)").format(prefix=W.prefix("error")) + "\"/matrix\" " + "(see /help matrix)").format(prefix=W.prefix("error")) W.prnt("", message) return W.WEECHAT_RC_ERROR diff --git a/matrix/server.py b/matrix/server.py index 0ca08c0..1661b38 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -528,6 +528,14 @@ class MatrixServer(object): user_id) self.send_or_queue(request) + def room_join(self, room_id): + _, request = self.client.join(room_id) + self.send_or_queue(request) + + def room_leave(self, room_id): + _, request = self.client.room_leave(room_id) + self.send_or_queue(request) + def room_send_message(self, room_buffer, formatted, msgtype="m.text"): # type: (RoomBuffer, Formatted, str) -> None if room_buffer.room.encrypted: @@ -617,17 +625,19 @@ class MatrixServer(object): self.sync(timeout=0, filter=sync_filter) def _handle_room_info(self, response): - for room_id, join_info in response.rooms.join.items(): + for room_id, info in response.rooms.leave.items(): + if room_id not in self.buffers: + continue + + room_buffer = self.find_room_from_id(room_id) + room_buffer.handle_left_room(info) + + for room_id, info in response.rooms.join.items(): if room_id not in self.buffers: self.create_room_buffer(room_id) room_buffer = self.find_room_from_id(room_id) - - for event in join_info.state: - room_buffer.handle_state_event(event) - - for event in join_info.timeline.events: - room_buffer.handle_timeline_event(event) + room_buffer.handle_joined_room(info) def _handle_sync(self, response): # we got the same batch again, nothing to do From 084cb1985e4a2cbbbaf587c2e6abac9ed6c9f447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 26 Aug 2018 20:44:25 +0200 Subject: [PATCH 112/269] buffer: Fix buffer name format. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index d346ba4..a98a888 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -786,7 +786,7 @@ class RoomBuffer(object): self.joined = True self.leave_event_id = None # type: Optional[str] - buffer_name = "{}.{}".format(room.room_id, server_name) + buffer_name = "{}.{}".format(server_name, room.room_id) # This dict remembers the connection from a user_id to the name we # displayed in the buffer From cca6464cc7e64d6f46a29874e4b7d01a7766aa93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 26 Aug 2018 20:52:18 +0200 Subject: [PATCH 113/269] buffer: Don't print out our leave event if we're rejoining. --- matrix/buffer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index a98a888..e3ddae6 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -1224,7 +1224,10 @@ class RoomBuffer(object): break if leave_index: - timeline_events = info.timeline.events[leave_index:] + timeline_events = info.timeline.events[leave_index+1:] + # Handle our leave as a state event since we're not in the + # nicklist anymore but we're already printed out our leave + self.handle_state_event(info.timeline.events[leave_index]) else: timeline_events = info.timeline.events From 57c392a643608b91087ee2666d25a48e3a353618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 27 Aug 2018 20:48:20 +0200 Subject: [PATCH 114/269] server: Handle invites. --- matrix/server.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index 1661b38..142ed7b 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -334,7 +334,7 @@ class MatrixServer(object): # type: (MatrixServer) -> None self.send_buffer = b"" - def error(self, message): + def info(self, message): buf = "" if self.server_buffer: buf = self.server_buffer @@ -342,6 +342,14 @@ class MatrixServer(object): msg = "{}{}: {}".format(W.prefix("network"), SCRIPT_NAME, message) W.prnt(buf, msg) + def error(self, message): + buf = "" + if self.server_buffer: + buf = self.server_buffer + + msg = "{}{}: {}".format(W.prefix("error"), SCRIPT_NAME, message) + W.prnt(buf, msg) + def send(self, data): # type: (bytes) -> bool self.try_send(data) @@ -625,6 +633,30 @@ class MatrixServer(object): self.sync(timeout=0, filter=sync_filter) def _handle_room_info(self, response): + for room_id, info in response.rooms.invite.items(): + room = self.client.invited_rooms.get(room_id, None) + + if room: + if room.inviter: + inviter_msg = " by {}{}".format( + W.color("chat_nick_other"), + room.inviter) + else: + inviter_msg = "" + + self.info("You have been invited to {} {}({}{}{}){}" + "{}".format( + room.display_name(), + W.color("chat_delimiters"), + W.color("chat_channel"), + room_id, + W.color("chat_delimiters"), + W.color("reset"), + inviter_msg + )) + else: + self.info("You have been invited to {}.".format(room_id)) + for room_id, info in response.rooms.leave.items(): if room_id not in self.buffers: continue From cb5a4ad1d7a4ab8a7a1fcd12048a04a93ef14914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 28 Aug 2018 10:53:29 +0200 Subject: [PATCH 115/269] colors: Remove additional newlines from code segments. --- matrix/colors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 85cbe24..322197f 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -290,7 +290,9 @@ class Formatted(): except ClassNotFound: lexer = guess_lexer(string) - return highlight(string, lexer, WeechatFormatter()) + # highlight adds a newline to the end of the string, remove it + # from the output + return highlight(string, lexer, WeechatFormatter())[:-1] elif name == "fgcolor" and value: return "{color_on}{text}{color_off}".format( @@ -433,8 +435,6 @@ class MatrixHtmlParser(HTMLParser): elif tag == "code": if self.text: self.add_substring(self.text, self.attributes.copy()) - self.text = "\n" - self.add_substring(self.text, DEFAULT_ATRIBUTES.copy()) self.text = "" self.attributes["code"] = None elif tag == "blockquote": From 7c7612bdcde939eb0846ef9be48b647bdc1ecb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 28 Aug 2018 11:32:15 +0200 Subject: [PATCH 116/269] buffer: Pass the input data through string_input_for_buffer(). --- matrix/buffer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index e3ddae6..72126fa 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -76,7 +76,12 @@ def room_buffer_input_cb(server_name, buffer, input_data): room_buffer.error("You are not connected to the server") return W.WEECHAT_RC_ERROR - formatted_data = Formatted.from_input_line(input_data) + data = W.string_input_for_buffer(input_data) + + if not data: + data = input_data + + formatted_data = Formatted.from_input_line(data) server.room_send_message(room_buffer, formatted_data, "m.text") From b05c016fbd72950006ff0d36be1952cf201ab393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 28 Aug 2018 17:22:56 +0200 Subject: [PATCH 117/269] server: Use a property factory for the server config. --- main.py | 1 + matrix/server.py | 66 +++++++++++++++++------------------------------- 2 files changed, 24 insertions(+), 43 deletions(-) diff --git a/main.py b/main.py index fbcdeaf..48810d4 100644 --- a/main.py +++ b/main.py @@ -151,6 +151,7 @@ def print_certificate_info(buff, sock, cert): W.prnt(buff, message) + def wrap_socket(server, file_descriptor): # type: (MatrixServer, int) -> None sock = None # type: socket.socket diff --git a/matrix/server.py b/matrix/server.py index 142ed7b..50e81a1 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -49,7 +49,22 @@ except NameError: class ServerConfig(object): - _section_name = "{}.{}".format(SCRIPT_NAME, "server") + def option_property(name, option_type): + def bool_getter(self): + return bool(W.config_boolean(self.options[name])) + + def str_getter(self): + return W.config_string(self.options[name]) + + def int_getter(self): + return W.config_integer(self.options[name]) + + if option_type == str: + return property(str_getter) + elif option_type == bool: + return property(bool_getter) + elif option_type == int: + return property(int_getter) def __init__(self, server_name, config_ptr): # type: (str, str) -> None @@ -94,43 +109,13 @@ class ServerConfig(object): def _get_str_option(self, option_name): return W.config_string(self.options[option_name]) - def _get_bool_option(self, option_name): - return bool(W.config_boolean(self.options[option_name])) - - @property - def config_section(self): - # type: () -> str - return "{}.{}".format(self._server_name, self._server_name) - - @property - def autoconnect(self): - # type: () -> bool - return self._get_bool_option("autoconnect") - - @property - def address(self): - # type: () -> str - return self._get_str_option("address") - - @property - def port(self): - # type: () -> int - return W.config_integer(self.options["port"]) - - @property - def proxy(self): - # type: () -> str - return self._get_str_option("proxy") - - @property - def ssl_verify(self): - # type: () -> bool - return self._get_bool_option("ssl_verify") - - @property - def username(self): - # type: () -> str - return self._get_str_option("username") + autoconnect = option_property("autoconnect", bool) + address = option_property("address", str) + port = option_property("port", int) + proxy = option_property("proxy", str) + ssl_verify = option_property("ssl_verify", bool) + username = option_property("username", str) + device_name = option_property("device_name", str) @property def password(self): @@ -142,11 +127,6 @@ class ServerConfig(object): {} ) - @property - def device_name(self): - # type: () -> str - return self._get_str_option("device_name") - class MatrixServer(object): # pylint: disable=too-many-instance-attributes From 19eeec4f3b71b68f62963bb51acd7322b3114383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Aug 2018 15:35:36 +0200 Subject: [PATCH 118/269] config: Big config refactor. --- main.py | 41 ++-- matrix/buffer.py | 13 +- matrix/colors.py | 12 +- matrix/commands.py | 24 +-- matrix/completion.py | 10 +- matrix/config.py | 423 ++++++++++++++++++++++++++------------- matrix/globals.py | 4 +- matrix/plugin_options.py | 70 ------- matrix/server.py | 119 ++++++----- matrix/utils.py | 34 +--- 10 files changed, 401 insertions(+), 349 deletions(-) delete mode 100644 matrix/plugin_options.py diff --git a/main.py b/main.py index 48810d4..91abc9e 100644 --- a/main.py +++ b/main.py @@ -71,18 +71,20 @@ from matrix.completion import ( matrix_olm_user_completion_cb, matrix_olm_device_completion_cb, matrix_user_completion_cb) -from matrix.utils import (key_from_value, server_buffer_prnt, prnt_debug, +from matrix.utils import (key_from_value, server_buffer_prnt, server_buffer_set_title) -from matrix.plugin_options import (DebugType, RedactType) +from matrix.config import ( + matrix_config_reload_cb, + MatrixConfig, + config_log_level_cb, + config_log_category_cb, + config_server_buffer_cb +) -from matrix.config import (matrix_config_init, matrix_config_read, - matrix_config_free, matrix_config_change_cb, - matrix_config_reload_cb) +from matrix import globals as G -import matrix.globals - -from matrix.globals import W, SERVERS, SCRIPT_NAME, OPTIONS +from matrix.globals import W, SERVERS, SCRIPT_NAME # yapf: disable WEECHAT_SCRIPT_NAME = SCRIPT_NAME @@ -415,7 +417,10 @@ def room_close_cb(data, buffer): @utf8_decode def matrix_unload_cb(): - matrix_config_free(matrix.globals.CONFIG) + for server in SERVERS.values(): + server.config.free() + + G.CONFIG.free() # for server in SERVERS.values(): # server.store_olm() @@ -430,7 +435,7 @@ def autoconnect(servers): def debug_buffer_close_cb(data, buffer): - OPTIONS.debug_buffer_ptr = "" + G.CONFIG.debug_buffer = "" return W.WEECHAT_RC_OK @@ -450,12 +455,12 @@ class WeechatHandler(StreamHandler): def write(self, item): buf = "" - if OPTIONS.debug_buffer: - if not OPTIONS.debug_buffer_ptr: - OPTIONS.debug_buffer_ptr = W.buffer_new( + if G.CONFIG.network.debug_buffer: + if not G.CONFIG.debug_buffer: + G.CONFIG.debug_buffer = W.buffer_new( "Matrix Debug", "", "", "debug_buffer_close_cb", "") - buf = OPTIONS.debug_buffer_ptr + buf = G.CONFIG.debug_buffer W.prnt(buf, item) @@ -475,16 +480,14 @@ if __name__ == "__main__": handler.push_application() # TODO if this fails we should abort and unload the script. - matrix.globals.CONFIG = W.config_new("matrix", - "matrix_config_reload_cb", "") - matrix_config_init(matrix.globals.CONFIG) - matrix_config_read(matrix.globals.CONFIG) + G.CONFIG = MatrixConfig() + G.CONFIG.read() hook_commands() init_bar_items() init_completion() if not SERVERS: - create_default_server(matrix.globals.CONFIG) + create_default_server(G.CONFIG) autoconnect(SERVERS) diff --git a/matrix/buffer.py b/matrix/buffer.py index 72126fa..e8928e1 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -22,7 +22,8 @@ from builtins import super from functools import partial from typing import NamedTuple -from .globals import W, SERVERS, OPTIONS, SCRIPT_NAME +from . import globals as G +from .globals import W, SERVERS, SCRIPT_NAME from .utf import utf8_decode from .colors import Formatted from .utils import ( @@ -30,7 +31,7 @@ from .utils import ( server_ts_to_weechat, string_strikethrough, ) -from .plugin_options import RedactType +from .config import RedactType from nio import ( Api, @@ -128,7 +129,7 @@ class RoomUser(WeechatUser): def __init__(self, nick, user_id=None, power_level=0, join_time=None): # type: (str, str, int) -> None prefix = self._get_prefix(power_level) - return super().__init__(nick, user_id, prefix, join_time) + super().__init__(nick, user_id, prefix, join_time) @property def power_level(self): @@ -922,12 +923,12 @@ class RoomBuffer(object): new_message = "" - if OPTIONS.redaction_type == RedactType.STRIKETHROUGH: + if G.CONFIG.look.redaction_type == RedactType.STRIKETHROUGH: plaintext_msg = W.string_remove_color(message, '') new_message = string_strikethrough(plaintext_msg) - elif OPTIONS.redaction_type == RedactType.NOTICE: + elif G.CONFIG.look.redaction_type == RedactType.NOTICE: new_message = message - elif OPTIONS.redaction_type == RedactType.DELETE: + elif G.CONFIG.look.redaction_type == RedactType.DELETE: pass message = " ".join(s for s in [new_message, redaction_msg] if s) diff --git a/matrix/colors.py b/matrix/colors.py index 322197f..730ec27 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -21,7 +21,8 @@ from __future__ import unicode_literals # pylint: disable=redefined-builtin from builtins import str from collections import namedtuple -from matrix.globals import W, OPTIONS +from matrix import globals as G +from matrix.globals import W from matrix.utils import string_strikethrough import re @@ -53,12 +54,9 @@ class Formatted(): def textwrapper(self): return textwrap.TextWrapper( width=67, - initial_indent="{}> ".format( - W.color(W.config_string(OPTIONS.options["quote"])) - ), - subsequent_indent="{}> ".format( - W.color(W.config_string(OPTIONS.options["quote"])) - )) + initial_indent="{}> ".format(G.CONFIG.color.quote), + subsequent_indent="{}> ".format(G.CONFIG.color.quote) + ) def is_formatted(self): # type: (Formatted) -> bool diff --git a/matrix/commands.py b/matrix/commands.py index 51b2707..280149e 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -21,7 +21,7 @@ import re import argparse import matrix.globals -from matrix.globals import W, OPTIONS, SERVERS +from matrix.globals import W, SERVERS from matrix.utf import utf8_decode from matrix.utils import key_from_value, tags_from_line_data @@ -661,7 +661,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.config.options["autoconnect"] + option = server.config._option_ptrs["autoconnect"] default_value = W.config_string_default(option) value = W.config_string(option) @@ -670,7 +670,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.config.options["address"] + option = server.config._option_ptrs["address"] default_value = W.config_string_default(option) value = W.config_string(option) @@ -679,7 +679,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.config.options["port"] + option = server.config._option_ptrs["port"] default_value = str(W.config_integer_default(option)) value = str(W.config_integer(option)) @@ -688,7 +688,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.config.options["username"] + option = server.config._option_ptrs["username"] default_value = W.config_string_default(option) value = W.config_string(option) @@ -697,7 +697,7 @@ def matrix_server_command_listfull(args): W.prnt("", message) - option = server.config.options["password"] + option = server.config._option_ptrs["password"] value = W.config_string(option) if value: @@ -732,7 +732,7 @@ def matrix_server_command_delete(args): if server.server_buffer: W.buffer_close(server.server_buffer) - for option in server.config.options.values(): + for option in server.config._option_ptrs.values(): W.config_option_free(option) message = ("matrix: server {color}{server}{ncolor} has been " @@ -762,7 +762,7 @@ def matrix_server_command_add(args): return def remove_server(server): - for option in server.config.options.values(): + for option in server.config._option_ptrs.values(): W.config_option_free(option) del SERVERS[server.name] @@ -788,7 +788,7 @@ def matrix_server_command_add(args): host, port = args[1], None return_code = W.config_option_set( - server.config.options["address"], + server.config._option_ptrs["address"], host, 1 ) @@ -809,7 +809,7 @@ def matrix_server_command_add(args): if port: return_code = W.config_option_set( - server.config.options["port"], + server.config._option_ptrs["port"], port, 1 ) @@ -830,7 +830,7 @@ def matrix_server_command_add(args): if len(args) >= 3: user = args[2] return_code = W.config_option_set( - server.config.options["username"], + server.config._option_ptrs["username"], user, 1 ) @@ -853,7 +853,7 @@ def matrix_server_command_add(args): password = args[3] return_code = W.config_option_set( - server.config.options["password"], + server.config._option_ptrs["password"], password, 1 ) diff --git a/matrix/completion.py b/matrix/completion.py index a15d9d2..ce5c471 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -17,7 +17,7 @@ from __future__ import unicode_literals from matrix.utf import utf8_decode -from matrix.globals import W, SERVERS, OPTIONS +from matrix.globals import W, SERVERS from matrix.utils import tags_from_line_data @@ -82,6 +82,10 @@ def matrix_debug_completion_cb(data, completion_item, buffer, completion): return W.WEECHAT_RC_OK +# TODO this should be configurable +REDACTION_COMP_LEN = 50 + + @utf8_decode def matrix_message_completion_cb(data, completion_item, buffer, completion): own_lines = W.hdata_pointer(W.hdata_get('buffer'), buffer, 'own_lines') @@ -103,8 +107,8 @@ def matrix_message_completion_cb(data, completion_item, buffer, completion): if (message and 'matrix_message' in tags and 'matrix_redacted' not in tags): - if len(message) > OPTIONS.redaction_comp_len + 2: - message = (message[:OPTIONS.redaction_comp_len] + '..') + if len(message) > REDACTION_COMP_LEN + 2: + message = (message[:REDACTION_COMP_LEN] + '..') item = ("{number}:\"{message}\"").format( number=line_number, message=message) diff --git a/matrix/config.py b/matrix/config.py index 3382049..06b9c4a 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -14,18 +14,83 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import unicode_literals +# from __future__ import unicode_literals +from builtins import super import nio import logbook -from matrix.plugin_options import (Option, RedactType, ServerBufferType) - -import matrix.globals -from matrix.globals import W, OPTIONS, SERVERS +from . import globals as G +from matrix.globals import W, SERVERS, SCRIPT_NAME from matrix.utf import utf8_decode -from matrix.utils import key_from_value, server_buffer_merge -from matrix.commands import hook_page_up + +from enum import Enum, unique +from collections import namedtuple + + +@unique +class RedactType(Enum): + STRIKETHROUGH = 0 + NOTICE = 1 + DELETE = 2 + + +@unique +class ServerBufferType(Enum): + MERGE_CORE = 0 + MERGE = 1 + INDEPENDENT = 2 + + +@unique +class DebugType(Enum): + MESSAGING = 0 + NETWORK = 1 + TIMING = 2 + + +class Option(namedtuple( + 'Option', + [ + 'name', + 'type', + 'string_values', + 'min', + 'max', + 'value', + 'description', + 'cast_func', + 'change_callback' + ] +)): + __slots__ = () + + def __new__( + cls, + name, + type, + string_values, + min, + max, + value, + description, + cast=None, + change_callback=None + ): + return super().__new__( + cls, + name, + type, + string_values, + min, + max, + value, + description, + cast, + change_callback + ) + + @utf8_decode @@ -47,148 +112,220 @@ def change_log_level(category, level): @utf8_decode -def matrix_config_change_cb(data, option): - option_name = key_from_value(OPTIONS.options, option) - - if option_name == "redactions": - OPTIONS.redaction_type = RedactType(W.config_integer(option)) - - elif option_name == "server_buffer": - OPTIONS.look_server_buf = ServerBufferType(W.config_integer(option)) - for server in SERVERS.values(): - if server.server_buffer: - server_buffer_merge(server.server_buffer) - - elif option_name == "max_initial_sync_events": - OPTIONS.sync_limit = W.config_integer(option) - - elif option_name == "max_backlog_sync_events": - OPTIONS.backlog_limit = W.config_integer(option) - - elif option_name == "debug_level": - value = W.config_integer(option) - if value == 0: - OPTIONS.debug_level = logbook.ERROR - elif value == 1: - OPTIONS.debug_level = logbook.WARNING - elif value == 2: - OPTIONS.debug_level = logbook.INFO - elif value == 3: - OPTIONS.debug_level = logbook.DEBUG - - change_log_level(OPTIONS.debug_category, OPTIONS.debug_level) - - elif option_name == "debug_category": - value = W.config_integer(option) - change_log_level(OPTIONS.debug_category, logbook.ERROR) - - if value == 0: - OPTIONS.debug_category = "all" - elif value == 1: - OPTIONS.debug_category = "http" - elif value == 2: - OPTIONS.debug_category = "client" - elif value == 3: - OPTIONS.debug_category = "events" - elif value == 4: - OPTIONS.debug_category = "responses" - - change_log_level(OPTIONS.debug_category, OPTIONS.debug_level) - - elif option_name == "debug_buffer": - OPTIONS.debug_buffer = W.config_boolean(option) - - elif option_name == "fetch_backlog_on_pgup": - OPTIONS.enable_backlog = W.config_boolean(option) - - if OPTIONS.enable_backlog: - if not OPTIONS.page_up_hook: - hook_page_up(matrix.globals.CONFIG) - else: - if OPTIONS.page_up_hook: - W.unhook(OPTIONS.page_up_hook) - OPTIONS.page_up_hook = None - +def config_server_buffer_cb(data, option): + for server in SERVERS.values(): + server.buffer_merge() return 1 -def matrix_config_init(config_file): - look_options = [ - Option("redactions", "integer", "strikethrough|notice|delete", 0, 0, - "strikethrough", - ("Only notice redactions, strike through or delete " - "redacted messages")), - Option("server_buffer", "integer", - "merge_with_core|merge_without_core|independent", 0, 0, - "merge_with_core", "Merge server buffers") - ] - - network_options = [ - Option("max_initial_sync_events", "integer", "", 1, 10000, "30", - ("How many events to fetch during the initial sync")), - Option("max_backlog_sync_events", "integer", "", 1, 100, "10", - ("How many events to fetch during backlog fetching")), - Option("fetch_backlog_on_pgup", "boolean", "", 0, 0, "on", - ("Fetch messages in the backlog on a window page up event")), - Option("debug_level", "integer", "error|warn|info|debug", 0, 0, - "off", "Enable network protocol debugging."), - Option("debug_category", "integer", "all|http|client|events|responses", - 0, 0, "all", "Debugging category"), - Option("debug_buffer", "boolean", "", 0, 0, "off", - ("Use a separate buffer for debug logs.")), - ] - - color_options = [ - Option("quote", "color", "", 0, 0, "lightgreen", - ("Color for matrix style blockquotes")) - ] - - def add_global_options(section, options): - for option in options: - OPTIONS.options[option.name] = W.config_new_option( - config_file, section, option.name, option.type, - option.description, option.string_values, option.min, - option.max, option.value, option.value, 0, "", "", - "matrix_config_change_cb", "", "", "") - - section = W.config_new_section(config_file, "color", 0, 0, "", "", "", "", - "", "", "", "", "", "") - - add_global_options(section, color_options) - - section = W.config_new_section(config_file, "look", 0, 0, "", "", "", "", - "", "", "", "", "", "") - - add_global_options(section, look_options) - - section = W.config_new_section(config_file, "network", 0, 0, "", "", "", - "", "", "", "", "", "", "") - - add_global_options(section, network_options) - - W.config_new_section( - config_file, "server", 0, 0, "matrix_config_server_read_cb", "", - "matrix_config_server_write_cb", "", "", "", "", "", "", "") - - return config_file +@utf8_decode +def config_log_level_cb(data, option): + change_log_level( + G.CONFIG.network.debug_category, + G.CONFIG.network.debug_level + ) + return 1 -def matrix_config_read(config): - # type: (str) -> bool - return_code = W.config_read(config) - if return_code == W.WEECHAT_CONFIG_READ_OK: - return True - elif return_code == W.WEECHAT_CONFIG_READ_MEMORY_ERROR: +@utf8_decode +def config_log_category_cb(data, option): + change_log_level(G.CONFIG.debug_category, logbook.ERROR) + G.CONFIG.debug_category = G.CONFIG.network.debug_category + change_log_level( + G.CONFIG.network.debug_category, + G.CONFIG.network.debug_level + ) + return 1 + + +def level_to_logbook(value): + if value == 0: + return logbook.ERROR + elif value == 1: + return logbook.WARNING + elif value == 2: + return logbook.INFO + elif value == 3: + return logbook.DEBUG + + return logbook.ERROR + + +def logbook_category(value): + if value == 0: + return "all" + elif value == 1: + return "http" + elif value == 2: + return "client" + elif value == 3: + return "events" + elif value == 4: + return "responses" + + return "all" + + +class WeechatConfig(object): + def __init__(self, sections): + self._ptr = W.config_new( + SCRIPT_NAME, + SCRIPT_NAME + "_config_reload_cb", + "" + ) + + for section in sections: + name, options = section + section_class = ConfigSection.build(name, options) + setattr(self, name, section_class(name, self._ptr, options)) + + def free(self): + for section in [getattr(self, a) for a in dir(self) if + isinstance(getattr(self, a), ConfigSection)]: + section.free() + + W.config_free(self._ptr) + + def read(self): + return_code = W.config_read(self._ptr) + if return_code == W.WEECHAT_CONFIG_READ_OK: + return True + elif return_code == W.WEECHAT_CONFIG_READ_MEMORY_ERROR: + return False + elif return_code == W.WEECHAT_CONFIG_READ_FILE_NOT_FOUND: + return True return False - elif return_code == W.WEECHAT_CONFIG_READ_FILE_NOT_FOUND: - return True - return False -def matrix_config_free(config): - for section in ["network", "look", "color", "server"]: - section_pointer = W.config_search_section(config, section) - W.config_section_free_options(section_pointer) - W.config_section_free(section_pointer) +class ConfigSection(object): + @classmethod + def build(cls, name, options): + def constructor(self, name, config_ptr, options): + self._ptr = W.config_new_section(config_ptr, name, 0, 0, "", "", + "", "", "", "", "", "", "", "") + self._config_ptr = config_ptr + self._option_ptrs = {} - W.config_free(config) + for option in options: + self._add_option(option) + + attributes = { + option.name: cls.option_property( + option.name, + option.type, + cast_func=option.cast_func + ) for option in options + } + attributes["__init__"] = constructor + + section_class = type( + name.title() + "Section", + (cls,), + attributes + ) + return section_class + + def free(self): + W.config_section_free_options(self._ptr) + W.config_section_free(self._ptr) + + def _add_option(self, option): + cb = option.change_callback.__name__ if option.change_callback else "" + option_ptr = W.config_new_option( + self._config_ptr, self._ptr, option.name, option.type, + option.description, option.string_values, option.min, + option.max, option.value, option.value, 0, "", "", + cb, "", "", "") + + self._option_ptrs[option.name] = option_ptr + + @staticmethod + def option_property(name, option_type, evaluate=False, cast_func=None): + def bool_getter(self): + return bool(W.config_boolean(self._option_ptrs[name])) + + def str_getter(self): + return W.config_string(self._option_ptrs[name]) + + def str_evaluate_getter(self): + return W.string_eval_expression( + W.config_string(self._option_ptrs[name]), + {}, + {}, + {} + ) + + def int_getter(self): + if cast_func: + return cast_func(W.config_integer(self._option_ptrs[name])) + else: + return W.config_integer(self._option_ptrs[name]) + + if option_type == "string" or option_type == "color": + if evaluate: + return property(str_evaluate_getter) + return property(str_getter) + elif option_type == "boolean": + return property(bool_getter) + elif option_type == "integer": + return property(int_getter) + + +class MatrixConfig(WeechatConfig): + def __init__(self): + + self.debug_buffer = "" + self.debug_category = "all" + + look_options = [ + Option("redactions", "integer", "strikethrough|notice|delete", 0, + 0, "strikethrough", + ("Only notice redactions, strike through or delete " + "redacted messages"), RedactType), + Option("server_buffer", "integer", + "merge_with_core|merge_without_core|independent", 0, 0, + "merge_with_core", "Merge server buffers", ServerBufferType, + config_server_buffer_cb) + ] + + network_options = [ + Option("max_initial_sync_events", "integer", "", 1, 10000, "30", + ("How many events to fetch during the initial sync")), + Option("max_backlog_sync_events", "integer", "", 1, 100, "10", + ("How many events to fetch during backlog fetching")), + Option("fetch_backlog_on_pgup", "boolean", "", 0, 0, "on", + ("Fetch messages in the backlog on a window page up event") + ), + Option("debug_level", "integer", "error|warn|info|debug", 0, 0, + "error", "Enable network protocol debugging.", + level_to_logbook, config_log_level_cb), + Option("debug_category", "integer", + "all|http|client|events|responses", + 0, 0, "all", "Debugging category", logbook_category), + Option("debug_buffer", "boolean", "", 0, 0, "off", + ("Use a separate buffer for debug logs.")), + ] + + color_options = [ + Option("quote", "color", "", 0, 0, "lightgreen", + ("Color for matrix style blockquotes")) + ] + + sections = [ + ("network", network_options), + ("look", look_options), + ("color", color_options) + ] + + super().__init__(sections) + + # The server section is essentially a section with subsections and no + # options, handle that case independently. + W.config_new_section( + self._ptr, "server", 0, 0, "matrix_config_server_read_cb", "", + "matrix_config_server_write_cb", "", "", "", "", "", "", "") + + def free(self): + section_ptr = W.config_search_section(self._ptr, 'server') + W.config_section_free(section_ptr) + super().free() diff --git a/matrix/globals.py b/matrix/globals.py index efca89e..db3f7d4 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -19,7 +19,6 @@ from __future__ import unicode_literals import sys from matrix.utf import WeechatWrapper -from matrix.plugin_options import PluginOptions try: import weechat @@ -28,8 +27,7 @@ except ImportError: import matrix._weechat as weechat W = weechat -OPTIONS = PluginOptions() # type: PluginOptions SERVERS = dict() # type: Dict[str, MatrixServer] -CONFIG = None # type: weechat.config +CONFIG = None # type: MatrixConfig ENCRYPTION = True # type: bool SCRIPT_NAME = "matrix" # type: str diff --git a/matrix/plugin_options.py b/matrix/plugin_options.py deleted file mode 100644 index 7fbd9a6..0000000 --- a/matrix/plugin_options.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2018 Damir Jelić -# -# Permission to use, copy, modify, and/or distribute this software for -# any purpose with or without fee is hereby granted, provided that the -# above copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -from __future__ import unicode_literals - -import logbook - -from collections import namedtuple -from enum import Enum, unique - - -@unique -class RedactType(Enum): - STRIKETHROUGH = 0 - NOTICE = 1 - DELETE = 2 - - -@unique -class ServerBufferType(Enum): - MERGE_CORE = 0 - MERGE = 1 - INDEPENDENT = 2 - - -@unique -class DebugType(Enum): - MESSAGING = 0 - NETWORK = 1 - TIMING = 2 - - -Option = namedtuple( - 'Option', - ['name', 'type', 'string_values', 'min', 'max', 'value', 'description']) - - -class PluginOptions: - - def __init__(self): - self.redaction_type = RedactType.STRIKETHROUGH # type: RedactType - self.look_server_buf = ServerBufferType.MERGE_CORE \ - # type: ServerBufferType - - self.sync_limit = 30 # type: int - self.backlog_limit = 10 # type: int - self.enable_backlog = True # type: bool - self.page_up_hook = None # type: weechat.hook - - self.redaction_comp_len = 50 # type: int - - self.options = dict() # type: Dict[str, weechat.config_option] - self.debug = [] - self.debug_level = logbook.ERROR - self.debug_category = "all" - self.debug_buffer = False - self.debug_buffer_ptr = "" diff --git a/matrix/server.py b/matrix/server.py index 50e81a1..ba7a215 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -29,18 +29,18 @@ from nio import ( LoginResponse, SyncRepsponse, RoomSendResponse, - RoomPutStateResponse, TransportResponse, TransportType, LocalProtocolError ) -from matrix.plugin_options import Option, DebugType -from matrix.utils import (key_from_value, prnt_debug, server_buffer_prnt, +from matrix.utils import (key_from_value, server_buffer_prnt, create_server_buffer) from matrix.utf import utf8_decode -from matrix.globals import W, SERVERS, SCRIPT_NAME, OPTIONS +from . import globals as G +from matrix.globals import W, SERVERS, SCRIPT_NAME from .buffer import RoomBuffer, OwnMessage, OwnAction +from .config import Option, ServerBufferType, ConfigSection try: FileNotFoundError @@ -48,29 +48,12 @@ except NameError: FileNotFoundError = IOError -class ServerConfig(object): - def option_property(name, option_type): - def bool_getter(self): - return bool(W.config_boolean(self.options[name])) - - def str_getter(self): - return W.config_string(self.options[name]) - - def int_getter(self): - return W.config_integer(self.options[name]) - - if option_type == str: - return property(str_getter) - elif option_type == bool: - return property(bool_getter) - elif option_type == int: - return property(int_getter) - +class ServerConfig(ConfigSection): def __init__(self, server_name, config_ptr): # type: (str, str) -> None self._server_name = server_name - self._ptr = config_ptr - self.options = {} + self._config_ptr = config_ptr + self._option_ptrs = {} options = [ Option('autoconnect', 'boolean', '', 0, 0, 'off', @@ -95,37 +78,33 @@ class ServerConfig(object): ] section = W.config_search_section(config_ptr, 'server') + self._ptr = section for option in options: option_name = "{server}.{option}".format( server=self._server_name, option=option.name) - self.options[option.name] = W.config_new_option( + self._option_ptrs[option.name] = W.config_new_option( config_ptr, section, option_name, option.type, option.description, option.string_values, option.min, option.max, option.value, option.value, 0, "", "", "matrix_config_server_change_cb", self._server_name, "", "") - def _get_str_option(self, option_name): - return W.config_string(self.options[option_name]) + autoconnect = ConfigSection.option_property("autoconnect", "boolean") + address = ConfigSection.option_property("address", "string") + port = ConfigSection.option_property("port", "integer") + proxy = ConfigSection.option_property("proxy", "string") + ssl_verify = ConfigSection.option_property("ssl_verify", "boolean") + username = ConfigSection.option_property("username", "string") + device_name = ConfigSection.option_property("device_name", "string") + password = ConfigSection.option_property( + "password", + "string", + evaluate=True + ) - autoconnect = option_property("autoconnect", bool) - address = option_property("address", str) - port = option_property("port", int) - proxy = option_property("proxy", str) - ssl_verify = option_property("ssl_verify", bool) - username = option_property("username", str) - device_name = option_property("device_name", str) - - @property - def password(self): - # type: () -> str - return W.string_eval_expression( - self._get_str_option("password"), - {}, - {}, - {} - ) + def free(self): + W.config_section_free_options(self._ptr) class MatrixServer(object): @@ -609,7 +588,13 @@ class MatrixServer(object): # self.store_olm() # self.upload_keys(device_keys=True, one_time_keys=False) - sync_filter = {"room": {"timeline": {"limit": OPTIONS.sync_limit}}} + sync_filter = { + "room": { + "timeline": { + "limit": G.CONFIG.network.max_initial_sync_events + } + } + } self.sync(timeout=0, filter=sync_filter) def _handle_room_info(self, response): @@ -718,6 +703,35 @@ class MatrixServer(object): room_buffer = self.room_buffers[room_id] return room_buffer + def buffer_merge(self): + if not self.server_buffer: + return + + buf = self.server_buffer + + if G.CONFIG.look.server_buffer == ServerBufferType.MERGE_CORE: + num = W.buffer_get_integer(W.buffer_search_main(), "number") + W.buffer_unmerge(buf, num + 1) + W.buffer_merge(buf, W.buffer_search_main()) + elif G.CONFIG.look.server_buffer == ServerBufferType.MERGE: + if SERVERS: + first = None + for server in SERVERS.values(): + if server.server_buffer: + first = server.server_buffer + break + if first: + num = W.buffer_get_integer( + W.buffer_search_main(), + "number" + ) + W.buffer_unmerge(buf, num + 1) + if buf is not first: + W.buffer_merge(buf, first) + else: + num = W.buffer_get_integer(W.buffer_search_main(), "number") + W.buffer_unmerge(buf, num + 1) + @utf8_decode def matrix_config_server_read_cb(data, config_file, section, option_name, @@ -736,9 +750,9 @@ def matrix_config_server_read_cb(data, config_file, section, option_name, SERVERS[server.name] = server # Ignore invalid options - if option in server.config.options: + if option in server.config._option_ptrs: return_code = W.config_option_set( - server.config.options[option], + server.config._option_ptrs[option], value, 1 ) @@ -754,7 +768,7 @@ def matrix_config_server_write_cb(data, config_file, section_name): return W.WECHAT_CONFIG_WRITE_ERROR for server in SERVERS.values(): - for option in server.config.options.values(): + for option in server.config._option_ptrs.values(): if not W.config_write_option(config_file, option): return W.WECHAT_CONFIG_WRITE_ERROR @@ -770,7 +784,7 @@ def matrix_config_server_change_cb(server_name, option): # The function config_option_get_string() is used to get differing # properties from a config option, sadly it's only available in the plugin # API of weechat. - option_name = key_from_value(server.config.options, option) + option_name = key_from_value(server.config._option_ptrs, option) server.update_option(option, option_name) return 1 @@ -807,11 +821,6 @@ def matrix_timer_cb(server_name, remaining_calls): while server.send_queue: message = server.send_queue.popleft() - prnt_debug( - DebugType.MESSAGING, - server, ("Timer hook found message of type {t} in queue. Sending " - "out.".format(t=message.__class__.__name__))) - if not server.send(message): # We got an error while sending the last message return the message # to the queue and exit the loop @@ -835,7 +844,7 @@ def matrix_timer_cb(server_name, remaining_calls): def create_default_server(config_file): - server = MatrixServer('matrix_org', config_file) + server = MatrixServer('matrix_org', config_file._ptr) SERVERS[server.name] = server option = W.config_get(SCRIPT_NAME + ".server." + server.name + ".address") diff --git a/matrix/utils.py b/matrix/utils.py index 3f3662b..7172b4e 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -20,9 +20,8 @@ from builtins import str import time import math -from matrix.globals import W, SERVERS, OPTIONS - -from matrix.plugin_options import ServerBufferType +from matrix import globals as G +from matrix.globals import W, SERVERS try: from urlparse import urlparse @@ -35,11 +34,6 @@ def key_from_value(dictionary, value): return list(dictionary.keys())[list(dictionary.values()).index(value)] -def prnt_debug(debug_type, server, message): - if debug_type in OPTIONS.debug: - W.prnt(server.server_buffer, message) - - def server_buffer_prnt(server, string): # type: (MatrixServer, str) -> None assert server.server_buffer @@ -79,29 +73,7 @@ def create_server_buffer(server): W.buffer_set(server.server_buffer, "localvar_set_server", server.name) W.buffer_set(server.server_buffer, "localvar_set_channel", server.name) - server_buffer_merge(server.server_buffer) - - -def server_buffer_merge(buffer): - if OPTIONS.look_server_buf == ServerBufferType.MERGE_CORE: - num = W.buffer_get_integer(W.buffer_search_main(), "number") - W.buffer_unmerge(buffer, num + 1) - W.buffer_merge(buffer, W.buffer_search_main()) - elif OPTIONS.look_server_buf == ServerBufferType.MERGE: - if SERVERS: - first = None - for server in SERVERS.values(): - if server.server_buffer: - first = server.server_buffer - break - if first: - num = W.buffer_get_integer(W.buffer_search_main(), "number") - W.buffer_unmerge(buffer, num + 1) - if buffer is not first: - W.buffer_merge(buffer, first) - else: - num = W.buffer_get_integer(W.buffer_search_main(), "number") - W.buffer_unmerge(buffer, num + 1) + server.buffer_merge() def server_buffer_set_title(server): From f8a318fef25f0644dd57f6c2a4598101807e55f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Aug 2018 15:57:39 +0200 Subject: [PATCH 119/269] colors: Fix blockquote coloring. --- matrix/colors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 730ec27..6ba9bff 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -54,8 +54,8 @@ class Formatted(): def textwrapper(self): return textwrap.TextWrapper( width=67, - initial_indent="{}> ".format(G.CONFIG.color.quote), - subsequent_indent="{}> ".format(G.CONFIG.color.quote) + initial_indent="{}> ".format(W.color(G.CONFIG.color.quote)), + subsequent_indent="{}> ".format(W.color(G.CONFIG.color.quote)) ) def is_formatted(self): From 05a413f7cbe89a44f2e8d3cde9fb17f2c267bac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Aug 2018 19:40:59 +0200 Subject: [PATCH 120/269] matrix: Style fixes and cleanup. --- .pylintrc | 3 +- main.py | 97 +++---- matrix/_weechat.py | 53 ++-- matrix/bar_items.py | 40 +-- matrix/buffer.py | 498 ++++++++++++++++------------------ matrix/colors.py | 127 +++++---- matrix/commands.py | 631 ++++++++++++++++++++++++------------------- matrix/completion.py | 157 +++++++---- matrix/config.py | 272 ++++++++++++------- matrix/globals.py | 8 +- matrix/server.py | 386 +++++++++++++++----------- matrix/utf.py | 11 +- matrix/utils.py | 273 +------------------ 13 files changed, 1278 insertions(+), 1278 deletions(-) diff --git a/.pylintrc b/.pylintrc index 3059c6d..0caa73e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -130,7 +130,8 @@ disable=print-statement, bad-whitespace, too-few-public-methods, too-many-lines, - missing-docstring + missing-docstring, + bad-continuation, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/main.py b/main.py index 91abc9e..323e572 100644 --- a/main.py +++ b/main.py @@ -19,72 +19,51 @@ from __future__ import unicode_literals import socket import ssl -import time -import pprint -import OpenSSL.crypto as crypto import textwrap -from itertools import chain - # pylint: disable=redefined-builtin from builtins import str -from future.utils import bytes_to_native_str as n - +from itertools import chain # pylint: disable=unused-import -from typing import (List, Set, Dict, Tuple, Text, Optional, AnyStr, Deque, Any) +from typing import Any, AnyStr, Deque, Dict, List, Optional, Set, Text, Tuple + import logbook -from logbook import Logger, StderrHandler, StreamHandler - -import nio -from nio import TransportType, RemoteTransportError, RemoteProtocolError - -from matrix.colors import Formatted -from matrix.utf import utf8_decode - -# Weechat searches for the registered callbacks in the scope of the main script -# file, import the callbacks here so weechat can find them. -from matrix.commands import (hook_commands, hook_page_up, matrix_command_cb, - matrix_topic_command_cb, matrix_join_command_cb, - matrix_part_command_cb, matrix_invite_command_cb, - matrix_command_pgup_cb, matrix_redact_command_cb, - matrix_command_buf_clear_cb, matrix_me_command_cb, - matrix_kick_command_cb) -from matrix.buffer import room_buffer_input_cb, room_buffer_close_cb - -from matrix.server import ( - MatrixServer, - create_default_server, - send_cb, - matrix_timer_cb, - matrix_config_server_read_cb, - matrix_config_server_write_cb, - matrix_config_server_change_cb, -) - -from matrix.bar_items import (init_bar_items, matrix_bar_item_name, - matrix_bar_item_plugin, matrix_bar_item_lag, - matrix_bar_item_buffer_modes) - -from matrix.completion import ( - init_completion, matrix_command_completion_cb, - matrix_server_command_completion_cb, matrix_debug_completion_cb, - matrix_message_completion_cb, matrix_server_completion_cb, - matrix_olm_user_completion_cb, matrix_olm_device_completion_cb, - matrix_user_completion_cb) - -from matrix.utils import (key_from_value, server_buffer_prnt, - server_buffer_set_title) - -from matrix.config import ( - matrix_config_reload_cb, - MatrixConfig, - config_log_level_cb, - config_log_category_cb, - config_server_buffer_cb -) +import OpenSSL.crypto as crypto +from future.utils import bytes_to_native_str as n +from logbook import Logger, StreamHandler +from nio import RemoteProtocolError, RemoteTransportError, TransportType from matrix import globals as G - -from matrix.globals import W, SERVERS, SCRIPT_NAME +from matrix.bar_items import (init_bar_items, matrix_bar_item_buffer_modes, + matrix_bar_item_lag, matrix_bar_item_name, + matrix_bar_item_plugin) +from matrix.buffer import room_buffer_close_cb, room_buffer_input_cb +# Weechat searches for the registered callbacks in the scope of the main script +# file, import the callbacks here so weechat can find them. +from matrix.commands import (hook_commands, hook_page_up, + matrix_command_buf_clear_cb, matrix_command_cb, + matrix_command_pgup_cb, matrix_invite_command_cb, + matrix_join_command_cb, matrix_kick_command_cb, + matrix_me_command_cb, matrix_part_command_cb, + matrix_redact_command_cb, matrix_topic_command_cb) +from matrix.completion import (init_completion, matrix_command_completion_cb, + matrix_debug_completion_cb, + matrix_message_completion_cb, + matrix_olm_device_completion_cb, + matrix_olm_user_completion_cb, + matrix_server_command_completion_cb, + matrix_server_completion_cb, + matrix_user_completion_cb) +from matrix.config import (MatrixConfig, config_log_category_cb, + config_log_level_cb, config_server_buffer_cb, + matrix_config_reload_cb) +from matrix.globals import SCRIPT_NAME, SERVERS, W +from matrix.server import (MatrixServer, create_default_server, + matrix_config_server_change_cb, + matrix_config_server_read_cb, + matrix_config_server_write_cb, matrix_timer_cb, + send_cb) +from matrix.utf import utf8_decode +from matrix.utils import server_buffer_prnt, server_buffer_set_title # yapf: disable WEECHAT_SCRIPT_NAME = SCRIPT_NAME diff --git a/matrix/_weechat.py b/matrix/_weechat.py index 6914e37..a4ab139 100644 --- a/matrix/_weechat.py +++ b/matrix/_weechat.py @@ -1,9 +1,8 @@ +import datetime import random import string -import datetime - -weechat_base_colors = { +WEECHAT_BASE_COLORS = { "black": "0", "red": "1", "green": "2", @@ -29,11 +28,11 @@ def color(color_name): escape_codes = [] reset_code = "0" - def make_fg_color(color): - return "38;5;{}".format(color) + def make_fg_color(color_code): + return "38;5;{}".format(color_code) - def make_bg_color(color): - return "48;5;{}".format(color) + def make_bg_color(color_code): + return "48;5;{}".format(color_code) attributes = { "bold": "1", @@ -76,21 +75,21 @@ def color(color_name): stripped_color = fg_color.lstrip("*_|/!") - if stripped_color in weechat_base_colors: + if stripped_color in WEECHAT_BASE_COLORS: escape_codes.append( - make_fg_color(weechat_base_colors[stripped_color])) + make_fg_color(WEECHAT_BASE_COLORS[stripped_color])) elif stripped_color.isdigit(): num_color = int(stripped_color) - if num_color >= 0 and num_color < 256: + if 0 <= num_color < 256: escape_codes.append(make_fg_color(stripped_color)) - if bg_color in weechat_base_colors: - escape_codes.append(make_bg_color(weechat_base_colors[bg_color])) + if bg_color in WEECHAT_BASE_COLORS: + escape_codes.append(make_bg_color(WEECHAT_BASE_COLORS[bg_color])) else: if bg_color.isdigit(): num_color = int(bg_color) - if num_color >= 0 and num_color < 256: + if 0 <= num_color < 256: escape_codes.append(make_bg_color(bg_color)) escape_string = "\033[{}{}m".format(reset_code, ";".join(escape_codes)) @@ -98,7 +97,7 @@ def color(color_name): return escape_string -def prefix(prefix): +def prefix(prefix_string): prefix_to_symbol = { "error": "=!=", "network": "--", @@ -107,14 +106,14 @@ def prefix(prefix): "quit": "<--" } - if prefix in prefix_to_symbol: + if prefix_string in prefix_to_symbol: return prefix_to_symbol[prefix] return "" -def prnt(_, string): - print(string) +def prnt(_, message): + print(message) def prnt_date_tags(_, date, tags_string, data): @@ -126,44 +125,44 @@ def prnt_date_tags(_, date, tags_string, data): print(message) -def config_search_section(*args, **kwargs): +def config_search_section(*_, **__): pass -def config_new_option(*args, **kwargs): +def config_new_option(*_, **__): pass -def mkdir_home(*args, **kwargs): +def mkdir_home(*_, **__): return True -def info_get(info, *args): +def info_get(info, *_): if info == "nick_color_name": - return random.choice(list(weechat_base_colors.keys())) + return random.choice(list(WEECHAT_BASE_COLORS.keys())) return "" -def buffer_new(*args, **kwargs): +def buffer_new(*_, **__): return "".join( random.choice(string.ascii_uppercase + string.digits) for _ in range(8) ) -def buffer_set(*args, **kwargs): +def buffer_set(*_, **__): return -def nicklist_add_group(*args, **kwargs): +def nicklist_add_group(*_, **__): return -def nicklist_add_nick(*args, **kwargs): +def nicklist_add_nick(*_, **__): return -def nicklist_remove_nick(*args, **kwargs): +def nicklist_remove_nick(*_, **__): return diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 5a45547..56d1965 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -16,21 +16,20 @@ from __future__ import unicode_literals -from matrix.utf import utf8_decode - -from matrix.globals import W, SERVERS +from .globals import SERVERS, W +from .utf import utf8_decode @utf8_decode def matrix_bar_item_plugin(data, item, window, buffer, extra_info): # pylint: disable=unused-argument for server in SERVERS.values(): - if (buffer in server.buffers.values() or - buffer == server.server_buffer): + if buffer in server.buffers.values() or buffer == server.server_buffer: return "matrix{color}/{color_fg}{name}".format( color=W.color("bar_delim"), color_fg=W.color("bar_fg"), - name=server.name) + name=server.name, + ) return "" @@ -40,24 +39,31 @@ def matrix_bar_item_name(data, item, window, buffer, extra_info): # pylint: disable=unused-argument for server in SERVERS.values(): if buffer in server.buffers.values(): - color = ("status_name_ssl" - if server.ssl_context.check_hostname else "status_name") + color = ( + "status_name_ssl" + if server.ssl_context.check_hostname + else "status_name" + ) room_buffer = server.find_room_from_ptr(buffer) room = room_buffer.room return "{color}{name}".format( - color=W.color(color), - name=room.display_name()) + color=W.color(color), name=room.display_name() + ) - elif buffer == server.server_buffer: - color = ("status_name_ssl" - if server.ssl_context.check_hostname else "status_name") + if buffer == server.server_buffer: + color = ( + "status_name_ssl" + if server.ssl_context.check_hostname + else "status_name" + ) return "{color}server{del_color}[{color}{name}{del_color}]".format( color=W.color(color), del_color=W.color("bar_delim"), - name=server.name) + name=server.name, + ) return "" @@ -66,8 +72,7 @@ def matrix_bar_item_name(data, item, window, buffer, extra_info): def matrix_bar_item_lag(data, item, window, buffer, extra_info): # pylint: disable=unused-argument for server in SERVERS.values(): - if (buffer in server.buffers.values() or - buffer == server.server_buffer): + if buffer in server.buffers.values() or buffer == server.server_buffer: if server.lag >= 500: color = W.color("irc.color.item_lag_counting") if server.lag_done: @@ -77,7 +82,8 @@ def matrix_bar_item_lag(data, item, window, buffer, extra_info): lag_string = "Lag: {color}{lag}{ncolor}".format( lag=lag.format((server.lag / 1000)), color=color, - ncolor=W.color("reset")) + ncolor=W.color("reset"), + ) return lag_string return "" diff --git a/matrix/buffer.py b/matrix/buffer.py index e8928e1..c9fffe7 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -22,42 +22,40 @@ from builtins import super from functools import partial from typing import NamedTuple -from . import globals as G -from .globals import W, SERVERS, SCRIPT_NAME -from .utf import utf8_decode -from .colors import Formatted -from .utils import ( - shorten_sender, - server_ts_to_weechat, - string_strikethrough, -) -from .config import RedactType - from nio import ( Api, - RoomMessageText, - RoomMemberEvent, PowerLevelsEvent, - RoomEncryptionEvent, RedactedEvent, + RedactionEvent, RoomAliasEvent, - RoomTopicEvent, + RoomEncryptionEvent, + RoomMemberEvent, RoomMessageEmote, - RoomNameEvent, RoomMessageMedia, RoomMessageNotice, + RoomMessageText, RoomMessageUnknown, - RedactionEvent + RoomNameEvent, + RoomTopicEvent, ) +from . import globals as G +from .colors import Formatted +from .config import RedactType +from .globals import SCRIPT_NAME, SERVERS, W +from .utf import utf8_decode +from .utils import server_ts_to_weechat, shorten_sender, string_strikethrough -OwnMessage = NamedTuple("OwnMessage", [ - ("sender", str), - ("age", int), - ("event_id", str), - ("room_id", str), - ("formatted_message", Formatted) -]) +OwnMessage = NamedTuple( + "OwnMessage", + [ + ("sender", str), + ("age", int), + ("event_id", str), + ("room_id", str), + ("formatted_message", Formatted), + ], +) class OwnAction(OwnMessage): @@ -71,7 +69,7 @@ def room_buffer_input_cb(server_name, buffer, input_data): if not room_buffer: # TODO log error - return + return W.WEECHAT_RC_ERROR if not server.connected: room_buffer.error("You are not connected to the server") @@ -96,7 +94,7 @@ def room_buffer_close_cb(data, buffer): class WeechatUser(object): def __init__(self, nick, host=None, prefix="", join_time=None): - # type: (str, str, str) -> None + # type: (str, str, str, int) -> None self.nick = nick self.host = host self.prefix = prefix @@ -127,7 +125,7 @@ class WeechatUser(object): class RoomUser(WeechatUser): def __init__(self, nick, user_id=None, power_level=0, join_time=None): - # type: (str, str, int) -> None + # type: (str, str, int, int) -> None prefix = self._get_prefix(power_level) super().__init__(nick, user_id, prefix, join_time) @@ -152,26 +150,22 @@ class RoomUser(WeechatUser): # type: (int) -> str if power_level >= 100: return "&" - elif power_level >= 50: + if power_level >= 50: return "@" - elif power_level > 0: + if power_level > 0: return "+" return "" class WeechatChannelBuffer(object): tags = { - "message": [ - SCRIPT_NAME + "_message", - "notify_message", - "log1" - ], + "message": [SCRIPT_NAME + "_message", "notify_message", "log1"], "self_message": [ SCRIPT_NAME + "_message", "notify_none", "no_highlight", "self_msg", - "log1" + "log1", ], "action": [ SCRIPT_NAME + "_message", @@ -179,44 +173,25 @@ class WeechatChannelBuffer(object): "notify_message", "log1", ], - "notice": [ - SCRIPT_NAME + "_notice", - "notify_message", - "log1", - ], + "notice": [SCRIPT_NAME + "_notice", "notify_message", "log1"], "old_message": [ SCRIPT_NAME + "_message", "notify_message", "no_log", - "no_highlight" + "no_highlight", ], - "join": [ - SCRIPT_NAME + "_join", - "log4" - ], - "part": [ - SCRIPT_NAME + "_leave", - "log4" - ], - "kick": [ - SCRIPT_NAME + "_kick", - "log4" - ], - "invite": [ - SCRIPT_NAME + "_invite", - "log4" - ], - "topic": [ - SCRIPT_NAME + "_topic", - "log3", - ] + "join": [SCRIPT_NAME + "_join", "log4"], + "part": [SCRIPT_NAME + "_leave", "log4"], + "kick": [SCRIPT_NAME + "_kick", "log4"], + "invite": [SCRIPT_NAME + "_invite", "log4"], + "topic": [SCRIPT_NAME + "_topic", "log3"], } membership_messages = { "join": "has joined", "part": "has left", "kick": "has been kicked from", - "invite": "has been invited to" + "invite": "has been invited to", } class Line(object): @@ -249,9 +224,7 @@ class WeechatChannelBuffer(object): @property def tags(self): tags_count = W.hdata_get_var_array_size( - self._hdata, - self._ptr, - "tags_array" + self._hdata, self._ptr, "tags_array" ) tags = [ @@ -300,7 +273,7 @@ class WeechatChannelBuffer(object): tags=None, prefix=None, message=None, - highlight=None + highlight=None, ): new_data = {} @@ -309,7 +282,7 @@ class WeechatChannelBuffer(object): if date_printed: new_data["date_printed"] = str(date_printed) if tags: - new_data["tags_array"] = ','.join(tags) + new_data["tags_array"] = ",".join(tags) if prefix: new_data["prefix"] = prefix if message: @@ -337,8 +310,8 @@ class WeechatChannelBuffer(object): self.topic_author = "" self.topic_date = None - W.buffer_set(self._ptr, "localvar_set_type", 'channel') - W.buffer_set(self._ptr, "type", 'formatted') + W.buffer_set(self._ptr, "localvar_set_type", "channel") + W.buffer_set(self._ptr, "type", "formatted") W.buffer_set(self._ptr, "localvar_set_channel", name) @@ -350,32 +323,16 @@ class WeechatChannelBuffer(object): # W.buffer_set(self._ptr, "short_name", short_name) W.nicklist_add_group( - self._ptr, - '', - "000|o", - "weechat.color.nicklist_group", - 1 + self._ptr, "", "000|o", "weechat.color.nicklist_group", 1 ) W.nicklist_add_group( - self._ptr, - '', - "001|h", - "weechat.color.nicklist_group", - 1 + self._ptr, "", "001|h", "weechat.color.nicklist_group", 1 ) W.nicklist_add_group( - self._ptr, - '', - "002|v", - "weechat.color.nicklist_group", - 1 + self._ptr, "", "002|v", "weechat.color.nicklist_group", 1 ) W.nicklist_add_group( - self._ptr, - '', - "999|...", - "weechat.color.nicklist_group", - 1 + self._ptr, "", "999|...", "weechat.color.nicklist_group", 1 ) W.buffer_set(self._ptr, "nicklist", "1") @@ -385,9 +342,7 @@ class WeechatChannelBuffer(object): # TODO make this configurable W.buffer_set( - self._ptr, - "highlight_tags_restrict", - SCRIPT_NAME + "_message" + self._ptr, "highlight_tags_restrict", SCRIPT_NAME + "_message" ) @property @@ -428,23 +383,18 @@ class WeechatChannelBuffer(object): @property def lines(self): - own_lines = W.hdata_pointer( - self._hdata, - self._ptr, - "own_lines" - ) + own_lines = W.hdata_pointer(self._hdata, self._ptr, "own_lines") if own_lines: hdata_line = W.hdata_get("line") line_pointer = W.hdata_pointer( - W.hdata_get("lines"), own_lines, "last_line") + W.hdata_get("lines"), own_lines, "last_line" + ) while line_pointer: data_pointer = W.hdata_pointer( - hdata_line, - line_pointer, - "data" + hdata_line, line_pointer, "data" ) if data_pointer: @@ -469,9 +419,7 @@ class WeechatChannelBuffer(object): # type: (str) -> None """ Print an error to the room buffer """ message = "{prefix}{script}: {message}".format( - prefix=W.prefix("error"), - script=SCRIPT_NAME, - message=string + prefix=W.prefix("error"), script=SCRIPT_NAME, message=string ) self._print(message) @@ -493,7 +441,7 @@ class WeechatChannelBuffer(object): color = self._color_for_tags(user.color) - if message_type != "action" and message_type != "notice": + if message_type not in ("action", "notice"): tags.append("prefix_nick_{color}".format(color=color)) return tags @@ -507,18 +455,23 @@ class WeechatChannelBuffer(object): return WeechatUser(nick) def _print_message(self, user, message, date, tags): - prefix_string = ("" if not user.prefix else "{}{}{}".format( - W.color(self._get_prefix_color(user.prefix)), - user.prefix, - W.color("reset") - )) + prefix_string = ( + "" + if not user.prefix + else "{}{}{}".format( + W.color(self._get_prefix_color(user.prefix)), + user.prefix, + W.color("reset"), + ) + ) data = "{prefix}{color}{author}{ncolor}\t{msg}".format( prefix=prefix_string, color=W.color(user.color), author=user.nick, ncolor=W.color("reset"), - msg=message) + msg=message, + ) self.print_date_tags(data, date, tags) @@ -534,27 +487,32 @@ class WeechatChannelBuffer(object): def notice(self, nick, message, date, extra_tags=None): # type: (str, str, int, Optional[List[str]]) -> None user = self._get_user(nick) - user_prefix = ("" if not user.prefix else "{}{}{}".format( - W.color(self._get_prefix_color(user.prefix)), - user.prefix, - W.color("reset") - )) - - user_string = "{}{}{}{}".format( - user_prefix, - user.color, - user.nick, - W.color("reset") + user_prefix = ( + "" + if not user.prefix + else "{}{}{}".format( + W.color(self._get_prefix_color(user.prefix)), + user.prefix, + W.color("reset"), + ) ) - data = ("{prefix}\t{color}Notice" - "{del_color}({ncolor}{user}{del_color}){ncolor}" - ": {message}").format(prefix=W.prefix("network"), - color=W.color("irc.color.notice"), - del_color=W.color("chat_delimiters"), - ncolor=W.color("reset"), - user=user_string, - message=message) + user_string = "{}{}{}{}".format( + user_prefix, user.color, user.nick, W.color("reset") + ) + + data = ( + "{prefix}\t{color}Notice" + "{del_color}({ncolor}{user}{del_color}){ncolor}" + ": {message}" + ).format( + prefix=W.prefix("network"), + color=W.color("irc.color.notice"), + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + user=user_string, + message=message, + ) tags = self._message_tags(user, "notice") + (extra_tags or []) self.print_date_tags(data, date, tags) @@ -563,27 +521,33 @@ class WeechatChannelBuffer(object): self.unmask_smart_filtered_nick(nick) def _print_action(self, user, message, date, tags): - nick_prefix = ("" if not user.prefix else "{}{}{}".format( - W.color(self._get_prefix_color(user.prefix)), - user.prefix, - W.color("reset") - )) + nick_prefix = ( + "" + if not user.prefix + else "{}{}{}".format( + W.color(self._get_prefix_color(user.prefix)), + user.prefix, + W.color("reset"), + ) + ) - data = ("{prefix}{nick_prefix}{nick_color}{author}" - "{ncolor} {msg}").format( + data = ( + "{prefix}{nick_prefix}{nick_color}{author}" "{ncolor} {msg}" + ).format( prefix=W.prefix("action"), nick_prefix=nick_prefix, nick_color=W.color(user.color), author=user.nick, ncolor=W.color("reset"), - msg=message) + msg=message, + ) self.print_date_tags(data, date, tags) - def action(self, nick, message, date, extra_tags=[]): + def action(self, nick, message, date, extra_tags=None): # type: (str, str, int, Optional[List[str]]) -> None user = self._get_user(nick) - tags = self._message_tags(user, "action") + extra_tags + tags = self._message_tags(user, "action") + (extra_tags or []) self._print_action(user, message, date, tags) user.update_speaking_time(date) @@ -624,9 +588,7 @@ class WeechatChannelBuffer(object): if not nick_pointer: group = W.nicklist_search_group( - self._ptr, - "", - self._get_nicklist_group(user) + self._ptr, "", self._get_nicklist_group(user) ) prefix = user.prefix if user.prefix else " " W.nicklist_add_nick( @@ -636,22 +598,22 @@ class WeechatChannelBuffer(object): user.color, prefix, self._get_prefix_color(user.prefix), - 1 + 1, ) def _membership_message(self, user, message_type): # type: (WeechatUser, str) -> str - action_color = ("green" if message_type == "join" - or message_type == "invite" else "red") - prefix = ("join" if message_type == "join" or message_type == "invite" - else "quit") + action_color = "green" if message_type in ("join", "invite") else "red" + prefix = "join" if message_type in ("join", "invite") else "quit" membership_message = self.membership_messages[message_type] - message = ("{prefix}{color}{author}{ncolor} " - "{del_color}({host_color}{host}{del_color})" - "{action_color} {message} " - "{channel_color}{room}{ncolor}").format( + message = ( + "{prefix}{color}{author}{ncolor} " + "{del_color}({host_color}{host}{del_color})" + "{action_color} {message} " + "{channel_color}{room}{ncolor}" + ).format( prefix=W.prefix(prefix), color=W.color(user.color), author=user.nick, @@ -662,11 +624,12 @@ class WeechatChannelBuffer(object): action_color=W.color(action_color), message=membership_message, channel_color=W.color("chat_channel"), - room=self.short_name) + room=self.short_name, + ) return message - def join(self, user, date, message=True, extra_tags=[]): + def join(self, user, date, message=True, extra_tags=None): # type: (WeechatUser, int, Optional[bool], Optional[List[str]]) -> None self._add_user_to_nicklist(user) self.users[user.nick] = user @@ -681,12 +644,12 @@ class WeechatChannelBuffer(object): self.print_date_tags(message, date, tags) self.add_smart_filtered_nick(user.nick) - def invite(self, nick, date, extra_tags=[]): + def invite(self, nick, date, extra_tags=None): # type: (str, int, Optional[bool], Optional[List[str]]) -> None user = self._get_user(nick) tags = self._message_tags(user, "invite") message = self._membership_message(user, "invite") - self.print_date_tags(message, date, tags + extra_tags) + self.print_date_tags(message, date, tags + (extra_tags or [])) def _remove_user_from_nicklist(self, user): # type: (WeechatUser) -> None @@ -695,7 +658,7 @@ class WeechatChannelBuffer(object): if nick_pointer: W.nicklist_remove_nick(self._ptr, nick_pointer) - def _leave(self, nick, date, message, leave_type, extra_tags): + def _leave(self, nick, date, message, leave_type, extra_tags=None): # type: (str, int, bool, str, List[str]) -> None user = self._get_user(nick) self._remove_user_from_nicklist(user) @@ -708,34 +671,36 @@ class WeechatChannelBuffer(object): tags.append(SCRIPT_NAME + "_smart_filter") message = self._membership_message(user, leave_type) - self.print_date_tags(message, date, tags + extra_tags) + self.print_date_tags(message, date, tags + (extra_tags or [])) self.remove_smart_filtered_nick(user.nick) if user.nick in self.users: del self.users[user.nick] - def part(self, nick, date, message=True, extra_tags=[]): + def part(self, nick, date, message=True, extra_tags=None): # type: (str, int, Optional[bool], Optional[List[str]]) -> None self._leave(nick, date, message, "part", extra_tags) - def kick(self, nick, date, message=True, extra_tags=[]): + def kick(self, nick, date, message=True, extra_tags=None): # type: (str, int, Optional[bool], Optional[List[str]]) -> None - self._leave(nick, date, message, "kick", extra_tags=[]) + self._leave(nick, date, message, "kick", extra_tags) def _print_topic(self, nick, topic, date): user = self._get_user(nick) tags = self._message_tags(user, "topic") - data = ("{prefix}{nick} has changed " - "the topic for {chan_color}{room}{ncolor} " - "to \"{topic}\"").format( - prefix=W.prefix("network"), - nick=user.nick, - chan_color=W.color("chat_channel"), - ncolor=W.color("reset"), - room=self.short_name, - topic=topic - ) + data = ( + "{prefix}{nick} has changed " + "the topic for {chan_color}{room}{ncolor} " + 'to "{topic}"' + ).format( + prefix=W.prefix("network"), + nick=user.nick, + chan_color=W.color("chat_channel"), + ncolor=W.color("reset"), + room=self.short_name, + topic=topic, + ) self.print_date_tags(data, date, tags) user.update_speaking_time(date) @@ -799,9 +764,7 @@ class RoomBuffer(object): self.displayed_nicks = {} user = shorten_sender(self.room.own_user_id) self.weechat_buffer = WeechatChannelBuffer( - buffer_name, - server_name, - user + buffer_name, server_name, user ) def find_nick(self, user_id): @@ -837,11 +800,7 @@ class RoomBuffer(object): buffer_user.color = "weechat.color.chat_nick_self" user.nick_color = "weechat.color.chat_nick_self" - self.weechat_buffer.join( - buffer_user, - date, - not is_state - ) + self.weechat_buffer.join(buffer_user, date, not is_state) date = server_ts_to_weechat(event.server_timestamp) @@ -909,22 +868,28 @@ class RoomBuffer(object): message = line.message tags = line.tags - reason = ("" if not event.reason else - ", reason: \"{reason}\"".format(reason=event.reason)) + reason = ( + "" + if not event.reason + else ', reason: "{reason}"'.format(reason=event.reason) + ) - redaction_msg = ("{del_color}<{log_color}Message redacted by: " - "{censor}{log_color}{reason}{del_color}>" - "{ncolor}").format( - del_color=W.color("chat_delimiters"), - ncolor=W.color("reset"), - log_color=W.color("logger.color.backlog_line"), - censor=censor, - reason=reason) + redaction_msg = ( + "{del_color}<{log_color}Message redacted by: " + "{censor}{log_color}{reason}{del_color}>" + "{ncolor}" + ).format( + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + log_color=W.color("logger.color.backlog_line"), + censor=censor, + reason=reason, + ) new_message = "" if G.CONFIG.look.redaction_type == RedactType.STRIKETHROUGH: - plaintext_msg = W.string_remove_color(message, '') + plaintext_msg = W.string_remove_color(message, "") new_message = string_strikethrough(plaintext_msg) elif G.CONFIG.look.redaction_type == RedactType.NOTICE: new_message = message @@ -944,18 +909,24 @@ class RoomBuffer(object): tags = self.get_event_tags(event) tags.append(SCRIPT_NAME + "_redacted") - reason = (", reason: \"{reason}\"".format(reason=event.reason) - if event.reason else "") + reason = ( + ', reason: "{reason}"'.format(reason=event.reason) + if event.reason + else "" + ) censor = self.find_nick(event.redacter) - data = ("{del_color}<{log_color}Message redacted by: " - "{censor}{log_color}{reason}{del_color}>{ncolor}").format( - del_color=W.color("chat_delimiters"), - ncolor=W.color("reset"), - log_color=W.color("logger.color.backlog_line"), - censor=censor, - reason=reason) + data = ( + "{del_color}<{log_color}Message redacted by: " + "{censor}{log_color}{reason}{del_color}>{ncolor}" + ).format( + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + log_color=W.color("logger.color.backlog_line"), + censor=censor, + reason=reason, + ) self.weechat_buffer.message(nick, data, date, tags) @@ -966,20 +937,22 @@ class RoomBuffer(object): nick, event.topic, server_ts_to_weechat(event.server_timestamp), - not is_state) + not is_state, + ) @staticmethod def get_event_tags(event): return ["matrix_id_{}".format(event.event_id)] - def _handle_power_level(self, event): + def _handle_power_level(self, _): for user_id in self.room.power_levels.users: if user_id in self.displayed_nicks: nick = self.find_nick(user_id) user = self.weechat_buffer.users[nick] user.power_level = self.room.power_levels.get_user_level( - user_id) + user_id + ) # There is no way to change the group of a user without # removing him from the nicklist @@ -994,9 +967,11 @@ class RoomBuffer(object): elif isinstance(event, PowerLevelsEvent): self._handle_power_level(event) elif isinstance(event, RoomEncryptionEvent): - message = ("This room is encrypted, encryption is " - "currently unsuported. Message sending is disabled for " - "this room.") + message = ( + "This room is encrypted, encryption is " + "currently unsuported. Message sending is disabled for " + "this room." + ) self.weechat_buffer.error(message) def handle_timeline_event(self, event): @@ -1016,10 +991,7 @@ class RoomBuffer(object): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer.action( - nick, - event.body, - date, - self.get_event_tags(event) + nick, event.body, date, self.get_event_tags(event) ) elif isinstance(event, RoomMessageText): @@ -1029,25 +1001,18 @@ class RoomBuffer(object): if event.formatted_body: formatted = Formatted.from_html(event.formatted_body) - data = (formatted.to_weechat() - if formatted else event.body) + data = formatted.to_weechat() if formatted else event.body date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer.message( - nick, - data, - date, - self.get_event_tags(event) + nick, data, date, self.get_event_tags(event) ) elif isinstance(event, RoomMessageNotice): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer.notice( - nick, - event.body, - date, - self.get_event_tags(event) + nick, event.body, date, self.get_event_tags(event) ) elif isinstance(event, RoomMessageMedia): @@ -1056,15 +1021,11 @@ class RoomBuffer(object): http_url = Api.mxc_to_http(event.url) url = http_url if http_url else event.url - description = ("/{}".format(event.body) - if event.body else "") + description = "/{}".format(event.body) if event.body else "" data = "{url}{desc}".format(url=url, desc=description) self.weechat_buffer.message( - nick, - data, - date, - self.get_event_tags(event) + nick, data, date, self.get_event_tags(event) ) elif isinstance(event, RoomMessageUnknown): @@ -1072,10 +1033,7 @@ class RoomBuffer(object): date = server_ts_to_weechat(event.server_timestamp) data = ("Unknown message of type {t}").format(t=event.type) self.weechat_buffer.message( - nick, - data, - date, - self.get_event_tags(event) + nick, data, date, self.get_event_tags(event) ) elif isinstance(event, RedactionEvent): @@ -1085,9 +1043,11 @@ class RoomBuffer(object): self._handle_redacted_message(event) elif isinstance(event, RoomEncryptionEvent): - message = ("This room is encrypted, encryption is " - "currently unsuported. Message sending is disabled for " - "this room.") + message = ( + "This room is encrypted, encryption is " + "currently unsuported. Message sending is disabled for " + "this room." + ) self.weechat_buffer.error(message) elif isinstance(event, PowerLevelsEvent): @@ -1106,8 +1066,9 @@ class RoomBuffer(object): # ) else: - W.prnt("", "Unhandled event of type {}.".format( - type(event).__name__)) + W.prnt( + "", "Unhandled event of type {}.".format(type(event).__name__) + ) def self_message(self, message): # type: (OwnMessage) -> None @@ -1125,10 +1086,7 @@ class RoomBuffer(object): tags = self.get_event_tags(message) self.weechat_buffer.self_action( - nick, - message.formatted_message.to_weechat(), - date, - tags + nick, message.formatted_message.to_weechat(), date, tags ) def old_redacted(self, event): @@ -1136,20 +1094,26 @@ class RoomBuffer(object): SCRIPT_NAME + "_message", "notify_message", "no_log", - "no_highlight" + "no_highlight", ] - reason = (", reason: \"{reason}\"".format(reason=event.reason) - if event.reason else "") + reason = ( + ', reason: "{reason}"'.format(reason=event.reason) + if event.reason + else "" + ) censor = self.find_nick(event.redacter) - data = ("{del_color}<{log_color}Message redacted by: " - "{censor}{log_color}{reason}{del_color}>{ncolor}").format( - del_color=W.color("chat_delimiters"), - ncolor=W.color("reset"), - log_color=W.color("logger.color.backlog_line"), - censor=censor, - reason=reason) + data = ( + "{del_color}<{log_color}Message redacted by: " + "{censor}{log_color}{reason}{del_color}>{ncolor}" + ).format( + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + log_color=W.color("logger.color.backlog_line"), + censor=censor, + reason=reason, + ) tags += self.get_event_tags(event) nick = self.find_nick(event.sender) @@ -1162,12 +1126,15 @@ class RoomBuffer(object): SCRIPT_NAME + "_message", "notify_message", "no_log", - "no_highlight" + "no_highlight", ] tags += self.get_event_tags(event) nick = self.find_nick(event.sender) - data = (event.formatted_message.to_weechat() - if event.formatted_message else event.message) + data = ( + event.formatted_message.to_weechat() + if event.formatted_message + else event.message + ) user = self.weechat_buffer._get_user(nick) date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer._print_message(user, data, date, tags) @@ -1175,13 +1142,7 @@ class RoomBuffer(object): def sort_messages(self): class LineCopy(object): def __init__( - self, - date, - date_printed, - tags, - prefix, - message, - highlight + self, date, date_printed, tags, prefix, message, highlight ): self.date = date self.date_printed = date_printed @@ -1192,18 +1153,25 @@ class RoomBuffer(object): @classmethod def from_line(cls, line): - return cls(line.date, line.date_printed, line.tags, - line.prefix, line.message, line.highlight) + return cls( + line.date, + line.date_printed, + line.tags, + line.prefix, + line.message, + line.highlight, + ) lines = [ LineCopy.from_line(line) for line in self.weechat_buffer.lines ] sorted_lines = sorted(lines, key=lambda line: line.date, reverse=True) - for n, line in enumerate(self.weechat_buffer.lines): - new = sorted_lines[n] - line.update(new.date, new.date_printed, new.tags, new.prefix, - new.message) + for line_number, line in enumerate(self.weechat_buffer.lines): + new = sorted_lines[line_number] + line.update( + new.date, new.date_printed, new.tags, new.prefix, new.message + ) def handle_backlog(self, events): for event in events: @@ -1230,7 +1198,7 @@ class RoomBuffer(object): break if leave_index: - timeline_events = info.timeline.events[leave_index+1:] + timeline_events = info.timeline.events[leave_index + 1 :] # Handle our leave as a state event since we're not in the # nicklist anymore but we're already printed out our leave self.handle_state_event(info.timeline.events[leave_index]) diff --git a/matrix/colors.py b/matrix/colors.py index 6ba9bff..bc4f214 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -18,34 +18,34 @@ from __future__ import unicode_literals +import html +import re +import textwrap + # pylint: disable=redefined-builtin from builtins import str from collections import namedtuple -from matrix import globals as G -from matrix.globals import W -from matrix.utils import string_strikethrough -import re -import textwrap import webcolors - from pygments import highlight -from pygments.lexers import guess_lexer, get_lexer_by_name from pygments.formatter import Formatter +from pygments.lexers import get_lexer_by_name, guess_lexer from pygments.util import ClassNotFound +from . import globals as G +from .globals import W +from .utils import string_strikethrough + try: from HTMLParser import HTMLParser except ImportError: from html.parser import HTMLParser -import html -FormattedString = namedtuple('FormattedString', ['text', 'attributes']) +FormattedString = namedtuple("FormattedString", ["text", "attributes"]) -class Formatted(): - +class Formatted: def __init__(self, substrings): # type: (List[FormattedString]) -> None self.substrings = substrings @@ -55,8 +55,8 @@ class Formatted(): return textwrap.TextWrapper( width=67, initial_indent="{}> ".format(W.color(G.CONFIG.color.quote)), - subsequent_indent="{}> ".format(W.color(G.CONFIG.color.quote)) - ) + subsequent_indent="{}> ".format(W.color(G.CONFIG.color.quote)), + ) def is_formatted(self): # type: (Formatted) -> bool @@ -89,21 +89,22 @@ class Formatted(): # Markdown bold elif line[i] == "*": - if attributes["italic"] and not line[i-1].isspace(): + if attributes["italic"] and not line[i - 1].isspace(): if text: - substrings.append(FormattedString( - text, attributes.copy())) + substrings.append( + FormattedString(text, attributes.copy()) + ) text = "" attributes["italic"] = not attributes["italic"] i = i + 1 continue - elif attributes["italic"] and line[i-1].isspace(): + elif attributes["italic"] and line[i - 1].isspace(): text = text + line[i] i = i + 1 continue - elif i+1 < len(line) and line[i+1].isspace(): + elif i + 1 < len(line) and line[i + 1].isspace(): text = text + line[i] i = i + 1 continue @@ -201,27 +202,34 @@ class Formatted(): def add_attribute(string, name, value): if name == "bold" and value: return "{bold_on}{text}{bold_off}".format( - bold_on="", text=string, bold_off="") - elif name == "italic" and value: + bold_on="", text=string, bold_off="" + ) + if name == "italic" and value: return "{italic_on}{text}{italic_off}".format( - italic_on="", text=string, italic_off="") - elif name == "underline" and value: + italic_on="", text=string, italic_off="" + ) + if name == "underline" and value: return "{underline_on}{text}{underline_off}".format( - underline_on="", text=string, underline_off="") - elif name == "strikethrough" and value: + underline_on="", text=string, underline_off="" + ) + if name == "strikethrough" and value: return "{strike_on}{text}{strike_off}".format( - strike_on="", text=string, strike_off="") - elif name == "quote" and value: + strike_on="", text=string, strike_off="" + ) + if name == "quote" and value: return "{quote_on}{text}{quote_off}".format( quote_on="

", text=string, - quote_off="
") - elif name == "fgcolor" and value: + quote_off="", + ) + if name == "fgcolor" and value: return "{color_on}{text}{color_off}".format( color_on="".format( - color=color_weechat_to_html(value)), + color=color_weechat_to_html(value) + ), text=string, - color_off="") + color_off="", + ) return string @@ -261,28 +269,32 @@ class Formatted(): return "{bold_on}{text}{bold_off}".format( bold_on=W.color("bold"), text=string, - bold_off=W.color("-bold")) + bold_off=W.color("-bold"), + ) - elif name == "italic" and value: + if name == "italic" and value: return "{italic_on}{text}{italic_off}".format( italic_on=W.color("italic"), text=string, - italic_off=W.color("-italic")) + italic_off=W.color("-italic"), + ) - elif name == "underline" and value: + if name == "underline" and value: return "{underline_on}{text}{underline_off}".format( underline_on=W.color("underline"), text=string, - underline_off=W.color("-underline")) + underline_off=W.color("-underline"), + ) - elif name == "strikethrough" and value: + if name == "strikethrough" and value: return string_strikethrough(string) - elif name == "quote" and value: + if name == "quote" and value: return self.textwrapper.fill( - W.string_remove_color(string.replace("\n", ""), "")) + W.string_remove_color(string.replace("\n", ""), "") + ) - elif name == "code" and value: + if name == "code" and value: try: lexer = get_lexer_by_name(value) except ClassNotFound: @@ -292,17 +304,19 @@ class Formatted(): # from the output return highlight(string, lexer, WeechatFormatter())[:-1] - elif name == "fgcolor" and value: + if name == "fgcolor" and value: return "{color_on}{text}{color_off}".format( color_on=W.color(value), text=string, - color_off=W.color("resetcolor")) + color_off=W.color("resetcolor"), + ) - elif name == "bgcolor" and value: + if name == "bgcolor" and value: return "{color_on}{text}{color_off}".format( color_on=W.color("," + value), text=string, - color_off=W.color("resetcolor")) + color_off=W.color("resetcolor"), + ) return string @@ -313,17 +327,18 @@ class Formatted(): # We need to handle strikethrough first, since doing # a strikethrough followed by other attributes succeeds in the # terminal, but doing it the other way around results in garbage. - if 'strikethrough' in attributes: - text = add_attribute(text, 'strikethrough', - attributes['strikethrough']) - attributes.pop('strikethrough') + if "strikethrough" in attributes: + text = add_attribute( + text, "strikethrough", attributes["strikethrough"] + ) + attributes.pop("strikethrough") for key, value in attributes.items(): text = add_attribute(text, key, value) return text weechat_strings = map(format_string, self.substrings) - return re.sub(r'\n+', '\n', "".join(weechat_strings)).strip() + return re.sub(r"\n+", "\n", "".join(weechat_strings)).strip() # TODO this should be a typed dict. @@ -335,7 +350,7 @@ DEFAULT_ATRIBUTES = { "quote": False, "code": None, "fgcolor": None, - "bgcolor": None + "bgcolor": None, } @@ -566,7 +581,7 @@ def color_line_to_weechat(color_string): "96": "250", "97": "254", "98": "231", - "99": "default" + "99": "default", } assert color_string in line_colors @@ -623,7 +638,7 @@ def colour_find_rgb(r, g, b): cb = q2c[qb] # If we have hit the colour exactly, return early. - if (cr == r and cg == g and cb == b): + if cr == r and cg == g and cb == b: return 16 + (36 * qr) + (6 * qg) + qb # Work out the closest grey (average of RGB). @@ -964,8 +979,7 @@ def color_weechat_to_html(color): # yapf: enable if color in weechat_basic_colors: return hex_colors[weechat_basic_colors[color]] - else: - return hex_colors[color] + return hex_colors[color] class WeechatFormatter(Formatter): @@ -977,7 +991,8 @@ class WeechatFormatter(Formatter): start = end = "" if style["color"]: start += "{}".format( - W.color(color_html_to_weechat(str(style["color"])))) + W.color(color_html_to_weechat(str(style["color"]))) + ) end = "{}".format(W.color("resetcolor")) + end if style["bold"]: start += W.color("bold") @@ -985,13 +1000,13 @@ class WeechatFormatter(Formatter): if style["italic"]: start += W.color("italic") end = W.color("-italic") + end - if style['underline']: + if style["underline"]: start += W.color("underline") end = W.color("-underline") + end self.styles[token] = (start, end) def format(self, tokensource, outfile): - lastval = '' + lastval = "" lasttype = None for ttype, value in tokensource: diff --git a/matrix/commands.py b/matrix/commands.py index 280149e..beaed12 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -15,18 +15,17 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import unicode_literals + +import argparse +import re from builtins import str -import re -import argparse - -import matrix.globals -from matrix.globals import W, SERVERS - -from matrix.utf import utf8_decode -from matrix.utils import key_from_value, tags_from_line_data -from matrix.server import MatrixServer -from matrix.colors import Formatted +from . import globals as G +from .colors import Formatted +from .globals import SERVERS, W +from .server import MatrixServer +from .utf import utf8_decode +from .utils import key_from_value, tags_from_line_data class ParseError(Exception): @@ -34,15 +33,15 @@ class ParseError(Exception): class WeechatArgParse(argparse.ArgumentParser): - def print_usage(self, file): + def print_usage(self, file=None): pass def error(self, message): - m = ("{prefix}Error: {message} for command {command} " - "(see /help {command})").format(prefix=W.prefix("error"), - message=message, - command=self.prog) - W.prnt("", m) + message = ( + "{prefix}Error: {message} for command {command} " + "(see /help {command})" + ).format(prefix=W.prefix("error"), message=message, command=self.prog) + W.prnt("", message) raise ParseError @@ -95,49 +94,59 @@ class WeechatCommandParser(object): def hook_commands(): W.hook_command( # Command name and short description - 'matrix', - 'Matrix chat protocol command', + "matrix", + "Matrix chat protocol command", # Synopsis - ('server add [:] ||' - 'server delete|list|listfull ||' - 'connect ||' - 'disconnect ||' - 'reconnect ||' - 'help '), + ( + "server add [:] ||" + "server delete|list|listfull ||" + "connect ||" + "disconnect ||" + "reconnect ||" + "help " + ), # Description - (' server: list, add, or remove Matrix servers\n' - ' connect: connect to Matrix servers\n' - 'disconnect: disconnect from one or all Matrix servers\n' - ' reconnect: reconnect to server(s)\n\n' - ' help: show detailed command help\n\n' - 'Use /matrix help [command] to find out more.\n'), + ( + " server: list, add, or remove Matrix servers\n" + " connect: connect to Matrix servers\n" + "disconnect: disconnect from one or all Matrix servers\n" + " reconnect: reconnect to server(s)\n\n" + " help: show detailed command help\n\n" + "Use /matrix help [command] to find out more.\n" + ), # Completions - ('server %(matrix_server_commands)|%* ||' - 'connect %(matrix_servers) ||' - 'disconnect %(matrix_servers) ||' - 'reconnect %(matrix_servers) ||' - 'help %(matrix_commands)'), + ( + "server %(matrix_server_commands)|%* ||" + "connect %(matrix_servers) ||" + "disconnect %(matrix_servers) ||" + "reconnect %(matrix_servers) ||" + "help %(matrix_commands)" + ), # Function name - 'matrix_command_cb', - '') + "matrix_command_cb", + "", + ) W.hook_command( # Command name and short description - 'redact', - 'redact messages', + "redact", + "redact messages", # Synopsis ('[:""] []'), # Description - ("message-number: number of message to redact (starting from 1 for\n" - " the last message received, counting up)\n" - " message-part: an initial part of the message (ignored, only used\n" - " as visual feedback when using completion)\n" - " reason: the redaction reason\n"), + ( + "message-number: number of message to redact (starting from 1 for\n" + " the last message received, counting up)\n" + " message-part: an initial part of the message (ignored, only used\n" + " as visual feedback when using completion)\n" + " reason: the redaction reason\n" + ), # Completions - ('%(matrix_messages)'), + ("%(matrix_messages)"), # Function name - 'matrix_redact_command_cb', - '') + "matrix_redact_command_cb", + "", + ) W.hook_command( # Command name and short description @@ -146,13 +155,13 @@ def hook_commands(): # Synopsis ("[|-delete]"), # Description - (" topic: topic to set\n" - "-delete: delete room topic"), + (" topic: topic to set\n" "-delete: delete room topic"), # Completions "", # Callback "matrix_topic_command_cb", - "") + "", + ) W.hook_command( # Command name and short description @@ -166,7 +175,8 @@ def hook_commands(): "", # Callback "matrix_me_command_cb", - "") + "", + ) W.hook_command( # Command name and short description @@ -175,13 +185,16 @@ def hook_commands(): # Synopsis (" []"), # Description - ("user-id: user-id to kick\n" - " reason: reason why the user was kicked"), + ( + "user-id: user-id to kick\n" + " reason: reason why the user was kicked" + ), # Completions ("%(matrix_users)"), # Callback "matrix_kick_command_cb", - "") + "", + ) W.hook_command( # Command name and short description @@ -195,7 +208,8 @@ def hook_commands(): ("%(matrix_users)"), # Callback "matrix_invite_command_cb", - "") + "", + ) W.hook_command( # Command name and short description @@ -204,13 +218,16 @@ def hook_commands(): # Synopsis ("|"), # Description - (" room-id: room-id of the room to join\n" - "room-alias: room alias of the room to join"), + ( + " room-id: room-id of the room to join\n" + "room-alias: room alias of the room to join" + ), # Completions "", # Callback "matrix_join_command_cb", - "") + "", + ) W.hook_command( # Command name and short description @@ -224,12 +241,13 @@ def hook_commands(): "", # Callback "matrix_part_command_cb", - "") + "", + ) - W.hook_command_run('/buffer clear', 'matrix_command_buf_clear_cb', '') + W.hook_command_run("/buffer clear", "matrix_command_buf_clear_cb", "") # if OPTIONS.enable_backlog: - # hook_page_up() + # hook_page_up() @utf8_decode @@ -238,8 +256,9 @@ def matrix_me_command_cb(data, buffer, args): if buffer in server.buffers.values(): if not server.connected: - message = ("{prefix}matrix: you are not connected to " - "the server").format(prefix=W.prefix("error")) + message = ( + "{prefix}matrix: you are not connected to " "the server" + ).format(prefix=W.prefix("error")) W.prnt(server.server_buffer, message) return W.WEECHAT_RC_ERROR @@ -253,13 +272,16 @@ def matrix_me_command_cb(data, buffer, args): server.room_send_message(room_buffer, formatted_data, "m.emote") return W.WEECHAT_RC_OK - elif buffer == server.server_buffer: - message = ("{prefix}matrix: command \"me\" must be " - "executed on a Matrix channel buffer" - ).format(prefix=W.prefix("error")) + if buffer == server.server_buffer: + message = ( + '{prefix}matrix: command "me" must be ' + "executed on a Matrix channel buffer" + ).format(prefix=W.prefix("error")) W.prnt("", message) return W.WEECHAT_RC_OK + return W.WEECHAT_RC_OK + @utf8_decode def matrix_topic_command_cb(data, buffer, args): @@ -269,8 +291,9 @@ def matrix_topic_command_cb(data, buffer, args): for server in SERVERS.values(): if buffer == server.server_buffer: - server.error("command \"topic\" must be " - "executed on a Matrix room buffer") + server.error( + 'command "topic" must be ' "executed on a Matrix room buffer" + ) return W.WEECHAT_RC_OK room = server.find_room_from_ptr(buffer) @@ -310,15 +333,17 @@ def matrix_fetch_old_messages(server, room_id): def check_server_existence(server_name, servers): if server_name not in servers: message = "{prefix}matrix: No such server: {server}".format( - prefix=W.prefix("error"), server=server_name) + prefix=W.prefix("error"), server=server_name + ) W.prnt("", message) return False return True def hook_page_up(): - OPTIONS.page_up_hook = W.hook_command_run('/window page_up', - 'matrix_command_pgup_cb', '') + G.CONFIG.page_up_hook = W.hook_command_run( + "/window page_up", "matrix_command_pgup_cb", "" + ) @utf8_decode @@ -345,7 +370,8 @@ def matrix_command_pgup_cb(data, buffer, command): window = W.window_search_with_buffer(buffer) first_line_displayed = bool( - W.window_get_integer(window, "first_line_displayed")) + W.window_get_integer(window, "first_line_displayed") + ) if first_line_displayed: room_id = key_from_value(server.buffers, buffer) @@ -382,9 +408,11 @@ def matrix_part_command_cb(data, buffer, args): if not room_id: if buffer == server.server_buffer: - server.error("command \"part\" must be " - "executed on a Matrix room buffer or a room " - "name needs to be given") + server.error( + 'command "part" must be ' + "executed on a Matrix room buffer or a room " + "name needs to be given" + ) return W.WEECHAT_RC_OK room_buffer = server.find_room_from_ptr(buffer) @@ -404,8 +432,9 @@ def matrix_invite_command_cb(data, buffer, args): for server in SERVERS.values(): if buffer == server.server_buffer: - server.error("command \"invite\" must be " - "executed on a Matrix room buffer") + server.error( + 'command "invite" must be ' "executed on a Matrix room buffer" + ) return W.WEECHAT_RC_OK room = server.find_room_from_ptr(buffer) @@ -429,8 +458,9 @@ def matrix_kick_command_cb(data, buffer, args): for server in SERVERS.values(): if buffer == server.server_buffer: - server.error("command \"kick\" must be " - "executed on a Matrix room buffer") + server.error( + 'command "kick" must be ' "executed on a Matrix room buffer" + ) return W.WEECHAT_RC_OK room = server.find_room_from_ptr(buffer) @@ -449,22 +479,24 @@ def matrix_kick_command_cb(data, buffer, args): def event_id_from_line(buf, target_number): # type: (weechat.buffer, int) -> str - own_lines = W.hdata_pointer(W.hdata_get('buffer'), buf, 'own_lines') + own_lines = W.hdata_pointer(W.hdata_get("buffer"), buf, "own_lines") if own_lines: - line = W.hdata_pointer(W.hdata_get('lines'), own_lines, 'last_line') + line = W.hdata_pointer(W.hdata_get("lines"), own_lines, "last_line") line_number = 1 while line: - line_data = W.hdata_pointer(W.hdata_get('line'), line, 'data') + line_data = W.hdata_pointer(W.hdata_get("line"), line, "data") if line_data: tags = tags_from_line_data(line_data) # Only count non redacted user messages - if ("matrix_message" in tags and - 'matrix_redacted' not in tags and - "matrix_new_redacted" not in tags): + if ( + "matrix_message" in tags + and "matrix_redacted" not in tags + and "matrix_new_redacted" not in tags + ): if line_number == target_number: for tag in tags: @@ -474,7 +506,7 @@ def event_id_from_line(buf, target_number): line_number += 1 - line = W.hdata_move(W.hdata_get('line'), line, -1) + line = W.hdata_move(W.hdata_get("line"), line, -1) return "" @@ -488,9 +520,10 @@ def matrix_redact_command_cb(data, buffer, args): matches = re.match(r"(\d+)(:\".*\")? ?(.*)?", args) if not matches: - message = ("{prefix}matrix: Invalid command " - "arguments (see /help redact)" - ).format(prefix=W.prefix("error")) + message = ( + "{prefix}matrix: Invalid command " + "arguments (see /help redact)" + ).format(prefix=W.prefix("error")) W.prnt("", message) return W.WEECHAT_RC_ERROR @@ -500,9 +533,10 @@ def matrix_redact_command_cb(data, buffer, args): event_id = event_id_from_line(buffer, line) if not event_id: - message = ("{prefix}matrix: No such message with number " - "{number} found").format( - prefix=W.prefix("error"), number=line) + message = ( + "{prefix}matrix: No such message with number " + "{number} found" + ).format(prefix=W.prefix("error"), number=line) W.prnt("", message) return W.WEECHAT_RC_OK @@ -510,10 +544,11 @@ def matrix_redact_command_cb(data, buffer, args): return W.WEECHAT_RC_OK - elif buffer == server.server_buffer: - message = ("{prefix}matrix: command \"redact\" must be " - "executed on a Matrix channel buffer" - ).format(prefix=W.prefix("error")) + if buffer == server.server_buffer: + message = ( + '{prefix}matrix: command "redact" must be ' + "executed on a Matrix channel buffer" + ).format(prefix=W.prefix("error")) W.prnt("", message) return W.WEECHAT_RC_OK @@ -522,9 +557,10 @@ def matrix_redact_command_cb(data, buffer, args): def matrix_command_help(args): if not args: - message = ("{prefix}matrix: Too few arguments for command " - "\"/matrix help\" (see /matrix help help)" - ).format(prefix=W.prefix("error")) + message = ( + "{prefix}matrix: Too few arguments for command " + '"/matrix help" (see /matrix help help)' + ).format(prefix=W.prefix("error")) W.prnt("", message) return @@ -532,88 +568,104 @@ def matrix_command_help(args): message = "" if command == "connect": - message = ("{delimiter_color}[{ncolor}matrix{delimiter_color}] " - "{ncolor}{cmd_color}/connect{ncolor} " - " [...]" - "\n\n" - "connect to Matrix server(s)" - "\n\n" - "server-name: server to connect to" - "(internal name)").format( - delimiter_color=W.color("chat_delimiters"), - cmd_color=W.color("chat_buffer"), - ncolor=W.color("reset")) + message = ( + "{delimiter_color}[{ncolor}matrix{delimiter_color}] " + "{ncolor}{cmd_color}/connect{ncolor} " + " [...]" + "\n\n" + "connect to Matrix server(s)" + "\n\n" + "server-name: server to connect to" + "(internal name)" + ).format( + delimiter_color=W.color("chat_delimiters"), + cmd_color=W.color("chat_buffer"), + ncolor=W.color("reset"), + ) elif command == "disconnect": - message = ("{delimiter_color}[{ncolor}matrix{delimiter_color}] " - "{ncolor}{cmd_color}/disconnect{ncolor} " - " [...]" - "\n\n" - "disconnect from Matrix server(s)" - "\n\n" - "server-name: server to disconnect" - "(internal name)").format( - delimiter_color=W.color("chat_delimiters"), - cmd_color=W.color("chat_buffer"), - ncolor=W.color("reset")) + message = ( + "{delimiter_color}[{ncolor}matrix{delimiter_color}] " + "{ncolor}{cmd_color}/disconnect{ncolor} " + " [...]" + "\n\n" + "disconnect from Matrix server(s)" + "\n\n" + "server-name: server to disconnect" + "(internal name)" + ).format( + delimiter_color=W.color("chat_delimiters"), + cmd_color=W.color("chat_buffer"), + ncolor=W.color("reset"), + ) elif command == "reconnect": - message = ("{delimiter_color}[{ncolor}matrix{delimiter_color}] " - "{ncolor}{cmd_color}/reconnect{ncolor} " - " [...]" - "\n\n" - "reconnect to Matrix server(s)" - "\n\n" - "server-name: server to reconnect" - "(internal name)").format( - delimiter_color=W.color("chat_delimiters"), - cmd_color=W.color("chat_buffer"), - ncolor=W.color("reset")) + message = ( + "{delimiter_color}[{ncolor}matrix{delimiter_color}] " + "{ncolor}{cmd_color}/reconnect{ncolor} " + " [...]" + "\n\n" + "reconnect to Matrix server(s)" + "\n\n" + "server-name: server to reconnect" + "(internal name)" + ).format( + delimiter_color=W.color("chat_delimiters"), + cmd_color=W.color("chat_buffer"), + ncolor=W.color("reset"), + ) elif command == "server": - message = ("{delimiter_color}[{ncolor}matrix{delimiter_color}] " - "{ncolor}{cmd_color}/server{ncolor} " - "add [:]" - "\n " - "delete|list|listfull " - "\n\n" - "list, add, or remove Matrix servers" - "\n\n" - " list: list servers (without argument, this " - "list is displayed)\n" - " listfull: list servers with detailed info for each " - "server\n" - " add: add a new server\n" - " delete: delete a server\n" - "server-name: server to reconnect (internal name)\n" - " hostname: name or IP address of server\n" - " port: port of server (default: 8448)\n" - "\n" - "Examples:" - "\n /matrix server listfull" - "\n /matrix server add matrix matrix.org:80" - "\n /matrix server del matrix").format( - delimiter_color=W.color("chat_delimiters"), - cmd_color=W.color("chat_buffer"), - ncolor=W.color("reset")) + message = ( + "{delimiter_color}[{ncolor}matrix{delimiter_color}] " + "{ncolor}{cmd_color}/server{ncolor} " + "add [:]" + "\n " + "delete|list|listfull " + "\n\n" + "list, add, or remove Matrix servers" + "\n\n" + " list: list servers (without argument, this " + "list is displayed)\n" + " listfull: list servers with detailed info for each " + "server\n" + " add: add a new server\n" + " delete: delete a server\n" + "server-name: server to reconnect (internal name)\n" + " hostname: name or IP address of server\n" + " port: port of server (default: 8448)\n" + "\n" + "Examples:" + "\n /matrix server listfull" + "\n /matrix server add matrix matrix.org:80" + "\n /matrix server del matrix" + ).format( + delimiter_color=W.color("chat_delimiters"), + cmd_color=W.color("chat_buffer"), + ncolor=W.color("reset"), + ) elif command == "help": - message = ("{delimiter_color}[{ncolor}matrix{delimiter_color}] " - "{ncolor}{cmd_color}/help{ncolor} " - " [...]" - "\n\n" - "display help about Matrix commands" - "\n\n" - "matrix-command: a Matrix command name" - "(internal name)").format( - delimiter_color=W.color("chat_delimiters"), - cmd_color=W.color("chat_buffer"), - ncolor=W.color("reset")) + message = ( + "{delimiter_color}[{ncolor}matrix{delimiter_color}] " + "{ncolor}{cmd_color}/help{ncolor} " + " [...]" + "\n\n" + "display help about Matrix commands" + "\n\n" + "matrix-command: a Matrix command name" + "(internal name)" + ).format( + delimiter_color=W.color("chat_delimiters"), + cmd_color=W.color("chat_buffer"), + ncolor=W.color("reset"), + ) else: - message = ("{prefix}matrix: No help available, \"{command}\" " - "is not a matrix command").format( - prefix=W.prefix("error"), command=command) + message = ( + '{prefix}matrix: No help available, "{command}" ' + "is not a matrix command" + ).format(prefix=W.prefix("error"), command=command) W.prnt("", "") W.prnt("", message) @@ -622,7 +674,6 @@ def matrix_command_help(args): def matrix_server_command_listfull(args): - def get_value_string(value, default_value): if value == default_value: if not value: @@ -632,7 +683,8 @@ def matrix_server_command_listfull(args): value_string = "{color}{value}{ncolor}".format( color=W.color("chat_value"), value=value, - ncolor=W.color("reset")) + ncolor=W.color("reset"), + ) return value_string @@ -650,14 +702,17 @@ def matrix_server_command_listfull(args): else: connected = "not connected" - message = ("Server: {server_color}{server}{delimiter_color}" - " [{ncolor}{connected}{delimiter_color}]" - "{ncolor}").format( - server_color=W.color("chat_server"), - server=server.name, - delimiter_color=W.color("chat_delimiters"), - connected=connected, - ncolor=W.color("reset")) + message = ( + "Server: {server_color}{server}{delimiter_color}" + " [{ncolor}{connected}{delimiter_color}]" + "{ncolor}" + ).format( + server_color=W.color("chat_server"), + server=server.name, + delimiter_color=W.color("chat_delimiters"), + connected=connected, + ncolor=W.color("reset"), + ) W.prnt("", message) @@ -703,7 +758,7 @@ def matrix_server_command_listfull(args): if value: value = "(hidden)" - value_string = get_value_string(value, '') + value_string = get_value_string(value, "") message = " password . . : {value}".format(value=value_string) W.prnt("", message) @@ -715,14 +770,17 @@ def matrix_server_command_delete(args): server = SERVERS[server_name] if server.connected: - message = ("{prefix}matrix: you can not delete server " - "{color}{server}{ncolor} because you are " - "connected to it. Try \"/matrix disconnect " - "{color}{server}{ncolor}\" before.").format( - prefix=W.prefix("error"), - color=W.color("chat_server"), - ncolor=W.color("reset"), - server=server.name) + message = ( + "{prefix}matrix: you can not delete server " + "{color}{server}{ncolor} because you are " + 'connected to it. Try "/matrix disconnect ' + '{color}{server}{ncolor}" before.' + ).format( + prefix=W.prefix("error"), + color=W.color("chat_server"), + ncolor=W.color("reset"), + server=server.name, + ) W.prnt("", message) return @@ -735,11 +793,13 @@ def matrix_server_command_delete(args): for option in server.config._option_ptrs.values(): W.config_option_free(option) - message = ("matrix: server {color}{server}{ncolor} has been " - "deleted").format( - server=server.name, - color=W.color("chat_server"), - ncolor=W.color("reset")) + message = ( + "matrix: server {color}{server}{ncolor} has been " "deleted" + ).format( + server=server.name, + color=W.color("chat_server"), + ncolor=W.color("reset"), + ) del SERVERS[server.name] server = None @@ -749,15 +809,17 @@ def matrix_server_command_delete(args): def matrix_server_command_add(args): if len(args) < 2: - message = ("{prefix}matrix: Too few arguments for command " - "\"/matrix server add\" (see /matrix help server)").format( - prefix=W.prefix("error")) + message = ( + "{prefix}matrix: Too few arguments for command " + '"/matrix server add" (see /matrix help server)' + ).format(prefix=W.prefix("error")) W.prnt("", message) return - elif len(args) > 4: - message = ("{prefix}matrix: Too many arguments for command " - "\"/matrix server add\" (see /matrix help server)" - ).format(prefix=W.prefix("error")) + if len(args) > 4: + message = ( + "{prefix}matrix: Too many arguments for command " + '"/matrix server add" (see /matrix help server)' + ).format(prefix=W.prefix("error")) W.prnt("", message) return @@ -769,16 +831,19 @@ def matrix_server_command_add(args): server_name = args[0] if server_name in SERVERS: - message = ("{prefix}matrix: server {color}{server}{ncolor} " - "already exists, can't add it").format( - prefix=W.prefix("error"), - color=W.color("chat_server"), - server=server_name, - ncolor=W.color("reset")) + message = ( + "{prefix}matrix: server {color}{server}{ncolor} " + "already exists, can't add it" + ).format( + prefix=W.prefix("error"), + color=W.color("chat_server"), + server=server_name, + ncolor=W.color("reset"), + ) W.prnt("", message) return - server = MatrixServer(server_name, matrix.globals.CONFIG) + server = MatrixServer(server_name, G.CONFIG._ptr) SERVERS[server.name] = server if len(args) >= 2: @@ -788,20 +853,21 @@ def matrix_server_command_add(args): host, port = args[1], None return_code = W.config_option_set( - server.config._option_ptrs["address"], - host, - 1 + server.config._option_ptrs["address"], host, 1 ) if return_code == W.WEECHAT_CONFIG_OPTION_SET_ERROR: remove_server(server) - message = ("{prefix}Failed to set address for server " - "{color}{server}{ncolor}, failed to add " - "server.").format( - prefix=W.prefix("error"), - color=W.color("chat_server"), - server=server.name, - ncolor=W.color("reset")) + message = ( + "{prefix}Failed to set address for server " + "{color}{server}{ncolor}, failed to add " + "server." + ).format( + prefix=W.prefix("error"), + color=W.color("chat_server"), + server=server.name, + ncolor=W.color("reset"), + ) W.prnt("", message) server = None @@ -809,19 +875,20 @@ def matrix_server_command_add(args): if port: return_code = W.config_option_set( - server.config._option_ptrs["port"], - port, - 1 + server.config._option_ptrs["port"], port, 1 ) if return_code == W.WEECHAT_CONFIG_OPTION_SET_ERROR: remove_server(server) - message = ("{prefix}Failed to set port for server " - "{color}{server}{ncolor}, failed to add " - "server.").format( - prefix=W.prefix("error"), - color=W.color("chat_server"), - server=server.name, - ncolor=W.color("reset")) + message = ( + "{prefix}Failed to set port for server " + "{color}{server}{ncolor}, failed to add " + "server." + ).format( + prefix=W.prefix("error"), + color=W.color("chat_server"), + server=server.name, + ncolor=W.color("reset"), + ) W.prnt("", message) server = None @@ -830,20 +897,21 @@ def matrix_server_command_add(args): if len(args) >= 3: user = args[2] return_code = W.config_option_set( - server.config._option_ptrs["username"], - user, - 1 + server.config._option_ptrs["username"], user, 1 ) if return_code == W.WEECHAT_CONFIG_OPTION_SET_ERROR: remove_server(server) - message = ("{prefix}Failed to set user for server " - "{color}{server}{ncolor}, failed to add " - "server.").format( - prefix=W.prefix("error"), - color=W.color("chat_server"), - server=server.name, - ncolor=W.color("reset")) + message = ( + "{prefix}Failed to set user for server " + "{color}{server}{ncolor}, failed to add " + "server." + ).format( + prefix=W.prefix("error"), + color=W.color("chat_server"), + server=server.name, + ncolor=W.color("reset"), + ) W.prnt("", message) server = None @@ -853,28 +921,31 @@ def matrix_server_command_add(args): password = args[3] return_code = W.config_option_set( - server.config._option_ptrs["password"], - password, - 1 + server.config._option_ptrs["password"], password, 1 ) if return_code == W.WEECHAT_CONFIG_OPTION_SET_ERROR: remove_server(server) - message = ("{prefix}Failed to set password for server " - "{color}{server}{ncolor}, failed to add " - "server.").format( - prefix=W.prefix("error"), - color=W.color("chat_server"), - server=server.name, - ncolor=W.color("reset")) + message = ( + "{prefix}Failed to set password for server " + "{color}{server}{ncolor}, failed to add " + "server." + ).format( + prefix=W.prefix("error"), + color=W.color("chat_server"), + server=server.name, + ncolor=W.color("reset"), + ) W.prnt("", message) server = None return - message = ("matrix: server {color}{server}{ncolor} " - "has been added").format( - server=server.name, - color=W.color("chat_server"), - ncolor=W.color("reset")) + message = ( + "matrix: server {color}{server}{ncolor} " "has been added" + ).format( + server=server.name, + color=W.color("chat_server"), + ncolor=W.color("reset"), + ) W.prnt("", message) @@ -883,23 +954,28 @@ def matrix_server_command(command, args): if SERVERS: W.prnt("", "\nAll matrix servers:") for server in SERVERS: - W.prnt("", " {color}{server}".format( - color=W.color("chat_server"), server=server)) + W.prnt( + "", + " {color}{server}".format( + color=W.color("chat_server"), server=server + ), + ) # TODO the argument for list and listfull is used as a match word to # find/filter servers, we're currently match exactly to the whole name - if command == 'list': + if command == "list": list_servers(args) - elif command == 'listfull': + elif command == "listfull": matrix_server_command_listfull(args) - elif command == 'add': + elif command == "add": matrix_server_command_add(args) - elif command == 'delete': + elif command == "delete": matrix_server_command_delete(args) else: - message = ("{prefix}matrix: Error: unknown matrix server command, " - "\"{command}\" (type /matrix help server for help)").format( - prefix=W.prefix("error"), command=command) + message = ( + "{prefix}matrix: Error: unknown matrix server command, " + '"{command}" (type /matrix help server for help)' + ).format(prefix=W.prefix("error"), command=command) W.prnt("", message) @@ -921,41 +997,44 @@ def matrix_command_cb(data, buffer, args): server.access_token = "" server.disconnect(reconnect=False) - split_args = list(filter(bool, args.split(' '))) + split_args = list(filter(bool, args.split(" "))) if len(split_args) < 1: - message = ("{prefix}matrix: Too few arguments for command " - "\"/matrix\" " - "(see /help matrix)").format(prefix=W.prefix("error")) + message = ( + "{prefix}matrix: Too few arguments for command " + '"/matrix" ' + "(see /help matrix)" + ).format(prefix=W.prefix("error")) W.prnt("", message) return W.WEECHAT_RC_ERROR command, args = split_args[0], split_args[1:] - if command == 'connect': + if command == "connect": connect_server(args) - elif command == 'disconnect': + elif command == "disconnect": disconnect_server(args) - elif command == 'reconnect': + elif command == "reconnect": disconnect_server(args) connect_server(args) - elif command == 'server': + elif command == "server": if len(args) >= 1: subcommand, args = args[0], args[1:] matrix_server_command(subcommand, args) else: matrix_server_command("list", "") - elif command == 'help': + elif command == "help": matrix_command_help(args) else: - message = ("{prefix}matrix: Error: unknown matrix command, " - "\"{command}\" (type /help matrix for help)").format( - prefix=W.prefix("error"), command=command) + message = ( + "{prefix}matrix: Error: unknown matrix command, " + '"{command}" (type /help matrix for help)' + ).format(prefix=W.prefix("error"), command=command) W.prnt("", message) return W.WEECHAT_RC_OK diff --git a/matrix/completion.py b/matrix/completion.py index ce5c471..bf00682 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -16,29 +16,32 @@ from __future__ import unicode_literals +from matrix.globals import SERVERS, W from matrix.utf import utf8_decode -from matrix.globals import W, SERVERS from matrix.utils import tags_from_line_data def add_servers_to_completion(completion): for server_name in SERVERS: - W.hook_completion_list_add(completion, server_name, 0, - W.WEECHAT_LIST_POS_SORT) + W.hook_completion_list_add( + completion, server_name, 0, W.WEECHAT_LIST_POS_SORT + ) @utf8_decode -def matrix_server_command_completion_cb(data, completion_item, buffer, - completion): +def matrix_server_command_completion_cb( + data, completion_item, buffer, completion +): buffer_input = W.buffer_get_string(buffer, "input").split() args = buffer_input[1:] - commands = ['add', 'delete', 'list', 'listfull'] + commands = ["add", "delete", "list", "listfull"] def complete_commands(): for command in commands: - W.hook_completion_list_add(completion, command, 0, - W.WEECHAT_LIST_POS_SORT) + W.hook_completion_list_add( + completion, command, 0, W.WEECHAT_LIST_POS_SORT + ) if len(args) == 1: complete_commands() @@ -47,11 +50,11 @@ def matrix_server_command_completion_cb(data, completion_item, buffer, if args[1] not in commands: complete_commands() else: - if args[1] == 'delete' or args[1] == 'listfull': + if args[1] == "delete" or args[1] == "listfull": add_servers_to_completion(completion) elif len(args) == 3: - if args[1] == 'delete' or args[1] == 'listfull': + if args[1] == "delete" or args[1] == "listfull": if args[2] not in SERVERS: add_servers_to_completion(completion) @@ -67,18 +70,25 @@ def matrix_server_completion_cb(data, completion_item, buffer, completion): @utf8_decode def matrix_command_completion_cb(data, completion_item, buffer, completion): for command in [ - "connect", "disconnect", "reconnect", "server", "help", "debug" + "connect", + "disconnect", + "reconnect", + "server", + "help", + "debug", ]: - W.hook_completion_list_add(completion, command, 0, - W.WEECHAT_LIST_POS_SORT) + W.hook_completion_list_add( + completion, command, 0, W.WEECHAT_LIST_POS_SORT + ) return W.WEECHAT_RC_OK @utf8_decode def matrix_debug_completion_cb(data, completion_item, buffer, completion): for debug_type in ["messaging", "network", "timing"]: - W.hook_completion_list_add(completion, debug_type, 0, - W.WEECHAT_LIST_POS_SORT) + W.hook_completion_list_add( + completion, debug_type, 0, W.WEECHAT_LIST_POS_SORT + ) return W.WEECHAT_RC_OK @@ -88,46 +98,52 @@ REDACTION_COMP_LEN = 50 @utf8_decode def matrix_message_completion_cb(data, completion_item, buffer, completion): - own_lines = W.hdata_pointer(W.hdata_get('buffer'), buffer, 'own_lines') + own_lines = W.hdata_pointer(W.hdata_get("buffer"), buffer, "own_lines") if own_lines: - line = W.hdata_pointer(W.hdata_get('lines'), own_lines, 'last_line') + line = W.hdata_pointer(W.hdata_get("lines"), own_lines, "last_line") line_number = 1 while line: - line_data = W.hdata_pointer(W.hdata_get('line'), line, 'data') + line_data = W.hdata_pointer(W.hdata_get("line"), line, "data") if line_data: message = W.hdata_string( - W.hdata_get('line_data'), line_data, 'message') + W.hdata_get("line_data"), line_data, "message" + ) tags = tags_from_line_data(line_data) # Only add non redacted user messages to the completion - if (message and 'matrix_message' in tags and - 'matrix_redacted' not in tags): + if ( + message + and "matrix_message" in tags + and "matrix_redacted" not in tags + ): if len(message) > REDACTION_COMP_LEN + 2: - message = (message[:REDACTION_COMP_LEN] + '..') + message = message[:REDACTION_COMP_LEN] + ".." - item = ("{number}:\"{message}\"").format( - number=line_number, message=message) + item = ('{number}:"{message}"').format( + number=line_number, message=message + ) - W.hook_completion_list_add(completion, item, 0, - W.WEECHAT_LIST_POS_END) + W.hook_completion_list_add( + completion, item, 0, W.WEECHAT_LIST_POS_END + ) line_number += 1 - line = W.hdata_move(W.hdata_get('line'), line, -1) + line = W.hdata_move(W.hdata_get("line"), line, -1) return W.WEECHAT_RC_OK def server_from_buffer(buffer): for server in SERVERS.values(): - if buffer in server.buffers.values(): - return server - elif buffer == server.server_buffer: - return server + if buffer in server.buffers.values(): + return server + if buffer == server.server_buffer: + return server return None @@ -141,8 +157,9 @@ def matrix_olm_user_completion_cb(data, completion_item, buffer, completion): olm = server.olm for user in olm.device_keys: - W.hook_completion_list_add(completion, user, 0, - W.WEECHAT_LIST_POS_SORT) + W.hook_completion_list_add( + completion, user, 0, W.WEECHAT_LIST_POS_SORT + ) return W.WEECHAT_RC_OK @@ -169,8 +186,9 @@ def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): return W.WEECHAT_RC_OK for device in olm.device_keys[user]: - W.hook_completion_list_add(completion, device.device_id, 0, - W.WEECHAT_LIST_POS_SORT) + W.hook_completion_list_add( + completion, device.device_id, 0, W.WEECHAT_LIST_POS_SORT + ) return W.WEECHAT_RC_OK @@ -178,8 +196,9 @@ def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): @utf8_decode def matrix_user_completion_cb(data, completion_item, buffer, completion): def add_user(completion, user): - W.hook_completion_list_add(completion, user, 0, - W.WEECHAT_LIST_POS_SORT) + W.hook_completion_list_add( + completion, user, 0, W.WEECHAT_LIST_POS_SORT + ) for server in SERVERS.values(): if buffer == server.server_buffer: @@ -201,26 +220,58 @@ def matrix_user_completion_cb(data, completion_item, buffer, completion): def init_completion(): - W.hook_completion("matrix_server_commands", "Matrix server completion", - "matrix_server_command_completion_cb", "") + W.hook_completion( + "matrix_server_commands", + "Matrix server completion", + "matrix_server_command_completion_cb", + "", + ) - W.hook_completion("matrix_servers", "Matrix server completion", - "matrix_server_completion_cb", "") + W.hook_completion( + "matrix_servers", + "Matrix server completion", + "matrix_server_completion_cb", + "", + ) - W.hook_completion("matrix_commands", "Matrix command completion", - "matrix_command_completion_cb", "") + W.hook_completion( + "matrix_commands", + "Matrix command completion", + "matrix_command_completion_cb", + "", + ) - W.hook_completion("matrix_messages", "Matrix message completion", - "matrix_message_completion_cb", "") + W.hook_completion( + "matrix_messages", + "Matrix message completion", + "matrix_message_completion_cb", + "", + ) - W.hook_completion("matrix_debug_types", "Matrix debugging type completion", - "matrix_debug_completion_cb", "") + W.hook_completion( + "matrix_debug_types", + "Matrix debugging type completion", + "matrix_debug_completion_cb", + "", + ) - W.hook_completion("olm_user_ids", "Matrix olm user id completion", - "matrix_olm_user_completion_cb", "") + W.hook_completion( + "olm_user_ids", + "Matrix olm user id completion", + "matrix_olm_user_completion_cb", + "", + ) - W.hook_completion("olm_devices", "Matrix olm device id completion", - "matrix_olm_device_completion_cb", "") + W.hook_completion( + "olm_devices", + "Matrix olm device id completion", + "matrix_olm_device_completion_cb", + "", + ) - W.hook_completion("matrix_users", "Matrix user id completion", - "matrix_user_completion_cb", "") + W.hook_completion( + "matrix_users", + "Matrix user id completion", + "matrix_user_completion_cb", + "", + ) diff --git a/matrix/config.py b/matrix/config.py index 06b9c4a..7a82c14 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -16,16 +16,16 @@ # from __future__ import unicode_literals from builtins import super +from collections import namedtuple +from enum import Enum, unique -import nio import logbook +import nio -from . import globals as G -from matrix.globals import W, SERVERS, SCRIPT_NAME +from matrix.globals import SCRIPT_NAME, SERVERS, W from matrix.utf import utf8_decode -from enum import Enum, unique -from collections import namedtuple +from . import globals as G @unique @@ -49,20 +49,22 @@ class DebugType(Enum): TIMING = 2 -class Option(namedtuple( - 'Option', - [ - 'name', - 'type', - 'string_values', - 'min', - 'max', - 'value', - 'description', - 'cast_func', - 'change_callback' - ] -)): +class Option( + namedtuple( + "Option", + [ + "name", + "type", + "string_values", + "min", + "max", + "value", + "description", + "cast_func", + "change_callback", + ], + ) +): __slots__ = () def __new__( @@ -75,7 +77,7 @@ class Option(namedtuple( value, description, cast=None, - change_callback=None + change_callback=None, ): return super().__new__( cls, @@ -87,12 +89,10 @@ class Option(namedtuple( value, description, cast, - change_callback + change_callback, ) - - @utf8_decode def matrix_config_reload_cb(data, config_file): return W.WEECHAT_RC_OK @@ -121,8 +121,7 @@ def config_server_buffer_cb(data, option): @utf8_decode def config_log_level_cb(data, option): change_log_level( - G.CONFIG.network.debug_category, - G.CONFIG.network.debug_level + G.CONFIG.network.debug_category, G.CONFIG.network.debug_level ) return 1 @@ -132,8 +131,7 @@ def config_log_category_cb(data, option): change_log_level(G.CONFIG.debug_category, logbook.ERROR) G.CONFIG.debug_category = G.CONFIG.network.debug_category change_log_level( - G.CONFIG.network.debug_category, - G.CONFIG.network.debug_level + G.CONFIG.network.debug_category, G.CONFIG.network.debug_level ) return 1 @@ -141,11 +139,11 @@ def config_log_category_cb(data, option): def level_to_logbook(value): if value == 0: return logbook.ERROR - elif value == 1: + if value == 1: return logbook.WARNING - elif value == 2: + if value == 2: return logbook.INFO - elif value == 3: + if value == 3: return logbook.DEBUG return logbook.ERROR @@ -154,13 +152,13 @@ def level_to_logbook(value): def logbook_category(value): if value == 0: return "all" - elif value == 1: + if value == 1: return "http" - elif value == 2: + if value == 2: return "client" - elif value == 3: + if value == 3: return "events" - elif value == 4: + if value == 4: return "responses" return "all" @@ -169,9 +167,7 @@ def logbook_category(value): class WeechatConfig(object): def __init__(self, sections): self._ptr = W.config_new( - SCRIPT_NAME, - SCRIPT_NAME + "_config_reload_cb", - "" + SCRIPT_NAME, SCRIPT_NAME + "_config_reload_cb", "" ) for section in sections: @@ -180,8 +176,11 @@ class WeechatConfig(object): setattr(self, name, section_class(name, self._ptr, options)) def free(self): - for section in [getattr(self, a) for a in dir(self) if - isinstance(getattr(self, a), ConfigSection)]: + for section in [ + getattr(self, a) + for a in dir(self) + if isinstance(getattr(self, a), ConfigSection) + ]: section.free() W.config_free(self._ptr) @@ -190,9 +189,9 @@ class WeechatConfig(object): return_code = W.config_read(self._ptr) if return_code == W.WEECHAT_CONFIG_READ_OK: return True - elif return_code == W.WEECHAT_CONFIG_READ_MEMORY_ERROR: + if return_code == W.WEECHAT_CONFIG_READ_MEMORY_ERROR: return False - elif return_code == W.WEECHAT_CONFIG_READ_FILE_NOT_FOUND: + if return_code == W.WEECHAT_CONFIG_READ_FILE_NOT_FOUND: return True return False @@ -201,8 +200,9 @@ class ConfigSection(object): @classmethod def build(cls, name, options): def constructor(self, name, config_ptr, options): - self._ptr = W.config_new_section(config_ptr, name, 0, 0, "", "", - "", "", "", "", "", "", "", "") + self._ptr = W.config_new_section( + config_ptr, name, 0, 0, "", "", "", "", "", "", "", "", "", "" + ) self._config_ptr = config_ptr self._option_ptrs = {} @@ -211,18 +211,13 @@ class ConfigSection(object): attributes = { option.name: cls.option_property( - option.name, - option.type, - cast_func=option.cast_func - ) for option in options + option.name, option.type, cast_func=option.cast_func + ) + for option in options } attributes["__init__"] = constructor - section_class = type( - name.title() + "Section", - (cls,), - attributes - ) + section_class = type(name.title() + "Section", (cls,), attributes) return section_class def free(self): @@ -232,10 +227,24 @@ class ConfigSection(object): def _add_option(self, option): cb = option.change_callback.__name__ if option.change_callback else "" option_ptr = W.config_new_option( - self._config_ptr, self._ptr, option.name, option.type, - option.description, option.string_values, option.min, - option.max, option.value, option.value, 0, "", "", - cb, "", "", "") + self._config_ptr, + self._ptr, + option.name, + option.type, + option.description, + option.string_values, + option.min, + option.max, + option.value, + option.value, + 0, + "", + "", + cb, + "", + "", + "", + ) self._option_ptrs[option.name] = option_ptr @@ -249,25 +258,21 @@ class ConfigSection(object): def str_evaluate_getter(self): return W.string_eval_expression( - W.config_string(self._option_ptrs[name]), - {}, - {}, - {} + W.config_string(self._option_ptrs[name]), {}, {}, {} ) def int_getter(self): if cast_func: return cast_func(W.config_integer(self._option_ptrs[name])) - else: - return W.config_integer(self._option_ptrs[name]) + return W.config_integer(self._option_ptrs[name]) - if option_type == "string" or option_type == "color": + if option_type in ("string", "color"): if evaluate: return property(str_evaluate_getter) return property(str_getter) - elif option_type == "boolean": + if option_type == "boolean": return property(bool_getter) - elif option_type == "integer": + if option_type == "integer": return property(int_getter) @@ -276,45 +281,111 @@ class MatrixConfig(WeechatConfig): self.debug_buffer = "" self.debug_category = "all" + self.page_up_hook = None look_options = [ - Option("redactions", "integer", "strikethrough|notice|delete", 0, - 0, "strikethrough", - ("Only notice redactions, strike through or delete " - "redacted messages"), RedactType), - Option("server_buffer", "integer", - "merge_with_core|merge_without_core|independent", 0, 0, - "merge_with_core", "Merge server buffers", ServerBufferType, - config_server_buffer_cb) + Option( + "redactions", + "integer", + "strikethrough|notice|delete", + 0, + 0, + "strikethrough", + ( + "Only notice redactions, strike through or delete " + "redacted messages" + ), + RedactType, + ), + Option( + "server_buffer", + "integer", + "merge_with_core|merge_without_core|independent", + 0, + 0, + "merge_with_core", + "Merge server buffers", + ServerBufferType, + config_server_buffer_cb, + ), ] network_options = [ - Option("max_initial_sync_events", "integer", "", 1, 10000, "30", - ("How many events to fetch during the initial sync")), - Option("max_backlog_sync_events", "integer", "", 1, 100, "10", - ("How many events to fetch during backlog fetching")), - Option("fetch_backlog_on_pgup", "boolean", "", 0, 0, "on", - ("Fetch messages in the backlog on a window page up event") - ), - Option("debug_level", "integer", "error|warn|info|debug", 0, 0, - "error", "Enable network protocol debugging.", - level_to_logbook, config_log_level_cb), - Option("debug_category", "integer", - "all|http|client|events|responses", - 0, 0, "all", "Debugging category", logbook_category), - Option("debug_buffer", "boolean", "", 0, 0, "off", - ("Use a separate buffer for debug logs.")), + Option( + "max_initial_sync_events", + "integer", + "", + 1, + 10000, + "30", + ("How many events to fetch during the initial sync"), + ), + Option( + "max_backlog_sync_events", + "integer", + "", + 1, + 100, + "10", + ("How many events to fetch during backlog fetching"), + ), + Option( + "fetch_backlog_on_pgup", + "boolean", + "", + 0, + 0, + "on", + ("Fetch messages in the backlog on a window page up event"), + ), + Option( + "debug_level", + "integer", + "error|warn|info|debug", + 0, + 0, + "error", + "Enable network protocol debugging.", + level_to_logbook, + config_log_level_cb, + ), + Option( + "debug_category", + "integer", + "all|http|client|events|responses", + 0, + 0, + "all", + "Debugging category", + logbook_category, + ), + Option( + "debug_buffer", + "boolean", + "", + 0, + 0, + "off", + ("Use a separate buffer for debug logs."), + ), ] color_options = [ - Option("quote", "color", "", 0, 0, "lightgreen", - ("Color for matrix style blockquotes")) + Option( + "quote", + "color", + "", + 0, + 0, + "lightgreen", + ("Color for matrix style blockquotes"), + ) ] sections = [ ("network", network_options), ("look", look_options), - ("color", color_options) + ("color", color_options), ] super().__init__(sections) @@ -322,10 +393,23 @@ class MatrixConfig(WeechatConfig): # The server section is essentially a section with subsections and no # options, handle that case independently. W.config_new_section( - self._ptr, "server", 0, 0, "matrix_config_server_read_cb", "", - "matrix_config_server_write_cb", "", "", "", "", "", "", "") + self._ptr, + "server", + 0, + 0, + "matrix_config_server_read_cb", + "", + "matrix_config_server_write_cb", + "", + "", + "", + "", + "", + "", + "", + ) def free(self): - section_ptr = W.config_search_section(self._ptr, 'server') + section_ptr = W.config_search_section(self._ptr, "server") W.config_section_free(section_ptr) super().free() diff --git a/matrix/globals.py b/matrix/globals.py index db3f7d4..87440ba 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -18,13 +18,19 @@ from __future__ import unicode_literals import sys -from matrix.utf import WeechatWrapper +from .utf import WeechatWrapper + +if False: + from typing import Dict + try: import weechat + W = weechat if sys.hexversion >= 0x3000000 else WeechatWrapper(weechat) except ImportError: import matrix._weechat as weechat + W = weechat SERVERS = dict() # type: Dict[str, MatrixServer] diff --git a/matrix/server.py b/matrix/server.py index ba7a215..1f49375 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -17,30 +17,28 @@ from __future__ import unicode_literals import os -import ssl -import socket -import time import pprint - -from collections import deque, defaultdict +import socket +import ssl +import time +from collections import defaultdict, deque from nio import ( HttpClient, + LocalProtocolError, LoginResponse, - SyncRepsponse, RoomSendResponse, + SyncRepsponse, TransportResponse, TransportType, - LocalProtocolError ) -from matrix.utils import (key_from_value, server_buffer_prnt, - create_server_buffer) -from matrix.utf import utf8_decode from . import globals as G -from matrix.globals import W, SERVERS, SCRIPT_NAME -from .buffer import RoomBuffer, OwnMessage, OwnAction -from .config import Option, ServerBufferType, ConfigSection +from .buffer import OwnAction, OwnMessage, RoomBuffer +from .config import ConfigSection, Option, ServerBufferType +from .globals import SCRIPT_NAME, SERVERS, W +from .utf import utf8_decode +from .utils import create_server_buffer, key_from_value, server_buffer_prnt try: FileNotFoundError @@ -56,39 +54,101 @@ class ServerConfig(ConfigSection): self._option_ptrs = {} options = [ - Option('autoconnect', 'boolean', '', 0, 0, 'off', - ("automatically connect to the matrix server when weechat " - "is starting")), - Option('address', 'string', '', 0, 0, '', - "Hostname or IP address for the server"), - Option('port', 'integer', '', 0, 65535, '443', - "Port for the server"), - Option('proxy', 'string', '', 0, 0, '', - ("Name of weechat proxy to use (see /help proxy)")), - Option('ssl_verify', 'boolean', '', 0, 0, 'on', - ("Check that the SSL connection is fully trusted")), - Option('username', 'string', '', 0, 0, '', - "Username to use on server"), Option( - 'password', 'string', '', 0, 0, '', - ("Password for server (note: content is evaluated, see /help " - "eval)")), - Option('device_name', 'string', '', 0, 0, 'Weechat Matrix', - "Device name to use while logging in to the matrix server"), + "autoconnect", + "boolean", + "", + 0, + 0, + "off", + ( + "automatically connect to the matrix server when weechat " + "is starting" + ), + ), + Option( + "address", + "string", + "", + 0, + 0, + "", + "Hostname or IP address for the server", + ), + Option( + "port", "integer", "", 0, 65535, "443", "Port for the server" + ), + Option( + "proxy", + "string", + "", + 0, + 0, + "", + ("Name of weechat proxy to use (see /help proxy)"), + ), + Option( + "ssl_verify", + "boolean", + "", + 0, + 0, + "on", + ("Check that the SSL connection is fully trusted"), + ), + Option( + "username", "string", "", 0, 0, "", "Username to use on server" + ), + Option( + "password", + "string", + "", + 0, + 0, + "", + ( + "Password for server (note: content is evaluated, see " + "/help eval)" + ), + ), + Option( + "device_name", + "string", + "", + 0, + 0, + "Weechat Matrix", + "Device name to use while logging in to the matrix server", + ), ] - section = W.config_search_section(config_ptr, 'server') + section = W.config_search_section(config_ptr, "server") self._ptr = section for option in options: option_name = "{server}.{option}".format( - server=self._server_name, option=option.name) + server=self._server_name, option=option.name + ) self._option_ptrs[option.name] = W.config_new_option( - config_ptr, section, option_name, option.type, - option.description, option.string_values, option.min, - option.max, option.value, option.value, 0, "", "", - "matrix_config_server_change_cb", self._server_name, "", "") + config_ptr, + section, + option_name, + option.type, + option.description, + option.string_values, + option.min, + option.max, + option.value, + option.value, + 0, + "", + "", + "matrix_config_server_change_cb", + self._server_name, + "", + "", + ) autoconnect = ConfigSection.option_property("autoconnect", "boolean") address = ConfigSection.option_property("address", "string") @@ -98,9 +158,7 @@ class ServerConfig(ConfigSection): username = ConfigSection.option_property("username", "string") device_name = ConfigSection.option_property("device_name", "string") password = ConfigSection.option_property( - "password", - "string", - evaluate=True + "password", "string", evaluate=True ) def free(self): @@ -169,12 +227,13 @@ class MatrixServer(object): def _create_session_dir(self): path = os.path.join("matrix", self.name) if not W.mkdir_home(path, 0o700): - message = ("{prefix}matrix: Error creating server session " - "directory").format(prefix=W.prefix("error")) + message = ( + "{prefix}matrix: Error creating server session " "directory" + ).format(prefix=W.prefix("error")) W.prnt("", message) def get_session_path(self): - home_dir = W.info_get('weechat_dir', '') + home_dir = W.info_get("weechat_dir", "") return os.path.join(home_dir, "matrix", self.name) def _load_device_id(self): @@ -184,8 +243,8 @@ class MatrixServer(object): if not os.path.isfile(path): return - with open(path, 'r') as f: - device_id = f.readline().rstrip() + with open(path, "r") as device_file: + device_id = device_file.readline().rstrip() if device_id: self.device_id = device_id @@ -193,11 +252,11 @@ class MatrixServer(object): file_name = "{}{}".format(self.config.username, ".device_id") path = os.path.join(self.get_session_path(), file_name) - with open(path, 'w') as f: - f.write(self.device_id) + with open(path, "w") as device_file: + device_file.write(self.device_id) def _change_client(self): - host = ':'.join([self.config.address, str(self.config.port)]) + host = ":".join([self.config.address, str(self.config.port)]) self.client = HttpClient(host, self.config.username, self.device_id) def update_option(self, option, option_name): @@ -255,14 +314,17 @@ class MatrixServer(object): strerr = error.strerror if error.strerror else "Unknown reason" strerr = errno + strerr - error_message = ("{prefix}Error while writing to " - "socket: {error}").format( - prefix=W.prefix("network"), error=strerr) + error_message = ( + "{prefix}Error while writing to " "socket: {error}" + ).format(prefix=W.prefix("network"), error=strerr) server_buffer_prnt(self, error_message) server_buffer_prnt( - self, ("{prefix}matrix: disconnecting from server..." - ).format(prefix=W.prefix("network"))) + self, + ("{prefix}matrix: disconnecting from server...").format( + prefix=W.prefix("network") + ), + ) self.disconnect() return False @@ -273,10 +335,15 @@ class MatrixServer(object): server_buffer_prnt( self, "{prefix}matrix: Error while writing to socket".format( - prefix=W.prefix("network"))) + prefix=W.prefix("network") + ), + ) server_buffer_prnt( - self, ("{prefix}matrix: disconnecting from server..." - ).format(prefix=W.prefix("network"))) + self, + ("{prefix}matrix: disconnecting from server...").format( + prefix=W.prefix("network") + ), + ) self.disconnect() return False @@ -286,7 +353,6 @@ class MatrixServer(object): return True def _abort_send(self): - self.current_message = None self.send_buffer = "" def _finalize_send(self): @@ -316,8 +382,9 @@ class MatrixServer(object): return True def reconnect(self): - message = ("{prefix}matrix: reconnecting to server..." - ).format(prefix=W.prefix("network")) + message = ("{prefix}matrix: reconnecting to server...").format( + prefix=W.prefix("network") + ) server_buffer_prnt(self, message) @@ -336,9 +403,9 @@ class MatrixServer(object): else: self.reconnect_delay = 10 - message = ("{prefix}matrix: reconnecting to server in {t} " - "seconds").format( - prefix=W.prefix("network"), t=self.reconnect_delay) + message = ( + "{prefix}matrix: reconnecting to server in {t} " "seconds" + ).format(prefix=W.prefix("network"), t=self.reconnect_delay) server_buffer_prnt(self, message) @@ -364,7 +431,6 @@ class MatrixServer(object): self.access_token = "" self.send_buffer = b"" - self.current_message = None self.transport_type = None try: @@ -378,8 +444,9 @@ class MatrixServer(object): self.reconnect_time = None if self.server_buffer: - message = ("{prefix}matrix: disconnected from server" - ).format(prefix=W.prefix("network")) + message = ("{prefix}matrix: disconnected from server").format( + prefix=W.prefix("network") + ) server_buffer_prnt(self, message) if reconnect: @@ -390,13 +457,15 @@ class MatrixServer(object): if not self.config.address or not self.config.port: W.prnt("", self.config.address) message = "{prefix}Server address or port not set".format( - prefix=W.prefix("error")) + prefix=W.prefix("error") + ) W.prnt("", message) return False if not self.config.username or not self.config.password: message = "{prefix}User or password not set".format( - prefix=W.prefix("error")) + prefix=W.prefix("error") + ) W.prnt("", message) return False @@ -407,54 +476,59 @@ class MatrixServer(object): create_server_buffer(self) if not self.timer_hook: - self.timer_hook = W.hook_timer(1 * 1000, 0, 0, "matrix_timer_cb", - self.name) + self.timer_hook = W.hook_timer( + 1 * 1000, 0, 0, "matrix_timer_cb", self.name + ) ssl_message = " (SSL)" if self.ssl_context.check_hostname else "" - message = ("{prefix}matrix: Connecting to " - "{server}:{port}{ssl}...").format( - prefix=W.prefix("network"), - server=self.config.address, - port=self.config.port, - ssl=ssl_message) + message = ( + "{prefix}matrix: Connecting to " "{server}:{port}{ssl}..." + ).format( + prefix=W.prefix("network"), + server=self.config.address, + port=self.config.port, + ssl=ssl_message, + ) W.prnt(self.server_buffer, message) - W.hook_connect(self.config.proxy, - self.config.address, self.config.port, - 1, 0, "", "connect_cb", - self.name) + W.hook_connect( + self.config.proxy, + self.config.address, + self.config.port, + 1, + 0, + "", + "connect_cb", + self.name, + ) return True def schedule_sync(self): self.sync_time = time.time() - def sync(self, timeout=None, filter=None): + def sync(self, timeout=None, sync_filter=None): # type: (Optional[int], Optional[Dict[Any, Any]]) -> None self.sync_time = None - _, request = self.client.sync(timeout, filter) + _, request = self.client.sync(timeout, sync_filter) self.send_or_queue(request) def login(self): # type: () -> None if self.client.logged_in: - msg = ("{prefix}{script_name}: Already logged in, " - "syncing...").format( - prefix=W.prefix("network"), - script_name=SCRIPT_NAME - ) + msg = ( + "{prefix}{script_name}: Already logged in, " "syncing..." + ).format(prefix=W.prefix("network"), script_name=SCRIPT_NAME) W.prnt(self.server_buffer, msg) - timeout = (0 if self.transport_type == TransportType.HTTP - else 30000) + timeout = 0 if self.transport_type == TransportType.HTTP else 30000 sync_filter = {"room": {"timeline": {"limit": 5000}}} self.sync(timeout, sync_filter) return _, request = self.client.login( - self.config.password, - self.config.device_name + self.config.password, self.config.device_name ) self.send_or_queue(request) @@ -469,30 +543,24 @@ class MatrixServer(object): return _, request = self.client.room_put_state( - room_buffer.room.room_id, - event_type, - body + room_buffer.room.room_id, event_type, body ) self.send_or_queue(request) def room_send_redaction(self, room_buffer, event_id, reason=None): _, request = self.client.room_redact( - room_buffer.room.room_id, - event_id, - reason) + room_buffer.room.room_id, event_id, reason + ) self.send_or_queue(request) def room_kick(self, room_buffer, user_id, reason=None): _, request = self.client.room_kick( - room_buffer.room.room_id, - user_id, - reason) + room_buffer.room.room_id, user_id, reason + ) self.send_or_queue(request) def room_invite(self, room_buffer, user_id): - _, request = self.client.room_invite( - room_buffer.room.room_id, - user_id) + _, request = self.client.room_invite(room_buffer.room.room_id, user_id) self.send_or_queue(request) def room_join(self, room_id): @@ -514,11 +582,7 @@ class MatrixServer(object): message_class = OwnMessage own_message = message_class( - self.user_id, - 0, - "", - room_buffer.room.room_id, - formatted + self.user_id, 0, "", room_buffer.room.room_id, formatted ) body = {"msgtype": msgtype, "body": formatted.to_plain()} @@ -528,20 +592,22 @@ class MatrixServer(object): body["formatted_body"] = formatted.to_html() uuid, request = self.client.room_send( - room_buffer.room.room_id, - "m.room.message", - body + room_buffer.room.room_id, "m.room.message", body ) self.own_message_queue[uuid] = own_message self.send_or_queue(request) def _print_message_error(self, message): - server_buffer_prnt(self, - ("{prefix}Unhandled {status_code} error, please " - "inform the developers about this.").format( - prefix=W.prefix("error"), - status_code=message.response.status)) + server_buffer_prnt( + self, + ( + "{prefix}Unhandled {status_code} error, please " + "inform the developers about this." + ).format( + prefix=W.prefix("error"), status_code=message.response.status + ), + ) server_buffer_prnt(self, pprint.pformat(message.__class__.__name__)) server_buffer_prnt(self, pprint.pformat(message.request.payload)) @@ -555,21 +621,13 @@ class MatrixServer(object): if isinstance(message, OwnAction): room_buffer.self_action(message) return - elif isinstance(message, OwnMessage): + if isinstance(message, OwnMessage): room_buffer.self_message(message) return - raise NotImplementedError("Unsupported message of type {}".format( - type(message))) - - def _handle_erorr_response(self, response): - message = ("{prefix}matrix: {error}").format( - prefix=W.prefix("error"), error=self.error_message) - - W.prnt(self.server.server_buffer, message) - - if self.fatal: - self.server.disconnect(reconnect=False) + raise NotImplementedError( + "Unsupported message of type {}".format(type(message)) + ) def _handle_login(self, response): self.access_token = response.access_token @@ -579,7 +637,8 @@ class MatrixServer(object): self.save_device_id() message = "{prefix}matrix: Logged in as {user}".format( - prefix=W.prefix("network"), user=self.user_id) + prefix=W.prefix("network"), user=self.user_id + ) W.prnt(self.server_buffer, message) @@ -590,12 +649,10 @@ class MatrixServer(object): sync_filter = { "room": { - "timeline": { - "limit": G.CONFIG.network.max_initial_sync_events - } + "timeline": {"limit": G.CONFIG.network.max_initial_sync_events} } } - self.sync(timeout=0, filter=sync_filter) + self.sync(timeout=0, sync_filter=sync_filter) def _handle_room_info(self, response): for room_id, info in response.rooms.invite.items(): @@ -604,21 +661,23 @@ class MatrixServer(object): if room: if room.inviter: inviter_msg = " by {}{}".format( - W.color("chat_nick_other"), - room.inviter) + W.color("chat_nick_other"), room.inviter + ) else: inviter_msg = "" - self.info("You have been invited to {} {}({}{}{}){}" - "{}".format( - room.display_name(), - W.color("chat_delimiters"), - W.color("chat_channel"), - room_id, - W.color("chat_delimiters"), - W.color("reset"), - inviter_msg - )) + self.info( + "You have been invited to {} {}({}{}{}){}" + "{}".format( + room.display_name(), + W.color("chat_delimiters"), + W.color("chat_channel"), + room_id, + W.color("chat_delimiters"), + W.color("reset"), + inviter_msg, + ) + ) else: self.info("You have been invited to {}.".format(room_id)) @@ -647,13 +706,17 @@ class MatrixServer(object): self.schedule_sync() def handle_transport_response(self, response): - self.error(("Error with response of type type: {}, " - "error code {}").format( - response.request_info.type, response.status_code)) + self.error( + ("Error with response of type type: {}, " "error code {}").format( + response.request_info.type, response.status_code + ) + ) # TODO better error handling. - if (response.request_info.type == "sync" or - response.request_info.type == "login"): + if ( + response.request_info.type == "sync" + or response.request_info.type == "login" + ): self.disconnect() def handle_response(self, response): @@ -681,8 +744,6 @@ class MatrixServer(object): elif isinstance(response, RoomSendResponse): self.handle_own_messages(response) - return - def create_room_buffer(self, room_id): room = self.client.rooms[room_id] buf = RoomBuffer(room, self.name) @@ -722,8 +783,7 @@ class MatrixServer(object): break if first: num = W.buffer_get_integer( - W.buffer_search_main(), - "number" + W.buffer_search_main(), "number" ) W.buffer_unmerge(buf, num + 1) if buf is not first: @@ -734,13 +794,14 @@ class MatrixServer(object): @utf8_decode -def matrix_config_server_read_cb(data, config_file, section, option_name, - value): +def matrix_config_server_read_cb( + data, config_file, section, option_name, value +): return_code = W.WEECHAT_CONFIG_OPTION_SET_ERROR if option_name: - server_name, option = option_name.rsplit('.', 1) + server_name, option = option_name.rsplit(".", 1) server = None if server_name in SERVERS: @@ -752,9 +813,7 @@ def matrix_config_server_read_cb(data, config_file, section, option_name, # Ignore invalid options if option in server.config._option_ptrs: return_code = W.config_option_set( - server.config._option_ptrs[option], - value, - 1 + server.config._option_ptrs[option], value, 1 ) # TODO print out error message in case of erroneous return_code @@ -796,8 +855,11 @@ def matrix_timer_cb(server_name, remaining_calls): current_time = time.time() - if ((not server.connected) and server.reconnect_time and - current_time >= (server.reconnect_time + server.reconnect_delay)): + if ( + (not server.connected) + and server.reconnect_time + and current_time >= (server.reconnect_time + server.reconnect_delay) + ): server.reconnect() return W.WEECHAT_RC_OK @@ -844,7 +906,7 @@ def matrix_timer_cb(server_name, remaining_calls): def create_default_server(config_file): - server = MatrixServer('matrix_org', config_file._ptr) + server = MatrixServer("matrix_org", config_file._ptr) SERVERS[server.name] = server option = W.config_get(SCRIPT_NAME + ".server." + server.name + ".address") diff --git a/matrix/utf.py b/matrix/utf.py index 6250763..320cb58 100644 --- a/matrix/utf.py +++ b/matrix/utf.py @@ -28,7 +28,7 @@ import sys # pylint: disable=redefined-builtin from builtins import bytes, str -from collections import Mapping, Iterable +from collections import Iterable, Mapping from functools import wraps # These functions were written by Trygve Aaberge for wee-slack and are under a @@ -39,13 +39,11 @@ from functools import wraps class WeechatWrapper(object): - def __init__(self, wrapped_class): self.wrapped_class = wrapped_class # Helper method used to encode/decode method calls. def wrap_for_utf8(self, method): - def hooked(*args, **kwargs): result = method(*encode_to_utf8(args), **encode_to_utf8(kwargs)) # Prevent wrapped_class from becoming unwrapped @@ -69,7 +67,8 @@ class WeechatWrapper(object): def prnt_date_tags(self, buffer, date, tags, message): message = message.replace("\n", "\n \t") return self.wrap_for_utf8(self.wrapped_class.prnt_date_tags)( - buffer, date, tags, message) + buffer, date, tags, message + ) def utf8_decode(function): @@ -92,7 +91,7 @@ def utf8_decode(function): def decode_from_utf8(data): if isinstance(data, bytes): - return data.decode('utf-8') + return data.decode("utf-8") if isinstance(data, str): return data elif isinstance(data, Mapping): @@ -104,7 +103,7 @@ def decode_from_utf8(data): def encode_to_utf8(data): if isinstance(data, str): - return data.encode('utf-8') + return data.encode("utf-8") if isinstance(data, bytes): return data elif isinstance(data, Mapping): diff --git a/matrix/utils.py b/matrix/utils.py index 7172b4e..b59e472 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -15,18 +15,10 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import unicode_literals -from builtins import str import time -import math -from matrix import globals as G -from matrix.globals import W, SERVERS - -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse +from .globals import W def key_from_value(dictionary, value): @@ -45,11 +37,13 @@ def server_buffer_prnt(server, string): def tags_from_line_data(line_data): # type: (weechat.hdata) -> List[str] tags_count = W.hdata_get_var_array_size( - W.hdata_get('line_data'), line_data, 'tags_array') + W.hdata_get("line_data"), line_data, "tags_array" + ) tags = [ W.hdata_string( - W.hdata_get('line_data'), line_data, '%d|tags_array' % i) + W.hdata_get("line_data"), line_data, "%d|tags_array" % i + ) for i in range(tags_count) ] @@ -59,16 +53,15 @@ def tags_from_line_data(line_data): def create_server_buffer(server): # type: (MatrixServer) -> None buffer_name = "server.{}".format(server.name) - server.server_buffer = W.buffer_new(buffer_name, "server_buffer_cb", - server.name, "", "") + server.server_buffer = W.buffer_new( + buffer_name, "server_buffer_cb", server.name, "", "" + ) server_buffer_set_title(server) W.buffer_set(server.server_buffer, "short_name", server.name) - W.buffer_set(server.server_buffer, "localvar_set_type", 'server') + W.buffer_set(server.server_buffer, "localvar_set_type", "server") W.buffer_set( - server.server_buffer, - "localvar_set_nick", - server.config.username + server.server_buffer, "localvar_set_nick", server.config.username ) W.buffer_set(server.server_buffer, "localvar_set_server", server.name) W.buffer_set(server.server_buffer, "localvar_set_channel", server.name) @@ -84,18 +77,12 @@ def server_buffer_set_title(server): ip_string = "" title = ("Matrix: {address}:{port}{ip}").format( - address=server.config.address, port=server.config.port, ip=ip_string) + address=server.config.address, port=server.config.port, ip=ip_string + ) W.buffer_set(server.server_buffer, "title", title) -def color_for_tags(color): - if color == "weechat.color.chat_nick_self": - option = W.config_get(color) - return W.config_string(option) - return color - - def server_ts_to_weechat(timestamp): # type: (float) -> int date = int(timestamp / 1000) @@ -112,241 +99,5 @@ def shorten_sender(sender): return strip_matrix_server(sender)[1:] -def sender_to_prefix_and_color(room, sender): - if sender in room.users: - user = room.users[sender] - prefix = user.prefix - prefix_color = get_prefix_color(prefix) - return prefix, prefix_color - - return None, None - - -def sender_to_nick_and_color(room, sender): - nick = sender - nick_color_name = "default" - - if sender in room.users: - user = room.users[sender] - nick = (user.display_name if user.display_name else user.name) - nick_color_name = user.nick_color - else: - nick = sender - nick_color_name = W.info_get("nick_color_name", nick) - - return (nick, nick_color_name) - - -def tags_for_message(message_type): - default_tags = { - "message": ["matrix_message", "notify_message", "log1"], - "backlog": - ["matrix_message", "notify_message", "no_log", "no_highlight"] - } - - return default_tags[message_type] - - -def add_event_tags(event_id, nick, color=None, tags=[]): - tags.append("nick_{nick}".format(nick=nick)) - - if color: - tags.append("prefix_nick_{color}".format(color=color_for_tags(color))) - - tags.append("matrix_id_{event_id}".format(event_id=event_id)) - - return tags - - -def sanitize_token(string): - # type: (str) -> str - string = sanitize_string(string) - - if len(string) > 512: - raise ValueError - - if string == "": - raise ValueError - - return string - - -def sanitize_string(string): - # type: (str) -> str - if not isinstance(string, str): - raise TypeError - - # string keys can have empty string values sometimes (e.g. room names that - # got deleted) - if string == "": - return None - - remap = { - ord('\b'): None, - ord('\f'): None, - ord('\n'): None, - ord('\r'): None, - ord('\t'): None, - ord('\0'): None - } - - return string.translate(remap) - - -def sanitize_id(string): - # type: (str) -> str - string = sanitize_string(string) - - if len(string) > 128: - raise ValueError - - if string == "": - raise ValueError - - return string - - -def sanitize_int(number, minimum=None, maximum=None): - # type: (int, int, int) -> int - if not isinstance(number, int): - raise TypeError - - if math.isnan(number): - raise ValueError - - if math.isinf(number): - raise ValueError - - if minimum: - if number < minimum: - raise ValueError - - if maximum: - if number > maximum: - raise ValueError - - return number - - -def sanitize_ts(timestamp): - # type: (int) -> int - return sanitize_int(timestamp, 0) - - -def sanitize_power_level(level): - # type: (int) -> int - return sanitize_int(level, 0, 100) - - -def sanitize_text(string): - # type: (str) -> str - if not isinstance(string, str): - raise TypeError - - # yapf: disable - remap = { - ord('\b'): None, - ord('\f'): None, - ord('\r'): None, - ord('\0'): None - } - # yapf: enable - - return string.translate(remap) - - -def add_user_to_nicklist(buf, user_id, user): - group_name = "999|..." - - if user.power_level >= 100: - group_name = "000|o" - elif user.power_level >= 50: - group_name = "001|h" - elif user.power_level > 0: - group_name = "002|v" - - group = W.nicklist_search_group(buf, "", group_name) - prefix = user.prefix if user.prefix else " " - - # TODO make it configurable so we can use a display name or user_id here - W.nicklist_add_nick(buf, group, user_id, user.nick_color, prefix, - get_prefix_color(user.prefix), 1) - - -def get_prefix_for_level(level): - # type: (int) -> str - if level >= 100: - return "&" - elif level >= 50: - return "@" - elif level > 0: - return "+" - return "" - - -# TODO make this configurable -def get_prefix_color(prefix): - # type: (str) -> str - if prefix == "&": - return "lightgreen" - elif prefix == "@": - return "lightgreen" - elif prefix == "+": - return "yellow" - return "" - - def string_strikethrough(string): return "".join(["{}\u0336".format(c) for c in string]) - - -def line_pointer_and_tags_from_event(buff, event_id): - # type: (str, str) -> str - own_lines = W.hdata_pointer(W.hdata_get('buffer'), buff, 'own_lines') - - if own_lines: - hdata_line = W.hdata_get('line') - - line_pointer = W.hdata_pointer( - W.hdata_get('lines'), own_lines, 'last_line') - - while line_pointer: - data_pointer = W.hdata_pointer(hdata_line, line_pointer, 'data') - - if data_pointer: - tags = tags_from_line_data(data_pointer) - - message_id = event_id_from_tags(tags) - - if event_id == message_id: - return data_pointer, tags - - line_pointer = W.hdata_move(hdata_line, line_pointer, -1) - - return None, [] - - -def event_id_from_tags(tags): - # type: (List[str]) -> str - for tag in tags: - if tag.startswith("matrix_id"): - return tag[10:] - - return "" - - -def mxc_to_http(mxc): - # type: (str) -> str - url = urlparse(mxc) - - if url.scheme != "mxc": - return None - - if not url.netloc or not url.path: - return None - - return "https://{}/_matrix/media/r0/download/{}{}".format( - url.netloc, - url.netloc, - url.path - ) From 67141c980a61186c2bfc5a6df60cfee5162b8aa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Aug 2018 20:57:12 +0200 Subject: [PATCH 121/269] matrix: mypy fixes. --- matrix/buffer.py | 30 ++++++++------- matrix/colors.py | 7 ++-- matrix/commands.py | 2 +- matrix/globals.py | 9 +++-- matrix/server.py | 95 ++++++++++++++++++++++++---------------------- matrix/utils.py | 6 ++- 6 files changed, 82 insertions(+), 67 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index c9fffe7..49e922d 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -20,7 +20,7 @@ from __future__ import unicode_literals import time from builtins import super from functools import partial -from typing import NamedTuple +from typing import Dict, List, NamedTuple, Set, Optional from nio import ( Api, @@ -46,8 +46,8 @@ from .globals import SCRIPT_NAME, SERVERS, W from .utf import utf8_decode from .utils import server_ts_to_weechat, shorten_sender, string_strikethrough -OwnMessage = NamedTuple( - "OwnMessage", +OwnMessages = NamedTuple( + "OwnMessages", [ ("sender", str), ("age", int), @@ -58,6 +58,10 @@ OwnMessage = NamedTuple( ) +class OwnMessage(OwnMessages): + pass + + class OwnAction(OwnMessage): pass @@ -100,7 +104,7 @@ class WeechatUser(object): self.prefix = prefix self.color = W.info_get("nick_color_name", nick) self.join_time = join_time or time.time() - self.speaking_time = None # type: int + self.speaking_time = None # type: Optional[int] def update_speaking_time(self, new_time=None): self.speaking_time = new_time or time.time() @@ -434,7 +438,7 @@ class WeechatChannelBuffer(object): return color def _message_tags(self, user, message_type): - # type: (str, RoomUser, str) -> List[str] + # type: (WeechatUser, str) -> List[str] tags = list(self.tags[message_type]) tags.append("nick_{nick}".format(nick=user.nick)) @@ -476,7 +480,7 @@ class WeechatChannelBuffer(object): self.print_date_tags(data, date, tags) def message(self, nick, message, date, extra_tags=None): - # type: (str, str, int, str) -> None + # type: (str, str, int, List[str]) -> None user = self._get_user(nick) tags = self._message_tags(user, "message") + (extra_tags or []) self._print_message(user, message, date, tags) @@ -636,16 +640,16 @@ class WeechatChannelBuffer(object): if message: tags = self._message_tags(user, "join") - message = self._membership_message(user, "join") + msg = self._membership_message(user, "join") # TODO add a option to disable smart filters tags.append(SCRIPT_NAME + "_smart_filter") - self.print_date_tags(message, date, tags) + self.print_date_tags(msg, date, tags) self.add_smart_filtered_nick(user.nick) def invite(self, nick, date, extra_tags=None): - # type: (str, int, Optional[bool], Optional[List[str]]) -> None + # type: (str, int, Optional[List[str]]) -> None user = self._get_user(nick) tags = self._message_tags(user, "invite") message = self._membership_message(user, "invite") @@ -670,19 +674,19 @@ class WeechatChannelBuffer(object): if not user.spoken_recently: tags.append(SCRIPT_NAME + "_smart_filter") - message = self._membership_message(user, leave_type) - self.print_date_tags(message, date, tags + (extra_tags or [])) + msg = self._membership_message(user, leave_type) + self.print_date_tags(msg, date, tags + (extra_tags or [])) self.remove_smart_filtered_nick(user.nick) if user.nick in self.users: del self.users[user.nick] def part(self, nick, date, message=True, extra_tags=None): - # type: (str, int, Optional[bool], Optional[List[str]]) -> None + # type: (str, int, bool, Optional[List[str]]) -> None self._leave(nick, date, message, "part", extra_tags) def kick(self, nick, date, message=True, extra_tags=None): - # type: (str, int, Optional[bool], Optional[List[str]]) -> None + # type: (str, int, bool, Optional[List[str]]) -> None self._leave(nick, date, message, "kick", extra_tags) def _print_topic(self, nick, topic, date): diff --git a/matrix/colors.py b/matrix/colors.py index bc4f214..7405e0b 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -25,6 +25,7 @@ import textwrap # pylint: disable=redefined-builtin from builtins import str from collections import namedtuple +from typing import List import webcolors from pygments import highlight @@ -45,7 +46,7 @@ except ImportError: FormattedString = namedtuple("FormattedString", ["text", "attributes"]) -class Formatted: +class Formatted(object): def __init__(self, substrings): # type: (List[FormattedString]) -> None self.substrings = substrings @@ -247,7 +248,7 @@ class Formatted: # TODO do we want at least some formatting using unicode # (strikethrough, quotes)? def to_plain(self): - # type: (List[FormattedString]) -> str + # type: () -> str def strip_atribute(string, _, __): return string @@ -688,7 +689,7 @@ def color_html_to_weechat(color): try: rgb_color = webcolors.html5_parse_legacy_color(color) except ValueError: - return None + return "" if rgb_color in weechat_basic_colors: return weechat_basic_colors[rgb_color] diff --git a/matrix/commands.py b/matrix/commands.py index beaed12..10d16d2 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -478,7 +478,7 @@ def matrix_kick_command_cb(data, buffer, args): def event_id_from_line(buf, target_number): - # type: (weechat.buffer, int) -> str + # type: (str, int) -> str own_lines = W.hdata_pointer(W.hdata_get("buffer"), buf, "own_lines") if own_lines: line = W.hdata_pointer(W.hdata_get("lines"), own_lines, "last_line") diff --git a/matrix/globals.py b/matrix/globals.py index 87440ba..f68c41a 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -20,8 +20,11 @@ import sys from .utf import WeechatWrapper +from typing import Dict, Optional + if False: - from typing import Dict + from .server import MatrixServer + from .config import MatrixConfig try: @@ -29,11 +32,11 @@ try: W = weechat if sys.hexversion >= 0x3000000 else WeechatWrapper(weechat) except ImportError: - import matrix._weechat as weechat + import matrix._weechat as weechat # type: ignore W = weechat SERVERS = dict() # type: Dict[str, MatrixServer] -CONFIG = None # type: MatrixConfig +CONFIG = None # type: Optional[MatrixConfig] ENCRYPTION = True # type: bool SCRIPT_NAME = "matrix" # type: str diff --git a/matrix/server.py b/matrix/server.py index 1f49375..4149b29 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -22,6 +22,10 @@ import socket import ssl import time from collections import defaultdict, deque +from typing import Any, Deque, Dict, Optional + +if False: + from .colors import Formatted from nio import ( HttpClient, @@ -31,6 +35,8 @@ from nio import ( SyncRepsponse, TransportResponse, TransportType, + Rooms, + Response ) from . import globals as G @@ -41,7 +47,7 @@ from .utf import utf8_decode from .utils import create_server_buffer, key_from_value, server_buffer_prnt try: - FileNotFoundError + FileNotFoundError # type: ignore except NameError: FileNotFoundError = IOError @@ -51,7 +57,7 @@ class ServerConfig(ConfigSection): # type: (str, str) -> None self._server_name = server_name self._config_ptr = config_ptr - self._option_ptrs = {} + self._option_ptrs = {} # type: Dict[str, str] options = [ Option( @@ -167,32 +173,29 @@ class ServerConfig(ConfigSection): class MatrixServer(object): # pylint: disable=too-many-instance-attributes - def __init__(self, name, config_file): - # type: (str, weechat.config) -> None + def __init__(self, name, config_ptr): + # type: (str, str) -> None # yapf: disable self.name = name # type: str self.user_id = "" self.device_id = "" # type: str - self.olm = None # type: Olm - self.encryption_queue = defaultdict(deque) - - self.room_buffers = dict() # type: Dict[str, WeechatChannelBuffer] - self.buffers = dict() # type: Dict[str, weechat.buffer] - self.server_buffer = None # type: weechat.buffer - self.fd_hook = None # type: weechat.hook - self.ssl_hook = None # type: weechat.hook - self.timer_hook = None # type: weechat.hook - self.numeric_address = "" # type: str + self.room_buffers = dict() # type: Dict[str, RoomBuffer] + self.buffers = dict() # type: Dict[str, str] + self.server_buffer = None # type: Optional[str] + self.fd_hook = None # type: Optional[str] + self.ssl_hook = None # type: Optional[str] + self.timer_hook = None # type: Optional[str] + self.numeric_address = "" # type: Optional[str] self.connected = False # type: bool self.connecting = False # type: bool self.reconnect_delay = 0 # type: int - self.reconnect_time = None # type: float + self.reconnect_time = None # type: Optional[float] self.sync_time = None # type: Optional[float] - self.socket = None # type: ssl.SSLSocket + self.socket = None # type: Optional[ssl.SSLSocket] self.ssl_context = ssl.create_default_context() # type: ssl.SSLContext - self.transport_type = None # type: Optional[nio.TransportType] + self.transport_type = None # type: Optional[TransportType] # Enable http2 negotiation on the ssl context. self.ssl_context.set_alpn_protocols(["h2", "http/1.1"]) @@ -203,24 +206,19 @@ class MatrixServer(object): pass self.client = None - self.access_token = None # type: str - self.next_batch = None # type: str + self.access_token = None # type: Optional[str] + self.next_batch = None # type: Optional[str] self.transaction_id = 0 # type: int self.lag = 0 # type: int self.lag_done = False # type: bool - self.send_fd_hook = None # type: weechat.hook + self.send_fd_hook = None # type: Optional[str] self.send_buffer = b"" # type: bytes - self.device_check_timestamp = None + self.device_check_timestamp = None # type: Optional[int] - self.send_queue = deque() - self.own_message_queue = dict() # type: Dict[OwnMessage] + self.own_message_queue = dict() # type: Dict[str, OwnMessage] - self.event_queue_timer = None - self.event_queue = deque() # type: Deque[RoomInfo] - - # self._create_options(config_file) - self.config = ServerConfig(self.name, config_file) + self.config = ServerConfig(self.name, config_ptr) self._create_session_dir() # yapf: enable @@ -287,13 +285,16 @@ class MatrixServer(object): def send_or_queue(self, request): # type: (bytes) -> None - if not self.send(request): - self.send_queue.append(request) + self.send(request) def try_send(self, message): # type: (MatrixServer, bytes) -> bool sock = self.socket + + if not sock: + return False + total_sent = 0 message_length = len(message) @@ -353,7 +354,7 @@ class MatrixServer(object): return True def _abort_send(self): - self.send_buffer = "" + self.send_buffer = b"" def _finalize_send(self): # type: (MatrixServer) -> None @@ -433,10 +434,11 @@ class MatrixServer(object): self.send_buffer = b"" self.transport_type = None - try: - self.client.disconnect() - except LocalProtocolError: - pass + if self.client: + try: + self.client.disconnect() + except LocalProtocolError: + pass self.lag = 0 W.bar_item_update("lag") @@ -511,12 +513,18 @@ class MatrixServer(object): def sync(self, timeout=None, sync_filter=None): # type: (Optional[int], Optional[Dict[Any, Any]]) -> None + if not self.client: + return + self.sync_time = None _, request = self.client.sync(timeout, sync_filter) self.send_or_queue(request) def login(self): # type: () -> None + if not self.client: + return + if self.client.logged_in: msg = ( "{prefix}{script_name}: Already logged in, " "syncing..." @@ -576,6 +584,9 @@ class MatrixServer(object): if room_buffer.room.encrypted: return + if not self.client: + return + if msgtype == "m.emote": message_class = OwnAction else: @@ -720,7 +731,7 @@ class MatrixServer(object): self.disconnect() def handle_response(self, response): - # type: (MatrixMessage) -> None + # type: (Response) -> None self.lag = response.elapsed * 1000 # If the response was a sync response and contained a timeout the @@ -836,7 +847,7 @@ def matrix_config_server_write_cb(data, config_file, section_name): @utf8_decode def matrix_config_server_change_cb(server_name, option): - # type: (str, weechat.config_option) -> int + # type: (str, str) -> int server = SERVERS[server_name] option_name = None @@ -881,14 +892,6 @@ def matrix_timer_cb(server_name, remaining_calls): sync_filter = {"room": {"timeline": {"limit": 5000}}} server.sync(timeout, sync_filter) - while server.send_queue: - message = server.send_queue.popleft() - if not server.send(message): - # We got an error while sending the last message return the message - # to the queue and exit the loop - server.send_queue.appendleft(message) - break - if not server.next_batch: return W.WEECHAT_RC_OK @@ -926,6 +929,6 @@ def send_cb(server_name, file_descriptor): server.send_fd_hook = None if server.send_buffer: - server.try_send(server, server.send_buffer) + server.try_send(server.send_buffer) return W.WEECHAT_RC_OK diff --git a/matrix/utils.py b/matrix/utils.py index b59e472..e10c3ea 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -17,6 +17,10 @@ from __future__ import unicode_literals import time +from typing import Any, Dict, List + +if False: + from .server import MatrixServer from .globals import W @@ -35,7 +39,7 @@ def server_buffer_prnt(server, string): def tags_from_line_data(line_data): - # type: (weechat.hdata) -> List[str] + # type: (str) -> List[str] tags_count = W.hdata_get_var_array_size( W.hdata_get("line_data"), line_data, "tags_array" ) From 798e77939147db186677cd74bb440cbe5cd37379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 30 Aug 2018 13:37:13 +0200 Subject: [PATCH 122/269] colors: Handle quoted code segments. --- matrix/_weechat.py | 4 ++++ matrix/colors.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/matrix/_weechat.py b/matrix/_weechat.py index a4ab139..54cbe37 100644 --- a/matrix/_weechat.py +++ b/matrix/_weechat.py @@ -168,3 +168,7 @@ def nicklist_remove_nick(*_, **__): def nicklist_search_nick(*args, **kwargs): return buffer_new(args, kwargs) + + +def string_remove_color(message, _): + return message diff --git a/matrix/colors.py b/matrix/colors.py index 7405e0b..6e0f878 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -334,12 +334,36 @@ class Formatted(object): ) attributes.pop("strikethrough") + def indent(text, prefix): + return prefix + text.replace("\n", "\n{}".format(prefix)) + for key, value in attributes.items(): + # Don't use textwrap to quote the code + if attributes["code"] and key == "quote" and value: + continue + text = add_attribute(text, key, value) - return text + + # If we're quoted code add quotation marks now. + if attributes["quote"] and key == "code" and value: + text = indent(text, "{}>{} ".format( + W.color(G.CONFIG.color.quote), + W.color("reset"))) + + # If we're code don't remove multiple newlines blindly + if attributes["code"]: + return text + return re.sub(r"\n+", "\n", text) weechat_strings = map(format_string, self.substrings) - return re.sub(r"\n+", "\n", "".join(weechat_strings)).strip() + + # Remove duplicate \n elements from the list + strings = [] + for string in weechat_strings: + if len(strings) == 0 or string != "\n" or string != strings[-1]: + strings.append(string) + + return "".join(strings).strip() # TODO this should be a typed dict. From d97655d68c33765969d0529f2b8897aac8d6e271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 30 Aug 2018 16:35:05 +0200 Subject: [PATCH 123/269] matrix: Style fixes. --- matrix/buffer.py | 2 +- matrix/colors.py | 9 ++++++--- matrix/config.py | 2 +- matrix/globals.py | 3 +-- matrix/server.py | 11 ++++++----- matrix/utils.py | 4 ++-- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 49e922d..ec36a2e 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -20,7 +20,7 @@ from __future__ import unicode_literals import time from builtins import super from functools import partial -from typing import Dict, List, NamedTuple, Set, Optional +from typing import Dict, List, NamedTuple, Optional, Set from nio import ( Api, diff --git a/matrix/colors.py b/matrix/colors.py index 6e0f878..89b578d 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -346,9 +346,12 @@ class Formatted(object): # If we're quoted code add quotation marks now. if attributes["quote"] and key == "code" and value: - text = indent(text, "{}>{} ".format( - W.color(G.CONFIG.color.quote), - W.color("reset"))) + text = indent( + text, + "{}>{} ".format( + W.color(G.CONFIG.color.quote), W.color("reset") + ), + ) # If we're code don't remove multiple newlines blindly if attributes["code"]: diff --git a/matrix/config.py b/matrix/config.py index 7a82c14..5a94fad 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -20,8 +20,8 @@ from collections import namedtuple from enum import Enum, unique import logbook -import nio +import nio from matrix.globals import SCRIPT_NAME, SERVERS, W from matrix.utf import utf8_decode diff --git a/matrix/globals.py b/matrix/globals.py index f68c41a..b8e60d4 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -17,11 +17,10 @@ from __future__ import unicode_literals import sys +from typing import Dict, Optional from .utf import WeechatWrapper -from typing import Dict, Optional - if False: from .server import MatrixServer from .config import MatrixConfig diff --git a/matrix/server.py b/matrix/server.py index 4149b29..eeb12b1 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -24,19 +24,16 @@ import time from collections import defaultdict, deque from typing import Any, Deque, Dict, Optional -if False: - from .colors import Formatted - from nio import ( HttpClient, LocalProtocolError, LoginResponse, + Response, + Rooms, RoomSendResponse, SyncRepsponse, TransportResponse, TransportType, - Rooms, - Response ) from . import globals as G @@ -46,6 +43,10 @@ from .globals import SCRIPT_NAME, SERVERS, W from .utf import utf8_decode from .utils import create_server_buffer, key_from_value, server_buffer_prnt +if False: + from .colors import Formatted + + try: FileNotFoundError # type: ignore except NameError: diff --git a/matrix/utils.py b/matrix/utils.py index e10c3ea..060fd14 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -19,11 +19,11 @@ from __future__ import unicode_literals import time from typing import Any, Dict, List +from .globals import W + if False: from .server import MatrixServer -from .globals import W - def key_from_value(dictionary, value): # type: (Dict[str, Any], Any) -> str From bbfc0d2e7db9d84740c5e4a8c87eb7ab98bebd47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 3 Sep 2018 15:31:49 +0200 Subject: [PATCH 124/269] buffer: Fix output format for notices. --- matrix/buffer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index ec36a2e..b4a64c0 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -502,11 +502,11 @@ class WeechatChannelBuffer(object): ) user_string = "{}{}{}{}".format( - user_prefix, user.color, user.nick, W.color("reset") + user_prefix, W.color(user.color), user.nick, W.color("reset") ) data = ( - "{prefix}\t{color}Notice" + "{prefix}{color}Notice" "{del_color}({ncolor}{user}{del_color}){ncolor}" ": {message}" ).format( From 08c2d0e1130a2b0417987b48fa38bdf409021bc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 3 Sep 2018 15:32:18 +0200 Subject: [PATCH 125/269] buffer: Don't use the user id for discord and freenode users. --- matrix/buffer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index b4a64c0..a07f4ea 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -790,6 +790,14 @@ class RoomBuffer(object): short_name = shorten_sender(user.user_id) + # TODO handle this special case for discord bridge users and + # freenode bridge users better + if user.user_id.startswith("@_discord_"): + if user.display_name: + short_name = user.display_name + elif user.user_id.startswith("@freenode_"): + short_name = shorten_sender(user.user_id[10:]) + # TODO make this configurable if not short_name or short_name in self.displayed_nicks.values(): # Use the full user id, but don't include the @ From 8edbb1bfa92ddfaefcb05fb24c2560cb8c34ffd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 3 Sep 2018 19:20:58 +0200 Subject: [PATCH 126/269] server: Fix reconnect delay. --- main.py | 1 + matrix/server.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 323e572..7230229 100644 --- a/main.py +++ b/main.py @@ -314,6 +314,7 @@ def finalize_connection(server): server.fd_hook = hook server.connected = True server.connecting = False + server.reconnect_delay = 0 negotiated_protocol = server.socket.selected_alpn_protocol() diff --git a/matrix/server.py b/matrix/server.py index eeb12b1..c53d623 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -443,7 +443,6 @@ class MatrixServer(object): self.lag = 0 W.bar_item_update("lag") - self.reconnect_delay = 0 self.reconnect_time = None if self.server_buffer: @@ -454,6 +453,8 @@ class MatrixServer(object): if reconnect: self.schedule_reconnect() + else: + self.reconnect_delay = 0 def connect(self): # type: (MatrixServer) -> int From dc8df67f556694eb788e4c5946ee0f381dc85277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 3 Sep 2018 20:39:16 +0200 Subject: [PATCH 127/269] commands: Add backlog fetching back. --- main.py | 2 +- matrix/buffer.py | 24 ++++++++++++++++-------- matrix/commands.py | 6 +++--- matrix/config.py | 17 +++++++++++++++++ matrix/server.py | 36 +++++++++++++++++++++++++++++++++--- 5 files changed, 70 insertions(+), 15 deletions(-) diff --git a/main.py b/main.py index 7230229..071fe23 100644 --- a/main.py +++ b/main.py @@ -55,7 +55,7 @@ from matrix.completion import (init_completion, matrix_command_completion_cb, matrix_user_completion_cb) from matrix.config import (MatrixConfig, config_log_category_cb, config_log_level_cb, config_server_buffer_cb, - matrix_config_reload_cb) + matrix_config_reload_cb, config_pgup_cb) from matrix.globals import SCRIPT_NAME, SERVERS, W from matrix.server import (MatrixServer, create_default_server, matrix_config_server_change_cb, diff --git a/matrix/buffer.py b/matrix/buffer.py index a07f4ea..630bcce 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -755,9 +755,10 @@ class WeechatChannelBuffer(object): class RoomBuffer(object): - def __init__(self, room, server_name): + def __init__(self, room, server_name, prev_batch): self.room = room self.backlog_pending = False + self.prev_batch = prev_batch self.joined = True self.leave_event_id = None # type: Optional[str] @@ -1142,11 +1143,13 @@ class RoomBuffer(object): ] tags += self.get_event_tags(event) nick = self.find_nick(event.sender) - data = ( - event.formatted_message.to_weechat() - if event.formatted_message - else event.message - ) + + formatted = None + + if event.formatted_body: + formatted = Formatted.from_html(event.formatted_body) + + data = formatted.to_weechat() if formatted else event.body user = self.weechat_buffer._get_user(nick) date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer._print_message(user, data, date, tags) @@ -1185,8 +1188,10 @@ class RoomBuffer(object): new.date, new.date_printed, new.tags, new.prefix, new.message ) - def handle_backlog(self, events): - for event in events: + def handle_backlog(self, response): + self.prev_batch = response.end + + for event in response.chunk: if isinstance(event, RoomMessageText): self.old_message(event) elif isinstance(event, RedactedEvent): @@ -1194,6 +1199,9 @@ class RoomBuffer(object): self.sort_messages() + self.backlog_pending = False + W.bar_item_update("buffer_modes") + def handle_joined_room(self, info): for event in info.state: self.handle_state_event(event) diff --git a/matrix/commands.py b/matrix/commands.py index 10d16d2..880f163 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -246,8 +246,8 @@ def hook_commands(): W.hook_command_run("/buffer clear", "matrix_command_buf_clear_cb", "") - # if OPTIONS.enable_backlog: - # hook_page_up() + if G.CONFIG.network.fetch_backlog_on_pgup: + hook_page_up() @utf8_decode @@ -375,7 +375,7 @@ def matrix_command_pgup_cb(data, buffer, command): if first_line_displayed: room_id = key_from_value(server.buffers, buffer) - matrix_fetch_old_messages(server, room_id) + server.room_get_messages(room_id) return W.WEECHAT_RC_OK diff --git a/matrix/config.py b/matrix/config.py index 5a94fad..8daa890 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -136,6 +136,21 @@ def config_log_category_cb(data, option): return 1 +@utf8_decode +def config_pgup_cb(data, option): + if G.CONFIG.network.fetch_backlog_on_pgup: + if not G.CONFIG.page_up_hook: + G.CONFIG.page_up_hook = W.hook_command_run( + "/window page_up", "matrix_command_pgup_cb", "" + ) + else: + if G.CONFIG.page_up_hook: + W.unhook(G.CONFIG.page_up_hook) + G.CONFIG.page_up_hook = None + + return 1 + + def level_to_logbook(value): if value == 0: return logbook.ERROR @@ -337,6 +352,8 @@ class MatrixConfig(WeechatConfig): 0, "on", ("Fetch messages in the backlog on a window page up event"), + None, + config_pgup_cb, ), Option( "debug_level", diff --git a/matrix/server.py b/matrix/server.py index c53d623..cfbdc94 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -34,6 +34,7 @@ from nio import ( SyncRepsponse, TransportResponse, TransportType, + RoomMessagesResponse, ) from . import globals as G @@ -218,6 +219,7 @@ class MatrixServer(object): self.device_check_timestamp = None # type: Optional[int] self.own_message_queue = dict() # type: Dict[str, OwnMessage] + self.backlog_queue = dict() # type: Dict[str, str] self.config = ServerConfig(self.name, config_ptr) self._create_session_dir() @@ -581,6 +583,26 @@ class MatrixServer(object): _, request = self.client.room_leave(room_id) self.send_or_queue(request) + def room_get_messages(self, room_id): + room_buffer = self.find_room_from_id(room_id) + + # We're already fetching old messages + if room_buffer.backlog_pending: + return + + if not room_buffer.prev_batch: + return + + uuid, request = self.client.room_messages( + room_id, + room_buffer.prev_batch, + limit=10) + + room_buffer.backlog_pending = True + W.bar_item_update("buffer_modes") + self.backlog_queue[uuid] = room_id + self.send_or_queue(request) + def room_send_message(self, room_buffer, formatted, msgtype="m.text"): # type: (RoomBuffer, Formatted, str) -> None if room_buffer.room.encrypted: @@ -642,6 +664,12 @@ class MatrixServer(object): "Unsupported message of type {}".format(type(message)) ) + def handle_backlog_response(self, response): + room_id = self.backlog_queue.pop(response.uuid) + room_buffer = self.find_room_from_id(room_id) + + room_buffer.handle_backlog(response) + def _handle_login(self, response): self.access_token = response.access_token self.user_id = response.user_id @@ -703,7 +731,7 @@ class MatrixServer(object): for room_id, info in response.rooms.join.items(): if room_id not in self.buffers: - self.create_room_buffer(room_id) + self.create_room_buffer(room_id, info.timeline.prev_batch) room_buffer = self.find_room_from_id(room_id) room_buffer.handle_joined_room(info) @@ -756,10 +784,12 @@ class MatrixServer(object): elif isinstance(response, RoomSendResponse): self.handle_own_messages(response) + elif isinstance(response, RoomMessagesResponse): + self.handle_backlog_response(response) - def create_room_buffer(self, room_id): + def create_room_buffer(self, room_id, prev_batch): room = self.client.rooms[room_id] - buf = RoomBuffer(room, self.name) + buf = RoomBuffer(room, self.name, prev_batch) # TODO this should turned into a propper class self.room_buffers[room_id] = buf self.buffers[room_id] = buf.weechat_buffer._ptr From eab12eed086107857efd3803082ebddf1f98d357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 3 Sep 2018 20:39:44 +0200 Subject: [PATCH 128/269] buffer: Use a property to update bar items. --- matrix/buffer.py | 12 ++++++++++-- matrix/server.py | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 630bcce..379a5dc 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -757,7 +757,7 @@ class WeechatChannelBuffer(object): class RoomBuffer(object): def __init__(self, room, server_name, prev_batch): self.room = room - self.backlog_pending = False + self._backlog_pending = False self.prev_batch = prev_batch self.joined = True self.leave_event_id = None # type: Optional[str] @@ -772,6 +772,15 @@ class RoomBuffer(object): buffer_name, server_name, user ) + @property + def backlog_pending(self): + return self._backlog_pending + + @backlog_pending.setter + def backlog_pending(self, value): + self._backlog_pending = value + W.bar_item_update("buffer_modes") + def find_nick(self, user_id): # type: (str) -> str """Find a suitable nick from a user_id""" @@ -1200,7 +1209,6 @@ class RoomBuffer(object): self.sort_messages() self.backlog_pending = False - W.bar_item_update("buffer_modes") def handle_joined_room(self, info): for event in info.state: diff --git a/matrix/server.py b/matrix/server.py index cfbdc94..8fde82b 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -599,7 +599,6 @@ class MatrixServer(object): limit=10) room_buffer.backlog_pending = True - W.bar_item_update("buffer_modes") self.backlog_queue[uuid] = room_id self.send_or_queue(request) From 096372030dd38f0868b7c55846fc920da5210c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 12 Sep 2018 20:13:32 +0200 Subject: [PATCH 129/269] Makefile: Add typecheck target. --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1278ac4..4bfafc6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: install install-lib phony test +.PHONY: install install-lib phony test typecheck WEECHAT_HOME ?= $(HOME)/.weechat PREFIX ?= $(WEECHAT_HOME) @@ -20,3 +20,6 @@ $(DESTDIR)$(PREFIX)/python/matrix/%.py: matrix/%.py phony test: python3 -m pytest python2 -m pytest + +typecheck: + mypy -p matrix --ignore-missing-imports --warn-redundant-casts From bb510e453aa8ace5d6a454662d6d31b22127bc5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 12 Sep 2018 20:13:58 +0200 Subject: [PATCH 130/269] main: Convert some certinfo fields to native strings. --- main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 071fe23..eba2f22 100644 --- a/main.py +++ b/main.py @@ -92,7 +92,8 @@ def print_certificate_info(buff, sock, cert): key_info = ("key info: {key_type} key {bits} bits, signed using " "{algo}").format( - key_type=key_type, bits=key_size, algo=signature_algorithm) + key_type=key_type, bits=key_size, + algo=n(signature_algorithm)) validity_info = (" Begins on: {before}\n" " Expires on: {after}").format( @@ -110,8 +111,8 @@ def print_certificate_info(buff, sock, cert): issuer = "issuer: {issuer}".format(issuer=issuer) fingerprints = (" SHA1: {}\n" - " SHA256: {}").format(sha1_fingerprint, - sha256_fingerprint) + " SHA256: {}").format(n(sha1_fingerprint), + n(sha256_fingerprint)) wrapper = textwrap.TextWrapper( initial_indent=" - ", subsequent_indent=" ") From c9ce402c2c66b331c2b3c951d196fe7a2136f56f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 12 Sep 2018 20:15:07 +0200 Subject: [PATCH 131/269] buffer: Set the buffer type to private if there are only 2 users. --- matrix/buffer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 379a5dc..3c55f78 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -314,7 +314,7 @@ class WeechatChannelBuffer(object): self.topic_author = "" self.topic_date = None - W.buffer_set(self._ptr, "localvar_set_type", "channel") + W.buffer_set(self._ptr, "localvar_set_type", "private") W.buffer_set(self._ptr, "type", "formatted") W.buffer_set(self._ptr, "localvar_set_channel", name) @@ -638,6 +638,9 @@ class WeechatChannelBuffer(object): self._add_user_to_nicklist(user) self.users[user.nick] = user + if len(self.users) > 2: + W.buffer_set(self._ptr, "localvar_set_type", "channel") + if message: tags = self._message_tags(user, "join") msg = self._membership_message(user, "join") @@ -667,6 +670,9 @@ class WeechatChannelBuffer(object): user = self._get_user(nick) self._remove_user_from_nicklist(user) + if len(self.users) <= 2: + W.buffer_set(self._ptr, "localvar_set_type", "private") + if message: tags = self._message_tags(user, leave_type) From 933764bf8e475e9d3fe42064762ce8acc5e75c62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 12 Sep 2018 20:16:17 +0200 Subject: [PATCH 132/269] buffer: Fix redaction type config change. --- matrix/buffer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 3c55f78..8c6db37 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -916,12 +916,12 @@ class RoomBuffer(object): new_message = "" - if G.CONFIG.look.redaction_type == RedactType.STRIKETHROUGH: + if G.CONFIG.look.redactions == RedactType.STRIKETHROUGH: plaintext_msg = W.string_remove_color(message, "") new_message = string_strikethrough(plaintext_msg) - elif G.CONFIG.look.redaction_type == RedactType.NOTICE: + elif G.CONFIG.look.redactions == RedactType.NOTICE: new_message = message - elif G.CONFIG.look.redaction_type == RedactType.DELETE: + elif G.CONFIG.look.redactions == RedactType.DELETE: pass message = " ".join(s for s in [new_message, redaction_msg] if s) From e9e614571d6376f1bc82e941edeb0cf11068c711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 12 Sep 2018 20:16:49 +0200 Subject: [PATCH 133/269] server: nio API change for error responses. The request type is now an enum. --- matrix/server.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 8fde82b..cc3f94d 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -35,6 +35,7 @@ from nio import ( TransportResponse, TransportType, RoomMessagesResponse, + RequestType, ) from . import globals as G @@ -753,10 +754,7 @@ class MatrixServer(object): ) # TODO better error handling. - if ( - response.request_info.type == "sync" - or response.request_info.type == "login" - ): + if response.request_info.type in (RequestType.sync, RequestType.login): self.disconnect() def handle_response(self, response): From e72014859d33e5c4166eb2dd37ade451f8fe98bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 13 Sep 2018 00:04:29 +0200 Subject: [PATCH 134/269] buffer: Change the notify tag for private messages. --- matrix/buffer.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 8c6db37..003e05a 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -164,6 +164,11 @@ class RoomUser(WeechatUser): class WeechatChannelBuffer(object): tags = { "message": [SCRIPT_NAME + "_message", "notify_message", "log1"], + "message_private": [ + SCRIPT_NAME + "_message", + "notify_private", + "log1" + ], "self_message": [ SCRIPT_NAME + "_message", "notify_none", @@ -177,6 +182,12 @@ class WeechatChannelBuffer(object): "notify_message", "log1", ], + "action_private": [ + SCRIPT_NAME + "_message", + SCRIPT_NAME + "_action", + "notify_private", + "log1", + ], "notice": [SCRIPT_NAME + "_notice", "notify_message", "log1"], "old_message": [ SCRIPT_NAME + "_message", @@ -482,7 +493,8 @@ class WeechatChannelBuffer(object): def message(self, nick, message, date, extra_tags=None): # type: (str, str, int, List[str]) -> None user = self._get_user(nick) - tags = self._message_tags(user, "message") + (extra_tags or []) + tags_type = "message_private" if self.type == "private" else "message" + tags = self._message_tags(user, tags_type) + (extra_tags or []) self._print_message(user, message, date, tags) user.update_speaking_time(date) @@ -551,7 +563,8 @@ class WeechatChannelBuffer(object): def action(self, nick, message, date, extra_tags=None): # type: (str, str, int, Optional[List[str]]) -> None user = self._get_user(nick) - tags = self._message_tags(user, "action") + (extra_tags or []) + tags_type = "action_private" if self.type == "private" else "action" + tags = self._message_tags(user, tags_type) + (extra_tags or []) self._print_action(user, message, date, tags) user.update_speaking_time(date) @@ -747,6 +760,10 @@ class WeechatChannelBuffer(object): def short_name(self): return W.buffer_get_string(self._ptr, "short_name") + @property + def type(self): + return W.buffer_get_string(self._ptr, "localvar_type") + @short_name.setter def short_name(self, name): W.buffer_set(self._ptr, "short_name", name) From 70d577b89e7d474a3e5004b4be58a030fb564251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 15 Sep 2018 23:26:36 +0200 Subject: [PATCH 135/269] buffer: Fix off by one error for freenode display names. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 003e05a..f100f84 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -829,7 +829,7 @@ class RoomBuffer(object): if user.display_name: short_name = user.display_name elif user.user_id.startswith("@freenode_"): - short_name = shorten_sender(user.user_id[10:]) + short_name = shorten_sender(user.user_id[9:]) # TODO make this configurable if not short_name or short_name in self.displayed_nicks.values(): From 76e0aba3aa0fc0eb61c41df0e3e7826a11c63838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 15 Sep 2018 23:27:43 +0200 Subject: [PATCH 136/269] config: Add encryption debug category. --- matrix/config.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/matrix/config.py b/matrix/config.py index 8daa890..0ce38e0 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -42,13 +42,6 @@ class ServerBufferType(Enum): INDEPENDENT = 2 -@unique -class DebugType(Enum): - MESSAGING = 0 - NETWORK = 1 - TIMING = 2 - - class Option( namedtuple( "Option", @@ -109,6 +102,8 @@ def change_log_level(category, level): nio.events.logger.level = level elif category == "responses": nio.responses.logger.level = level + elif category == "encryption": + nio.encryption.logger.level = level @utf8_decode @@ -175,6 +170,8 @@ def logbook_category(value): return "events" if value == 4: return "responses" + if value == 5: + return "encryption" return "all" @@ -369,7 +366,7 @@ class MatrixConfig(WeechatConfig): Option( "debug_category", "integer", - "all|http|client|events|responses", + "all|http|client|events|responses|encryption", 0, 0, "all", From 97d3a59e33c459c73df17a63ce975f273bb37ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 19 Sep 2018 11:40:08 +0200 Subject: [PATCH 137/269] server: Enable Olm support. --- matrix/buffer.py | 33 +++++++++++++++++++++------------ matrix/server.py | 22 +++++++++++++++++----- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index f100f84..980abee 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -37,6 +37,7 @@ from nio import ( RoomMessageUnknown, RoomNameEvent, RoomTopicEvent, + MegolmEvent ) from . import globals as G @@ -1098,17 +1099,25 @@ class RoomBuffer(object): elif isinstance(event, PowerLevelsEvent): self._handle_power_level(event) - # elif isinstance(event, UndecryptedEvent): - # nick = self.find_nick(event.sender) - # date = server_ts_to_weechat(event.server_timestamp) - # data = ("Error decrypting event session " - # "id: {}".format(event.session_id)) - # self.weechat_buffer.message( - # nick, - # data, - # date, - # self.get_event_tags(event) - # ) + elif isinstance(event, MegolmEvent): + nick = self.find_nick(event.sender) + date = server_ts_to_weechat(event.server_timestamp) + "{del_color}<{log_color}Message redacted by: " + "{censor}{log_color}{reason}{del_color}>" + "{ncolor}" + data = ("{del_color}<{log_color}Unable to decrypt: " + "The sender's device has not sent us " + "the keys for this message{del_color}>{ncolor}").format( + del_color=W.color("chat_delimiters"), + log_color=W.color("logger.color.backlog_line"), + ncolor=W.color("reset")) + session_id_tag = SCRIPT_NAME + "_sessionid_" + event.session_id + self.weechat_buffer.message( + nick, + data, + date, + self.get_event_tags(event) + [session_id_tag] + ) else: W.prnt( @@ -1249,7 +1258,7 @@ class RoomBuffer(object): break if leave_index: - timeline_events = info.timeline.events[leave_index + 1 :] + timeline_events = info.timeline.events[leave_index + 1:] # Handle our leave as a state event since we're not in the # nicklist anymore but we're already printed out our leave self.handle_state_event(info.timeline.events[leave_index]) diff --git a/matrix/server.py b/matrix/server.py index cc3f94d..b70914a 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -259,7 +259,12 @@ class MatrixServer(object): def _change_client(self): host = ":".join([self.config.address, str(self.config.port)]) - self.client = HttpClient(host, self.config.username, self.device_id) + self.client = HttpClient( + host, + self.config.username, + self.device_id, + self.get_session_path() + ) def update_option(self, option, option_name): if option_name == "address": @@ -633,6 +638,10 @@ class MatrixServer(object): self.own_message_queue[uuid] = own_message self.send_or_queue(request) + def keys_upload(self): + _, request = self.client.keys_upload() + self.send_or_queue(request) + def _print_message_error(self, message): server_buffer_prnt( self, @@ -683,10 +692,8 @@ class MatrixServer(object): W.prnt(self.server_buffer, message) - # if not self.olm: - # self.create_olm() - # self.store_olm() - # self.upload_keys(device_keys=True, one_time_keys=False) + if not self.client.olm_account_shared: + self.keys_upload() sync_filter = { "room": { @@ -743,7 +750,12 @@ class MatrixServer(object): return self._handle_room_info(response) + self.next_batch = response.next_batch + + if self.client.should_upload_keys: + self.keys_upload() + self.schedule_sync() def handle_transport_response(self, response): From ac5d1b823e5b9005a65dd69ec22067b9fa72c1c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 19 Sep 2018 12:47:53 +0200 Subject: [PATCH 138/269] buffer: Lazily add users to the nicklist. --- main.py | 2 +- matrix/buffer.py | 107 +++++++++++++++++++++++++++++++---------------- matrix/server.py | 63 +++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 37 deletions(-) diff --git a/main.py b/main.py index eba2f22..6bcba0f 100644 --- a/main.py +++ b/main.py @@ -61,7 +61,7 @@ from matrix.server import (MatrixServer, create_default_server, matrix_config_server_change_cb, matrix_config_server_read_cb, matrix_config_server_write_cb, matrix_timer_cb, - send_cb) + send_cb, matrix_load_users_cb) from matrix.utf import utf8_decode from matrix.utils import server_buffer_prnt, server_buffer_set_title diff --git a/matrix/buffer.py b/matrix/buffer.py index 980abee..95a97a2 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -30,6 +30,7 @@ from nio import ( RoomAliasEvent, RoomEncryptionEvent, RoomMemberEvent, + RoomMessage, RoomMessageEmote, RoomMessageMedia, RoomMessageNotice, @@ -785,6 +786,7 @@ class RoomBuffer(object): self.prev_batch = prev_batch self.joined = True self.leave_event_id = None # type: Optional[str] + self.unhandled_users = [] # type: List[str] buffer_name = "{}.{}".format(server_name, room.room_id) @@ -813,51 +815,67 @@ class RoomBuffer(object): return user_id + def add_user(self, user_id, date, is_state): + try: + user = self.room.users[user_id] + except KeyError: + # No user found, he must have left already in an event that is + # yet to come, so do nothing + # W.prnt("", "NOT ADDING USER {}".format(user_id)) + return + + short_name = shorten_sender(user.user_id) + + # TODO handle this special case for discord bridge users and + # freenode bridge users better + if user.user_id.startswith("@_discord_"): + if user.display_name: + short_name = user.display_name + elif user.user_id.startswith("@freenode_"): + short_name = shorten_sender(user.user_id[9:]) + + # TODO make this configurable + if not short_name or short_name in self.displayed_nicks.values(): + # Use the full user id, but don't include the @ + nick = user_id[1:] + else: + nick = short_name + + buffer_user = RoomUser(nick, user_id, user.power_level, date) + self.displayed_nicks[user_id] = nick + + if self.room.own_user_id == user_id: + buffer_user.color = "weechat.color.chat_nick_self" + user.nick_color = "weechat.color.chat_nick_self" + + self.weechat_buffer.join(buffer_user, date, not is_state) + def handle_membership_events(self, event, is_state): - def join(event, date, is_state): - try: - user = self.room.users[event.state_key] - except KeyError: - # No user found, he must have left already in an event that is - # yet to come, so do nothing - return - - short_name = shorten_sender(user.user_id) - - # TODO handle this special case for discord bridge users and - # freenode bridge users better - if user.user_id.startswith("@_discord_"): - if user.display_name: - short_name = user.display_name - elif user.user_id.startswith("@freenode_"): - short_name = shorten_sender(user.user_id[9:]) - - # TODO make this configurable - if not short_name or short_name in self.displayed_nicks.values(): - # Use the full user id, but don't include the @ - nick = event.sender[1:] - else: - nick = short_name - - buffer_user = RoomUser(nick, event.sender, user.power_level, date) - self.displayed_nicks[event.sender] = nick - - if self.room.own_user_id == event.sender: - buffer_user.color = "weechat.color.chat_nick_self" - user.nick_color = "weechat.color.chat_nick_self" - - self.weechat_buffer.join(buffer_user, date, not is_state) - date = server_ts_to_weechat(event.server_timestamp) if event.content["membership"] == "join": if event.state_key not in self.displayed_nicks: - join(event, date, is_state) + # Adding users to the nicklist is a O(1) + search time + # operation (the nicks are added to a linked list sorted). + # The search time is O(N * min(a,b)) where N is the number + # of nicks already added and a/b are the length of + # the strings that are compared at every itteration. + # Because the search time get's increasingly longer we're + # going to add nicks later in a timer hook. + if ((len(self.room.users) - len(self.displayed_nicks)) > 500 + and is_state): + self.unhandled_users.append(event.state_key) + else: + self.add_user(event.state_key, date, is_state) else: # TODO print out profile changes return elif event.content["membership"] == "leave": + if event.state_key in self.unhandled_users: + self.unhandled_users.remove(event.state_key) + return + nick = self.find_nick(event.state_key) if event.sender == event.state_key: self.weechat_buffer.part(nick, date, not is_state) @@ -1021,6 +1039,19 @@ class RoomBuffer(object): self.weechat_buffer.error(message) def handle_timeline_event(self, event): + # TODO this should be done for every messagetype that gets printed in + # the buffer + if isinstance(event, (RoomMessage, MegolmEvent)): + if (event.sender not in self.displayed_nicks and + event.sender in self.room.users): + + try: + self.unhandled_users.remove(event.sender) + except ValueError: + pass + + self.add_user(event.sender, 0, True) + if isinstance(event, RoomMemberEvent): self.handle_membership_events(event, False) @@ -1274,6 +1305,12 @@ class RoomBuffer(object): for event in timeline_events: self.handle_timeline_event(event) + # We didn't handle all joined users, the room display name might still + # be outdated because of that, update it now. + if self.unhandled_users: + room_name = self.room.display_name() + self.weechat_buffer.short_name = room_name + def handle_left_room(self, info): self.joined = False diff --git a/matrix/server.py b/matrix/server.py index b70914a..7db87b8 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -220,7 +220,10 @@ class MatrixServer(object): self.device_check_timestamp = None # type: Optional[int] self.own_message_queue = dict() # type: Dict[str, OwnMessage] - self.backlog_queue = dict() # type: Dict[str, str] + self.backlog_queue = dict() # type: Dict[str, str] + + self.unhandled_users = dict() # type: Dict[str, List[str]] + self.lazy_load_hook = None # type: str self.config = ServerConfig(self.name, config_ptr) self._create_session_dir() @@ -736,6 +739,8 @@ class MatrixServer(object): room_buffer = self.find_room_from_id(room_id) room_buffer.handle_left_room(info) + should_lazy_hook = False + for room_id, info in response.rooms.join.items(): if room_id not in self.buffers: self.create_room_buffer(room_id, info.timeline.prev_batch) @@ -743,6 +748,42 @@ class MatrixServer(object): room_buffer = self.find_room_from_id(room_id) room_buffer.handle_joined_room(info) + if room_buffer.unhandled_users: + should_lazy_hook = True + + if should_lazy_hook: + hook = W.hook_timer(1 * 100, 0, 0, "matrix_load_users_cb", + self.name) + self.lazy_load_hook = hook + + def add_unhandled_users(self, rooms, n): + # type: (List[RoomBuffer], int) -> bool + total_users = 0 + + while total_users <= n: + try: + room_buffer = rooms.pop() + except IndexError: + return False + + handled_users = 0 + + users = room_buffer.unhandled_users + + for user_id in users: + room_buffer.add_user(user_id, 0, True) + handled_users += 1 + total_users += 1 + + if total_users >= n: + room_buffer.unhandled_users = users[handled_users:] + rooms.append(room_buffer) + return True + + room_buffer.unhandled_users = [] + + return False + def _handle_sync(self, response): # we got the same batch again, nothing to do if self.next_batch == response.next_batch: @@ -901,6 +942,26 @@ def matrix_config_server_change_cb(server_name, option): return 1 +@utf8_decode +def matrix_load_users_cb(server_name, remaining_calls): + server = SERVERS[server_name] + start = time.time() + + rooms = [x for x in server.room_buffers.values() if x.unhandled_users] + + while server.add_unhandled_users(rooms, 100): + current = time.time() + + if current - start >= 0.1: + return W.WEECHAT_RC_OK + + # We are done adding users, we can unhook now. + W.unhook(server.lazy_load_hook) + server.lazy_load_hook = None + + return W.WEECHAT_RC_OK + + @utf8_decode def matrix_timer_cb(server_name, remaining_calls): server = SERVERS[server_name] From 0872948e9e9cc071f9beae6397c649b68e2de0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 20 Sep 2018 20:38:05 +0200 Subject: [PATCH 139/269] buffer: Reorder the short_name property to make mypy happy. --- matrix/buffer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 95a97a2..21c2c4b 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -758,14 +758,14 @@ class WeechatChannelBuffer(object): tags.append(SCRIPT_NAME + "_action") self._print_action(user, message, date, tags) - @property - def short_name(self): - return W.buffer_get_string(self._ptr, "short_name") - @property def type(self): return W.buffer_get_string(self._ptr, "localvar_type") + @property + def short_name(self): + return W.buffer_get_string(self._ptr, "short_name") + @short_name.setter def short_name(self, name): W.buffer_set(self._ptr, "short_name", name) From 8fbaf763b11498f9b00755ce81bca6d1e2b0b27b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 20 Sep 2018 20:38:49 +0200 Subject: [PATCH 140/269] server: Add key query functionality. --- matrix/server.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 7db87b8..5b58e47 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -22,7 +22,7 @@ import socket import ssl import time from collections import defaultdict, deque -from typing import Any, Deque, Dict, Optional +from typing import Any, Deque, Dict, Optional, List from nio import ( HttpClient, @@ -193,6 +193,7 @@ class MatrixServer(object): self.connected = False # type: bool self.connecting = False # type: bool + self.keys_queried = False # type: bool self.reconnect_delay = 0 # type: int self.reconnect_time = None # type: Optional[float] self.sync_time = None # type: Optional[float] @@ -223,7 +224,7 @@ class MatrixServer(object): self.backlog_queue = dict() # type: Dict[str, str] self.unhandled_users = dict() # type: Dict[str, List[str]] - self.lazy_load_hook = None # type: str + self.lazy_load_hook = None # type: Optional[str] self.config = ServerConfig(self.name, config_ptr) self._create_session_dir() @@ -645,6 +646,11 @@ class MatrixServer(object): _, request = self.client.keys_upload() self.send_or_queue(request) + def keys_query(self): + _, request = self.client.keys_query() + self.keys_queried = True + self.send_or_queue(request) + def _print_message_error(self, message): server_buffer_prnt( self, @@ -782,7 +788,7 @@ class MatrixServer(object): room_buffer.unhandled_users = [] - return False + return False def _handle_sync(self, response): # we got the same batch again, nothing to do @@ -797,6 +803,9 @@ class MatrixServer(object): if self.client.should_upload_keys: self.keys_upload() + if self.client.should_query_keys and not self.keys_queried: + self.keys_query() + self.schedule_sync() def handle_transport_response(self, response): From 8a73f2c119616776c6455d56476ad9a716253ddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 21 Sep 2018 11:57:51 +0200 Subject: [PATCH 141/269] server: Remove unneeded key query code in the timer callback. --- matrix/server.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 5b58e47..4259805 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1006,16 +1006,6 @@ def matrix_timer_cb(server_name, remaining_calls): if not server.next_batch: return W.WEECHAT_RC_OK - # check for new devices by users in encrypted rooms periodically - # if (not server.device_check_timestamp or - # current_time - server.device_check_timestamp > 600): - - # W.prnt(server.server_buffer, - # "{prefix}matrix: Querying user devices.".format( - # prefix=W.prefix("networ"))) - - # server.device_check_timestamp = current_time - return W.WEECHAT_RC_OK From 6cf275e87138cff5ed37d08f3f008ea9a9b5b85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 24 Sep 2018 15:27:39 +0200 Subject: [PATCH 142/269] buffer: Print out a warning prefix if the event was encrypted but not verified. --- matrix/bar_items.py | 4 +++ matrix/buffer.py | 83 ++++++++++++++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 56d1965..3087a6f 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -102,6 +102,10 @@ def matrix_bar_item_buffer_modes(data, item, window, buffer, extra_info): if room.encrypted: modes.append("🔐") + if (server.client + and server.client.room_contains_unverified(room.room_id)): + modes.append("⚠️ ") + if room_buffer.backlog_pending: modes.append("⏳") diff --git a/matrix/buffer.py b/matrix/buffer.py index 21c2c4b..aeb070b 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -38,7 +38,8 @@ from nio import ( RoomMessageUnknown, RoomNameEvent, RoomTopicEvent, - MegolmEvent + MegolmEvent, + Event ) from . import globals as G @@ -471,11 +472,12 @@ class WeechatChannelBuffer(object): # A message from a non joined user return WeechatUser(nick) - def _print_message(self, user, message, date, tags): + def _print_message(self, user, message, date, tags, extra_prefix=""): prefix_string = ( - "" + extra_prefix if not user.prefix - else "{}{}{}".format( + else "{}{}{}{}".format( + extra_prefix, W.color(self._get_prefix_color(user.prefix)), user.prefix, W.color("reset"), @@ -492,18 +494,18 @@ class WeechatChannelBuffer(object): self.print_date_tags(data, date, tags) - def message(self, nick, message, date, extra_tags=None): - # type: (str, str, int, List[str]) -> None + def message(self, nick, message, date, extra_tags=None, extra_prefix=""): + # type: (str, str, int, List[str], str) -> None user = self._get_user(nick) tags_type = "message_private" if self.type == "private" else "message" tags = self._message_tags(user, tags_type) + (extra_tags or []) - self._print_message(user, message, date, tags) + self._print_message(user, message, date, tags, extra_prefix) user.update_speaking_time(date) self.unmask_smart_filtered_nick(nick) - def notice(self, nick, message, date, extra_tags=None): - # type: (str, str, int, Optional[List[str]]) -> None + def notice(self, nick, message, date, extra_tags=None, extra_prefix=""): + # type: (str, str, int, Optional[List[str]], str) -> None user = self._get_user(nick) user_prefix = ( "" @@ -520,10 +522,11 @@ class WeechatChannelBuffer(object): ) data = ( - "{prefix}{color}Notice" + "{extra_prefix}{prefix}{color}Notice" "{del_color}({ncolor}{user}{del_color}){ncolor}" ": {message}" ).format( + extra_prefix=extra_prefix, prefix=W.prefix("network"), color=W.color("irc.color.notice"), del_color=W.color("chat_delimiters"), @@ -538,7 +541,7 @@ class WeechatChannelBuffer(object): user.update_speaking_time(date) self.unmask_smart_filtered_nick(nick) - def _print_action(self, user, message, date, tags): + def _print_action(self, user, message, date, tags, extra_prefix): nick_prefix = ( "" if not user.prefix @@ -550,8 +553,9 @@ class WeechatChannelBuffer(object): ) data = ( - "{prefix}{nick_prefix}{nick_color}{author}" "{ncolor} {msg}" - ).format( + "{extra_prefix}{prefix}{nick_prefix}{nick_color}{author}" + "{ncolor} {msg}").format( + extra_prefix=extra_prefix, prefix=W.prefix("action"), nick_prefix=nick_prefix, nick_color=W.color(user.color), @@ -562,12 +566,12 @@ class WeechatChannelBuffer(object): self.print_date_tags(data, date, tags) - def action(self, nick, message, date, extra_tags=None): - # type: (str, str, int, Optional[List[str]]) -> None + def action(self, nick, message, date, extra_tags=None, extra_prefix=""): + # type: (str, str, int, Optional[List[str]], str) -> None user = self._get_user(nick) tags_type = "action_private" if self.type == "private" else "action" tags = self._message_tags(user, tags_type) + (extra_tags or []) - self._print_action(user, message, date, tags) + self._print_action(user, message, date, tags, extra_prefix) user.update_speaking_time(date) self.unmask_smart_filtered_nick(nick) @@ -807,6 +811,10 @@ class RoomBuffer(object): self._backlog_pending = value W.bar_item_update("buffer_modes") + @property + def warning_prefix(self): + return "⚠️ " + def find_nick(self, user_id): # type: (str) -> str """Find a suitable nick from a user_id""" @@ -1006,7 +1014,12 @@ class RoomBuffer(object): @staticmethod def get_event_tags(event): - return ["matrix_id_{}".format(event.event_id)] + # type: (Event) -> List[str] + tags = [SCRIPT_NAME + "_id_{}".format(event.event_id)] + if event.sender_key: + tags.append(SCRIPT_NAME + "_senderkey_{}".format(event.sender_key)) + + return tags def _handle_power_level(self, _): for user_id in self.room.power_levels.users: @@ -1067,8 +1080,13 @@ class RoomBuffer(object): elif isinstance(event, RoomMessageEmote): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.server_timestamp) + + extra_prefix = (self.warning_prefix if event.decrypted + and not event.verified else "") + self.weechat_buffer.action( - nick, event.body, date, self.get_event_tags(event) + nick, event.body, date, self.get_event_tags(event), + extra_prefix ) elif isinstance(event, RoomMessageText): @@ -1080,16 +1098,23 @@ class RoomBuffer(object): data = formatted.to_weechat() if formatted else event.body + extra_prefix = (self.warning_prefix if event.decrypted + and not event.verified else "") + date = server_ts_to_weechat(event.server_timestamp) self.weechat_buffer.message( - nick, data, date, self.get_event_tags(event) + nick, data, date, self.get_event_tags(event), extra_prefix ) elif isinstance(event, RoomMessageNotice): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.server_timestamp) + extra_prefix = (self.warning_prefix if event.decrypted + and not event.verified else "") + self.weechat_buffer.notice( - nick, event.body, date, self.get_event_tags(event) + nick, event.body, date, self.get_event_tags(event), + extra_prefix ) elif isinstance(event, RoomMessageMedia): @@ -1101,16 +1126,22 @@ class RoomBuffer(object): description = "/{}".format(event.body) if event.body else "" data = "{url}{desc}".format(url=url, desc=description) + extra_prefix = (self.warning_prefix if event.decrypted + and not event.verified else "") + self.weechat_buffer.message( - nick, data, date, self.get_event_tags(event) + nick, data, date, self.get_event_tags(event), extra_prefix ) elif isinstance(event, RoomMessageUnknown): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.server_timestamp) data = ("Unknown message of type {t}").format(t=event.type) + extra_prefix = (self.warning_prefix if event.decrypted + and not event.verified else "") + self.weechat_buffer.message( - nick, data, date, self.get_event_tags(event) + nick, data, date, self.get_event_tags(event), extra_prefix ) elif isinstance(event, RedactionEvent): @@ -1133,9 +1164,7 @@ class RoomBuffer(object): elif isinstance(event, MegolmEvent): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.server_timestamp) - "{del_color}<{log_color}Message redacted by: " - "{censor}{log_color}{reason}{del_color}>" - "{ncolor}" + data = ("{del_color}<{log_color}Unable to decrypt: " "The sender's device has not sent us " "the keys for this message{del_color}>{ncolor}").format( @@ -1159,7 +1188,7 @@ class RoomBuffer(object): # type: (OwnMessage) -> None nick = self.find_nick(self.room.own_user_id) data = message.formatted_message.to_weechat() - tags = self.get_event_tags(message) + tags = [SCRIPT_NAME + "_id_{}".format(message.event_id)] date = message.age self.weechat_buffer.self_message(nick, data, date, tags) @@ -1168,7 +1197,7 @@ class RoomBuffer(object): # type: (OwnMessage) -> None nick = self.find_nick(self.room.own_user_id) date = message.age - tags = self.get_event_tags(message) + tags = [SCRIPT_NAME + "_id_{}".format(message.event_id)] self.weechat_buffer.self_action( nick, message.formatted_message.to_weechat(), date, tags From 595db64a55f4a287a16115a7a501aad66522ea47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 25 Sep 2018 20:04:03 +0200 Subject: [PATCH 143/269] server: Add support for encrypted messages. --- matrix/server.py | 58 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 4259805..1f4ce11 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -22,7 +22,7 @@ import socket import ssl import time from collections import defaultdict, deque -from typing import Any, Deque, Dict, Optional, List +from typing import Any, Deque, Dict, Optional, List, NamedTuple from nio import ( HttpClient, @@ -31,11 +31,13 @@ from nio import ( Response, Rooms, RoomSendResponse, - SyncRepsponse, + SyncResponse, TransportResponse, TransportType, RoomMessagesResponse, RequestType, + EncryptionError, + OlmTrustError, ) from . import globals as G @@ -45,8 +47,7 @@ from .globals import SCRIPT_NAME, SERVERS, W from .utf import utf8_decode from .utils import create_server_buffer, key_from_value, server_buffer_prnt -if False: - from .colors import Formatted +from .colors import Formatted try: @@ -55,6 +56,15 @@ except NameError: FileNotFoundError = IOError +EncrytpionQueueItem = NamedTuple( + "EncrytpionQueueItem", + [ + ("message_type", str), + ("formatted_message", Formatted), + ], +) + + class ServerConfig(ConfigSection): def __init__(self, server_name, config_ptr): # type: (str, str) -> None @@ -221,6 +231,7 @@ class MatrixServer(object): self.device_check_timestamp = None # type: Optional[int] self.own_message_queue = dict() # type: Dict[str, OwnMessage] + self.encryption_queue = defaultdict(deque) self.backlog_queue = dict() # type: Dict[str, str] self.unhandled_users = dict() # type: Dict[str, List[str]] @@ -614,29 +625,38 @@ class MatrixServer(object): def room_send_message(self, room_buffer, formatted, msgtype="m.text"): # type: (RoomBuffer, Formatted, str) -> None - if room_buffer.room.encrypted: - return + room = room_buffer.room if not self.client: return - if msgtype == "m.emote": - message_class = OwnAction - else: - message_class = OwnMessage - - own_message = message_class( - self.user_id, 0, "", room_buffer.room.room_id, formatted - ) - body = {"msgtype": msgtype, "body": formatted.to_plain()} if formatted.is_formatted(): body["format"] = "org.matrix.custom.html" body["formatted_body"] = formatted.to_html() - uuid, request = self.client.room_send( - room_buffer.room.room_id, "m.room.message", body + try: + uuid, request = self.client.room_send( + room.room_id, "m.room.message", body + ) + except EncryptionError: + try: + uuid, request = self.client.share_group_session(room.room_id) + message = EncrytpionQueueItem(msgtype, formatted) + self.encryption_queue[room.room_id].append(message) + except OlmTrustError as e: + m = ("Untrusted devices found in room: {}".format(e)) + self.error(m) + return + + if msgtype == "m.emote": + message_class = OwnAction + else: + message_class = OwnMessage + + own_message = message_class( + self.user_id, 0, "", room.room_id, formatted ) self.own_message_queue[uuid] = own_message @@ -826,7 +846,7 @@ class MatrixServer(object): # If the response was a sync response and contained a timeout the # timeout is expected and should be removed from the lag. # TODO the timeout isn't a constant - if isinstance(response, SyncRepsponse): + if isinstance(response, SyncResponse): self.lag = max(0, self.lag - (30000)) self.lag_done = True @@ -838,7 +858,7 @@ class MatrixServer(object): elif isinstance(response, LoginResponse): self._handle_login(response) - elif isinstance(response, SyncRepsponse): + elif isinstance(response, SyncResponse): self._handle_sync(response) elif isinstance(response, RoomSendResponse): From 4860ffee11debd213ac26cd317f6b92e280ca358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 1 Oct 2018 17:11:09 +0200 Subject: [PATCH 144/269] server: Handle group session sharing responses. --- matrix/buffer.py | 10 ++++++++-- matrix/server.py | 36 ++++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index aeb070b..cdd5623 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -39,7 +39,8 @@ from nio import ( RoomNameEvent, RoomTopicEvent, MegolmEvent, - Event + Event, + OlmTrustError ) from . import globals as G @@ -89,7 +90,12 @@ def room_buffer_input_cb(server_name, buffer, input_data): formatted_data = Formatted.from_input_line(data) - server.room_send_message(room_buffer, formatted_data, "m.text") + try: + server.room_send_message(room_buffer, formatted_data, "m.text") + except OlmTrustError as e: + m = ("Untrusted devices found in room: {}".format(e)) + server.error(m) + pass return W.WEECHAT_RC_OK diff --git a/matrix/server.py b/matrix/server.py index 1f4ce11..12f9b15 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -32,6 +32,7 @@ from nio import ( Rooms, RoomSendResponse, SyncResponse, + ShareGroupSessionResponse, TransportResponse, TransportType, RoomMessagesResponse, @@ -624,11 +625,11 @@ class MatrixServer(object): self.send_or_queue(request) def room_send_message(self, room_buffer, formatted, msgtype="m.text"): - # type: (RoomBuffer, Formatted, str) -> None + # type: (RoomBuffer, Formatted, str) -> bool room = room_buffer.room if not self.client: - return + return False body = {"msgtype": msgtype, "body": formatted.to_plain()} @@ -641,14 +642,11 @@ class MatrixServer(object): room.room_id, "m.room.message", body ) except EncryptionError: - try: - uuid, request = self.client.share_group_session(room.room_id) - message = EncrytpionQueueItem(msgtype, formatted) - self.encryption_queue[room.room_id].append(message) - except OlmTrustError as e: - m = ("Untrusted devices found in room: {}".format(e)) - self.error(m) - return + _, request = self.client.share_group_session(room.room_id) + message = EncrytpionQueueItem(msgtype, formatted) + self.encryption_queue[room.room_id].append(message) + self.send_or_queue(request) + return False if msgtype == "m.emote": message_class = OwnAction @@ -661,6 +659,7 @@ class MatrixServer(object): self.own_message_queue[uuid] = own_message self.send_or_queue(request) + return True def keys_upload(self): _, request = self.client.keys_upload() @@ -863,9 +862,26 @@ class MatrixServer(object): elif isinstance(response, RoomSendResponse): self.handle_own_messages(response) + elif isinstance(response, RoomMessagesResponse): self.handle_backlog_response(response) + elif isinstance(response, ShareGroupSessionResponse): + room_id = response.room_id + room_buffer = self.room_buffers[room_id] + + while self.encryption_queue[room_id]: + message = self.encryption_queue[room_id].popleft() + try: + if not self.room_send_message(room_buffer, + message.formatted_message, + message.message_type): + self.encryption_queue.pop() + self.encryption_queue[room_id].appendleft(message) + break + except OlmTrustError: + break + def create_room_buffer(self, room_id, prev_batch): room = self.client.rooms[room_id] buf = RoomBuffer(room, self.name, prev_batch) From d8e7d844459e614195218ed39b199d7d35e78735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 1 Oct 2018 17:12:36 +0200 Subject: [PATCH 145/269] server: Add one-time key claiming functionality. --- matrix/server.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 12f9b15..cbf743a 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -22,7 +22,7 @@ import socket import ssl import time from collections import defaultdict, deque -from typing import Any, Deque, Dict, Optional, List, NamedTuple +from typing import Any, Deque, Dict, Optional, List, NamedTuple, DefaultDict from nio import ( HttpClient, @@ -33,11 +33,13 @@ from nio import ( RoomSendResponse, SyncResponse, ShareGroupSessionResponse, + KeysClaimResponse, TransportResponse, TransportType, RoomMessagesResponse, RequestType, EncryptionError, + GroupEncryptionError, OlmTrustError, ) @@ -232,7 +234,8 @@ class MatrixServer(object): self.device_check_timestamp = None # type: Optional[int] self.own_message_queue = dict() # type: Dict[str, OwnMessage] - self.encryption_queue = defaultdict(deque) + self.encryption_queue = defaultdict(deque) \ + # type: DefaultDict[str, Deque[EncrytpionQueueItem]] self.backlog_queue = dict() # type: Dict[str, str] self.unhandled_users = dict() # type: Dict[str, List[str]] @@ -624,12 +627,16 @@ class MatrixServer(object): self.backlog_queue[uuid] = room_id self.send_or_queue(request) - def room_send_message(self, room_buffer, formatted, msgtype="m.text"): - # type: (RoomBuffer, Formatted, str) -> bool + def room_send_message( + self, + room_buffer, # type: RoomBuffer + formatted, # type: Formatted + msgtype="m.text", # type: str + ): + # type: (...) -> bool room = room_buffer.room - if not self.client: - return False + assert self.client body = {"msgtype": msgtype, "body": formatted.to_plain()} @@ -641,8 +648,12 @@ class MatrixServer(object): uuid, request = self.client.room_send( room.room_id, "m.room.message", body ) - except EncryptionError: - _, request = self.client.share_group_session(room.room_id) + except GroupEncryptionError: + try: + _, request = self.client.share_group_session(room.room_id) + except EncryptionError: + _, request = self.client.keys_claim(room.room_id) + message = EncrytpionQueueItem(msgtype, formatted) self.encryption_queue[room.room_id].append(message) self.send_or_queue(request) @@ -866,6 +877,13 @@ class MatrixServer(object): elif isinstance(response, RoomMessagesResponse): self.handle_backlog_response(response) + elif isinstance(response, KeysClaimResponse): + _, request = self.client.share_group_session( + response.room_id, + ignore_missing_sessions=True + ) + self.send(request) + elif isinstance(response, ShareGroupSessionResponse): room_id = response.room_id room_buffer = self.room_buffers[room_id] From a657316278477c8a6055ac9ad75fbdb96bcb1e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 1 Oct 2018 17:12:58 +0200 Subject: [PATCH 146/269] bar_items: Add a connection state status item. --- matrix/bar_items.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 3087a6f..f34cfdf 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -106,6 +106,9 @@ def matrix_bar_item_buffer_modes(data, item, window, buffer, extra_info): and server.client.room_contains_unverified(room.room_id)): modes.append("⚠️ ") + if not server.connected: + modes.append("❌") + if room_buffer.backlog_pending: modes.append("⏳") From df7069fb21a358984361afbd1e966cae935bc032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 1 Oct 2018 17:13:17 +0200 Subject: [PATCH 147/269] bar_items: Replicate buffer modes as a separate matrix_modes bar item. --- matrix/bar_items.py | 1 + 1 file changed, 1 insertion(+) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index f34cfdf..b154003 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -122,3 +122,4 @@ def init_bar_items(): W.bar_item_new("(extra)buffer_name", "matrix_bar_item_name", "") W.bar_item_new("(extra)lag", "matrix_bar_item_lag", "") W.bar_item_new("(extra)buffer_modes", "matrix_bar_item_buffer_modes", "") + W.bar_item_new("(extra)matrix_modes", "matrix_bar_item_buffer_modes", "") From 821932a32603dbee88e949ac54528ccc4e9c1e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 2 Oct 2018 15:06:49 +0200 Subject: [PATCH 148/269] commands: Remove a double newline from the matrix command. --- matrix/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/commands.py b/matrix/commands.py index 880f163..3c19b65 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -110,7 +110,7 @@ def hook_commands(): " server: list, add, or remove Matrix servers\n" " connect: connect to Matrix servers\n" "disconnect: disconnect from one or all Matrix servers\n" - " reconnect: reconnect to server(s)\n\n" + " reconnect: reconnect to server(s)\n" " help: show detailed command help\n\n" "Use /matrix help [command] to find out more.\n" ), From 15e4e74c7724e6a67941b832f5d18d12b0b84d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 2 Oct 2018 16:39:54 +0200 Subject: [PATCH 149/269] commands: Style fixes. --- matrix/commands.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index 3c19b65..112092c 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -135,9 +135,11 @@ def hook_commands(): ('[:""] []'), # Description ( - "message-number: number of message to redact (starting from 1 for\n" + "message-number: number of message to redact " + "(starting from 1 for\n" " the last message received, counting up)\n" - " message-part: an initial part of the message (ignored, only used\n" + " message-part: an initial part of the message (ignored, only " + "used\n" " as visual feedback when using completion)\n" " reason: the redaction reason\n" ), From 5fd5710225442f961746e45ba1445b498f32128b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 2 Oct 2018 16:40:18 +0200 Subject: [PATCH 150/269] server: Handle exception from group session sharing request. --- matrix/server.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index cbf743a..e64a15c 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -878,10 +878,16 @@ class MatrixServer(object): self.handle_backlog_response(response) elif isinstance(response, KeysClaimResponse): - _, request = self.client.share_group_session( - response.room_id, - ignore_missing_sessions=True - ) + try: + _, request = self.client.share_group_session( + response.room_id, + ignore_missing_sessions=True + ) + except OlmTrustError as e: + m = ("Untrusted devices found in room: {}".format(e)) + self.error(m) + return + self.send(request) elif isinstance(response, ShareGroupSessionResponse): From 583926560aec239d9a34a4f992332c78927a08f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 2 Oct 2018 18:16:09 +0200 Subject: [PATCH 151/269] matrix: Add a server buffer callback. --- main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/main.py b/main.py index 6bcba0f..2715d58 100644 --- a/main.py +++ b/main.py @@ -420,6 +420,13 @@ def debug_buffer_close_cb(data, buffer): return W.WEECHAT_RC_OK +def server_buffer_cb(server_name, buffer, input_data): + message = ("{}{}: this buffer is not a room buffer!").format( + W.prefix("error"), SCRIPT_NAME) + W.prnt(buffer, message) + return W.WEECHAT_RC_OK + + class WeechatHandler(StreamHandler): def __init__(self, level=logbook.NOTSET, format_string=None, filter=None, bubble=False): From 69db90dd4dfe82c1ccefe8440a6672dae9eaf969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 2 Oct 2018 18:16:36 +0200 Subject: [PATCH 152/269] server: Don't make multiple key claim and group share requests for a room. --- matrix/server.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index e64a15c..1e9f8a6 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -241,6 +241,9 @@ class MatrixServer(object): self.unhandled_users = dict() # type: Dict[str, List[str]] self.lazy_load_hook = None # type: Optional[str] + self.keys_claimed = defaultdict(bool) + self.group_session_shared = defaultdict(bool) + self.config = ServerConfig(self.name, config_ptr) self._create_session_dir() # yapf: enable @@ -649,14 +652,20 @@ class MatrixServer(object): room.room_id, "m.room.message", body ) except GroupEncryptionError: + request = None try: - _, request = self.client.share_group_session(room.room_id) + if not self.group_session_shared[room.room_id]: + _, request = self.client.share_group_session(room.room_id) + self.group_session_shared[room.room_id] = True except EncryptionError: - _, request = self.client.keys_claim(room.room_id) + if not self.keys_claimed[room.room_id]: + _, request = self.client.keys_claim(room.room_id) + self.keys_claimed[room.room_id] = True message = EncrytpionQueueItem(msgtype, formatted) self.encryption_queue[room.room_id].append(message) - self.send_or_queue(request) + if request: + self.send_or_queue(request) return False if msgtype == "m.emote": @@ -878,6 +887,7 @@ class MatrixServer(object): self.handle_backlog_response(response) elif isinstance(response, KeysClaimResponse): + self.keys_claimed[response.room_id] = False try: _, request = self.client.share_group_session( response.room_id, @@ -892,6 +902,7 @@ class MatrixServer(object): elif isinstance(response, ShareGroupSessionResponse): room_id = response.room_id + self.group_session_shared[response.room_id] = False room_buffer = self.room_buffers[room_id] while self.encryption_queue[room_id]: From 1208c9d4a2e38ae6741558af30569358f79f4051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 2 Oct 2018 13:01:15 +0200 Subject: [PATCH 153/269] commands: Add initial olm command. --- main.py | 3 +- matrix/commands.py | 310 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 310 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 2715d58..3c17504 100644 --- a/main.py +++ b/main.py @@ -44,7 +44,8 @@ from matrix.commands import (hook_commands, hook_page_up, matrix_command_pgup_cb, matrix_invite_command_cb, matrix_join_command_cb, matrix_kick_command_cb, matrix_me_command_cb, matrix_part_command_cb, - matrix_redact_command_cb, matrix_topic_command_cb) + matrix_redact_command_cb, matrix_topic_command_cb, + matrix_olm_command_cb) from matrix.completion import (init_completion, matrix_command_completion_cb, matrix_debug_completion_cb, matrix_message_completion_cb, diff --git a/matrix/commands.py b/matrix/commands.py index 112092c..e6f9701 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -14,11 +14,11 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import unicode_literals - import argparse import re from builtins import str +from future.moves.itertools import zip_longest +from collections import defaultdict from . import globals as G from .colors import Formatted @@ -90,6 +90,45 @@ class WeechatCommandParser(object): parser.add_argument("room_id", nargs="?") return WeechatCommandParser._run_parser(parser, args) + @staticmethod + def olm(args): + parser = WeechatArgParse(prog="olm") + subparsers = parser.add_subparsers(dest="subcommand") + + info_parser = subparsers.add_parser("info") + info_parser.add_argument( + "category", nargs="?", default="private", + choices=[ + "all", + "blacklisted", + "private", + "unverified", + "verified" + ]) + info_parser.add_argument("filter", nargs="?") + + verify_parser = subparsers.add_parser("verify") + verify_parser.add_argument("user_filter") + verify_parser.add_argument("device_filter", nargs="?") + + unverify_parser = subparsers.add_parser("unverify") + unverify_parser.add_argument("user_filter") + unverify_parser.add_argument("device_filter", nargs="?") + + return WeechatCommandParser._run_parser(parser, args) + + +def grouper(iterable, n, fillvalue=None): + "Collect data into fixed-length chunks or blocks" + # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" + args = [iter(iterable)] * n + return zip_longest(*args, fillvalue=fillvalue) + + +def partition_key(key): + groups = grouper(key, 4, " ") + return ' '.join(''.join(g) for g in groups) + def hook_commands(): W.hook_command( @@ -246,12 +285,279 @@ def hook_commands(): "", ) + W.hook_command( + # Command name and short description + "olm", + "Matrix olm encryption configuration command", + # Synopsis + ("info all|blacklisted|private|unverified|verified ||" + "blacklist ||" + "unverify ||" + "verify "), + # Description + (" info: show info about known devices and their keys\n" + "blacklist: blacklist a device\n" + " unverify: unverify a device\n" + " verify: verify a device\n\n" + "Examples:\n"), + # Completions + ('info all|blacklisted|private|unverified|verified ||' + 'blacklist %(device_ids) ||' + 'unverify %(olm_user_ids) %(olm_devices) ||' + 'verify %(olm_user_ids) %(olm_devices)'), + # Function name + 'matrix_olm_command_cb', + '') + W.hook_command_run("/buffer clear", "matrix_command_buf_clear_cb", "") if G.CONFIG.network.fetch_backlog_on_pgup: hook_page_up() +def format_device(device_id, fp_key): + fp_key = partition_key(fp_key) + message = (" - Device ID: {device_color}{device_id}{ncolor}\n" + " - Device key: {key_color}{fp_key}{ncolor}").format( + device_color=W.color("chat_channel"), + device_id=device_id, + ncolor=W.color("reset"), + key_color=W.color("chat_server"), + fp_key=fp_key) + return message + + +def olm_info_command(server, args): + def print_devices( + device_store, + filter_regex, + device_category="Device", + predicate=None, + ): + user_strings = [] + try: + filter_regex = re.compile(args.filter) if args.filter else None + except re.error as e: + server.error("Invalid regular expression: {}.".format(e.args[0])) + return + + for user_id in sorted(device_store.users): + device_strings = [] + for device in device_store[user_id].values(): + if filter_regex: + if (not filter_regex.search(user_id) and + not filter_regex.search(device.id)): + continue + + if predicate: + if not predicate(device): + continue + + device_strings.append(format_device( + device.id, + device.ed25519 + )) + + if not device_strings: + continue + + d_string = "\n".join(device_strings) + message = (" - User: {user_color}{user}{ncolor}\n").format( + user_color=W.color("chat_nick"), + user=user_id, + ncolor=W.color("reset")) + message += d_string + user_strings.append(message) + + if not user_strings: + message = ("{prefix}matrix: No matching devices " + "found.").format(prefix=W.prefix("error")) + W.prnt(server.server_buffer, message) + return + + W.prnt(server.server_buffer, + "{}matrix: {} keys:\n".format( + W.prefix("network"), + device_category + )) + W.prnt(server.server_buffer, "\n".join(user_strings)) + + olm = server.client.olm + + if args.category == "private": + fp_key = partition_key(olm.account.identity_keys["ed25519"]) + message = ("{prefix}matrix: Identity keys:\n" + " - User: {user_color}{user}{ncolor}\n" + " - Device ID: {device_color}{device_id}{ncolor}\n" + " - Device key: {key_color}{fp_key}{ncolor}\n" + "").format( + prefix=W.prefix("network"), + user_color=W.color("chat_self"), + ncolor=W.color("reset"), + user=olm.user_id, + device_color=W.color("chat_channel"), + device_id=olm.device_id, + key_color=W.color("chat_server"), + fp_key=fp_key) + W.prnt(server.server_buffer, message) + + elif args.category == "all": + print_devices(olm.device_store, args.filter) + + elif args.category == "verified": + print_devices( + olm.device_store, + args.filter, + "Verified", + olm.is_device_verified + ) + + elif args.category == "unverified": + def predicate(device): + return not olm.is_device_verified(device) + + print_devices( + olm.device_store, + args.filter, + "Unverified", + predicate + ) + + elif args.category == "blacklisted": + print_devices( + olm.device_store, + args.filter, + "Blacklisted", + olm.is_device_blacklisted + ) + + +def olm_action_command(server, args, category, error_category, prefix, action): + device_store = server.client.olm.device_store + users = [] + + if args.user_filter == "*": + users = device_store.users + else: + users = [x for x in device_store.users if args.user_filter in x] + + user_devices = {user: device_store[user].values() for user in users} + + if args.device_filter and args.device_filter != "*": + filtered_user_devices = {} + for user, device_list in user_devices.items(): + filtered_devices = filter( + lambda x: args.device_filter in x.id, + device_list + ) + filtered_user_devices[user] = list(filtered_devices) + user_devices = filtered_user_devices + + changed_devices = defaultdict(list) + + for user, device_list in user_devices.items(): + for device in device_list: + if action(device): + changed_devices[user].append(device) + + if not changed_devices: + message = ("{prefix}matrix: No matching {error_category} devices " + "found.").format( + prefix=W.prefix("error"), + error_category=error_category + ) + W.prnt(server.server_buffer, message) + return + + user_strings = [] + for user_id, device_list in changed_devices.items(): + device_strings = [] + message = (" - User: {user_color}{user}{ncolor}\n").format( + user_color=W.color("chat_nick"), + user=user_id, + ncolor=W.color("reset")) + for device in device_list: + device_strings.append(format_device( + device.id, + device.ed25519 + )) + if not device_strings: + continue + + d_string = "\n".join(device_strings) + message += d_string + user_strings.append(message) + + W.prnt(server.server_buffer, + "{}matrix: {} key(s):\n".format(W.prefix("prefix"), category)) + W.prnt(server.server_buffer, "\n".join(user_strings)) + pass + + +def olm_verify_command(server, args): + olm_action_command( + server, + args, + "Verified", + "unverified", + "join", + server.client.olm.verify_device + ) + + +def olm_unverify_command(server, args): + olm_action_command( + server, + args, + "Unverified", + "verified", + "quit", + server.client.olm.unverify_device + ) + + +@utf8_decode +def matrix_olm_command_cb(data, buffer, args): + def command(server, data, buffer, args): + parsed_args = WeechatCommandParser.olm(args) + if not parsed_args: + return W.WEECHAT_RC_OK + + if not parsed_args: + return W.WEECHAT_RC_OK + + if not server.client.olm: + W.prnt(server.server_buffer, "{}matrix: Olm account isn't " + "loaded.".format(W.prefix("error"))) + return W.WEECHAT_RC_OK + + if not parsed_args.subcommand or parsed_args.subcommand == "info": + olm_info_command(server, parsed_args) + elif parsed_args.subcommand == "verify": + olm_verify_command(server, parsed_args) + elif parsed_args.subcommand == "unverify": + olm_unverify_command(server, parsed_args) + else: + message = ("{prefix}matrix: Command not implemented.".format( + prefix=W.prefix("error"))) + W.prnt(server.server_buffer, message) + + return W.WEECHAT_RC_OK + + for server in SERVERS.values(): + if buffer in server.buffers.values(): + return command(server, data, buffer, args) + elif buffer == server.server_buffer: + return command(server, data, buffer, args) + + W.prnt("", "{prefix}matrix: command \"olm\" must be executed on a " + "matrix buffer (server or channel)".format( + prefix=W.prefix("error") + )) + + return W.WEECHAT_RC_OK + + @utf8_decode def matrix_me_command_cb(data, buffer, args): for server in SERVERS.values(): From cfae54576aa2521608d1ca1fffeee67f29e251eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 2 Oct 2018 13:01:32 +0200 Subject: [PATCH 154/269] completion: Fix olm user/device completion. --- matrix/completion.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/matrix/completion.py b/matrix/completion.py index bf00682..67c78b9 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -154,9 +154,12 @@ def matrix_olm_user_completion_cb(data, completion_item, buffer, completion): if not server: return W.WEECHAT_RC_OK - olm = server.olm + olm = server.client.olm - for user in olm.device_keys: + if not olm: + return W.WEECHAT_RC_OK + + for user in olm.device_store.users: W.hook_completion_list_add( completion, user, 0, W.WEECHAT_LIST_POS_SORT ) @@ -171,7 +174,10 @@ def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): if not server: return W.WEECHAT_RC_OK - olm = server.olm + olm = server.client.olm + + if not olm: + return W.WEECHAT_RC_OK args = W.hook_completion_get_string(completion, "args") @@ -182,12 +188,12 @@ def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): user = fields[1] - if user not in olm.device_keys: + if user not in olm.device_store.users: return W.WEECHAT_RC_OK - for device in olm.device_keys[user]: + for device in olm.device_store[user]: W.hook_completion_list_add( - completion, device.device_id, 0, W.WEECHAT_LIST_POS_SORT + completion, device, 0, W.WEECHAT_LIST_POS_SORT ) return W.WEECHAT_RC_OK From c1754123a90523008a886d14e186b3270c753328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 2 Oct 2018 15:06:15 +0200 Subject: [PATCH 155/269] commands: Add the ability to blacklist devices via the olm command. --- matrix/commands.py | 53 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index e6f9701..d24486d 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -115,6 +115,14 @@ class WeechatCommandParser(object): unverify_parser.add_argument("user_filter") unverify_parser.add_argument("device_filter", nargs="?") + blacklist_parser = subparsers.add_parser("blacklist") + blacklist_parser.add_argument("user_filter") + blacklist_parser.add_argument("device_filter", nargs="?") + + unblacklist_parser = subparsers.add_parser("unblacklist") + unblacklist_parser.add_argument("user_filter") + unblacklist_parser.add_argument("device_filter", nargs="?") + return WeechatCommandParser._run_parser(parser, args) @@ -295,14 +303,19 @@ def hook_commands(): "unverify ||" "verify "), # Description - (" info: show info about known devices and their keys\n" - "blacklist: blacklist a device\n" - " unverify: unverify a device\n" - " verify: verify a device\n\n" - "Examples:\n"), + (" info: show info about known devices and their keys\n" + " blacklist: blacklist a device\n" + "unblacklist: unblacklist a device\n" + " unverify: unverify a device\n" + " verify: verify a device\n\n" + "Examples:" + "\n /olm verify @example:example.com *" + "\n /olm info all example*" + ), # Completions ('info all|blacklisted|private|unverified|verified ||' - 'blacklist %(device_ids) ||' + 'blacklist %(olm_user_ids) %(olm_devices) ||' + 'unblacklist %(olm_user_ids) %(olm_devices) ||' 'unverify %(olm_user_ids) %(olm_devices) ||' 'verify %(olm_user_ids) %(olm_devices)'), # Function name @@ -376,7 +389,7 @@ def olm_info_command(server, args): return W.prnt(server.server_buffer, - "{}matrix: {} keys:\n".format( + "{}matrix: {} devices:\n".format( W.prefix("network"), device_category )) @@ -516,6 +529,28 @@ def olm_unverify_command(server, args): ) +def olm_blacklist_command(server, args): + olm_action_command( + server, + args, + "Blacklisted", + "unblacklisted", + "join", + server.client.olm.blacklist_device + ) + + +def olm_unblacklist_command(server, args): + olm_action_command( + server, + args, + "Unblacklisted", + "blacklisted", + "join", + server.client.olm.unblacklist_device + ) + + @utf8_decode def matrix_olm_command_cb(data, buffer, args): def command(server, data, buffer, args): @@ -537,6 +572,10 @@ def matrix_olm_command_cb(data, buffer, args): olm_verify_command(server, parsed_args) elif parsed_args.subcommand == "unverify": olm_unverify_command(server, parsed_args) + elif parsed_args.subcommand == "blacklist": + olm_blacklist_command(server, parsed_args) + elif parsed_args.subcommand == "unblacklist": + olm_unblacklist_command(server, parsed_args) else: message = ("{prefix}matrix: Command not implemented.".format( prefix=W.prefix("error"))) From 7a5c0c9c0e3bca66ec08a37b2d89cf4986fe7841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 2 Oct 2018 17:27:29 +0200 Subject: [PATCH 156/269] commands: Use the client verify commands instead of the olm ones. --- matrix/commands.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index d24486d..299e2d0 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -514,7 +514,7 @@ def olm_verify_command(server, args): "Verified", "unverified", "join", - server.client.olm.verify_device + server.client.verify_device ) @@ -525,7 +525,7 @@ def olm_unverify_command(server, args): "Unverified", "verified", "quit", - server.client.olm.unverify_device + server.client.unverify_device ) @@ -536,7 +536,7 @@ def olm_blacklist_command(server, args): "Blacklisted", "unblacklisted", "join", - server.client.olm.blacklist_device + server.client.blacklist_device ) @@ -547,7 +547,7 @@ def olm_unblacklist_command(server, args): "Unblacklisted", "blacklisted", "join", - server.client.olm.unblacklist_device + server.client.unblacklist_device ) From a8b62577d7e6df7c36fe90955ea722d7b8593434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 12 Oct 2018 13:52:48 +0200 Subject: [PATCH 157/269] server: Utilize partial sync responses. --- main.py | 7 ++--- matrix/bar_items.py | 2 +- matrix/buffer.py | 1 + matrix/globals.py | 1 + matrix/server.py | 66 ++++++++++++++++++++++++++++++++++----------- 5 files changed, 58 insertions(+), 19 deletions(-) diff --git a/main.py b/main.py index 3c17504..9727c9e 100644 --- a/main.py +++ b/main.py @@ -57,12 +57,13 @@ from matrix.completion import (init_completion, matrix_command_completion_cb, from matrix.config import (MatrixConfig, config_log_category_cb, config_log_level_cb, config_server_buffer_cb, matrix_config_reload_cb, config_pgup_cb) -from matrix.globals import SCRIPT_NAME, SERVERS, W +from matrix.globals import SCRIPT_NAME, SERVERS, W, MAX_EVENTS from matrix.server import (MatrixServer, create_default_server, matrix_config_server_change_cb, matrix_config_server_read_cb, matrix_config_server_write_cb, matrix_timer_cb, - send_cb, matrix_load_users_cb) + send_cb, matrix_load_users_cb, + matrix_partial_sync_cb) from matrix.utf import utf8_decode from matrix.utils import server_buffer_prnt, server_buffer_set_title @@ -288,7 +289,7 @@ def receive_cb(server_name, file_descriptor): server.disconnect() break - response = server.client.next_response() + response = server.client.next_response(MAX_EVENTS) # Check if we need to send some data back data_to_send = server.client.data_to_send() diff --git a/matrix/bar_items.py b/matrix/bar_items.py index b154003..7d7f111 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -109,7 +109,7 @@ def matrix_bar_item_buffer_modes(data, item, window, buffer, extra_info): if not server.connected: modes.append("❌") - if room_buffer.backlog_pending: + if room_buffer.backlog_pending or server.busy: modes.append("⏳") return "".join(modes) diff --git a/matrix/buffer.py b/matrix/buffer.py index cdd5623..c14e3f6 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -816,6 +816,7 @@ class RoomBuffer(object): def backlog_pending(self, value): self._backlog_pending = value W.bar_item_update("buffer_modes") + W.bar_item_update("matrix_modes") @property def warning_prefix(self): diff --git a/matrix/globals.py b/matrix/globals.py index b8e60d4..8f2b6bb 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -39,3 +39,4 @@ SERVERS = dict() # type: Dict[str, MatrixServer] CONFIG = None # type: Optional[MatrixConfig] ENCRYPTION = True # type: bool SCRIPT_NAME = "matrix" # type: str +MAX_EVENTS = 10 diff --git a/matrix/server.py b/matrix/server.py index 1e9f8a6..8e6c7c2 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -32,6 +32,7 @@ from nio import ( Rooms, RoomSendResponse, SyncResponse, + PartialSyncResponse, ShareGroupSessionResponse, KeysClaimResponse, TransportResponse, @@ -46,7 +47,7 @@ from nio import ( from . import globals as G from .buffer import OwnAction, OwnMessage, RoomBuffer from .config import ConfigSection, Option, ServerBufferType -from .globals import SCRIPT_NAME, SERVERS, W +from .globals import SCRIPT_NAME, SERVERS, W, MAX_EVENTS from .utf import utf8_decode from .utils import create_server_buffer, key_from_value, server_buffer_prnt @@ -228,6 +229,7 @@ class MatrixServer(object): self.transaction_id = 0 # type: int self.lag = 0 # type: int self.lag_done = False # type: bool + self.busy = False # type: bool self.send_fd_hook = None # type: Optional[str] self.send_buffer = b"" # type: bytes @@ -240,6 +242,7 @@ class MatrixServer(object): self.unhandled_users = dict() # type: Dict[str, List[str]] self.lazy_load_hook = None # type: Optional[str] + self.partial_sync_hook = None # type: Optional[str] self.keys_claimed = defaultdict(bool) self.group_session_shared = defaultdict(bool) @@ -793,13 +796,6 @@ class MatrixServer(object): room_buffer = self.find_room_from_id(room_id) room_buffer.handle_joined_room(info) - if room_buffer.unhandled_users: - should_lazy_hook = True - - if should_lazy_hook: - hook = W.hook_timer(1 * 100, 0, 0, "matrix_load_users_cb", - self.name) - self.lazy_load_hook = hook def add_unhandled_users(self, rooms, n): # type: (List[RoomBuffer], int) -> bool @@ -837,15 +833,31 @@ class MatrixServer(object): self._handle_room_info(response) - self.next_batch = response.next_batch + # Full sync response handle everything. + if isinstance(response, SyncResponse): + if self.client.should_upload_keys: + self.keys_upload() - if self.client.should_upload_keys: - self.keys_upload() + if self.client.should_query_keys and not self.keys_queried: + self.keys_query() - if self.client.should_query_keys and not self.keys_queried: - self.keys_query() + for room_buffer in self.room_buffers.values(): + if room_buffer.unhandled_users: + hook = W.hook_timer(1 * 100, 0, 0, "matrix_load_users_cb", + self.name) + self.lazy_load_hook = hook + break - self.schedule_sync() + self.next_batch = response.next_batch + self.schedule_sync() + else: + if not self.partial_sync_hook: + hook = W.hook_timer(1 * 100, 0, 0, "matrix_partial_sync_cb", + self.name) + self.partial_sync_hook = hook + self.busy = True + W.bar_item_update("buffer_modes") + W.bar_item_update("matrix_modes") def handle_transport_response(self, response): self.error( @@ -877,7 +889,7 @@ class MatrixServer(object): elif isinstance(response, LoginResponse): self._handle_login(response) - elif isinstance(response, SyncResponse): + elif isinstance(response, (SyncResponse, PartialSyncResponse)): self._handle_sync(response) elif isinstance(response, RoomSendResponse): @@ -1022,6 +1034,30 @@ def matrix_config_server_change_cb(server_name, option): return 1 +@utf8_decode +def matrix_partial_sync_cb(server_name, remaining_calls): + start = time.time() + server = SERVERS[server_name] + W.unhook(server.partial_sync_hook) + server.partial_sync_hook = None + + response = server.client.next_response(MAX_EVENTS) + + while response: + server.handle_response(response) + current = time.time() + if current - start >= 0.1: + break + response = server.client.next_response(MAX_EVENTS) + + if not server.partial_sync_hook: + server.busy = False + W.bar_item_update("buffer_modes") + W.bar_item_update("matrix_modes") + + return W.WEECHAT_RC_OK + + @utf8_decode def matrix_load_users_cb(server_name, remaining_calls): server = SERVERS[server_name] From 4ebcfe59a2eb0dd8b368f64d24f86f91386fefd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 12 Oct 2018 14:05:32 +0200 Subject: [PATCH 158/269] buffer: Change the deferred user loading logic. Now that we get partial sync responses the difference between added users and total room users won't ever be a large number. Just deffer user adding based on the number of already added users. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index c14e3f6..2b587ae 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -877,7 +877,7 @@ class RoomBuffer(object): # the strings that are compared at every itteration. # Because the search time get's increasingly longer we're # going to add nicks later in a timer hook. - if ((len(self.room.users) - len(self.displayed_nicks)) > 500 + if (len(self.displayed_nicks) > 100 and is_state): self.unhandled_users.append(event.state_key) else: From 701085f8ffe8ce89ad088b14d61518cff6e98a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 12 Oct 2018 14:07:24 +0200 Subject: [PATCH 159/269] buffer: Always add users with a higher power level. --- matrix/buffer.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 2b587ae..880cd07 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -879,7 +879,16 @@ class RoomBuffer(object): # going to add nicks later in a timer hook. if (len(self.displayed_nicks) > 100 and is_state): - self.unhandled_users.append(event.state_key) + # Always add users with a high power level + try: + user = self.room.users[event.state_key] + except KeyError: + self.unhandled_users.append(event.state_key) + else: + if user.power_level > 0: + self.add_user(event.state_key, date, is_state) + else: + self.unhandled_users.append(event.state_key) else: self.add_user(event.state_key, date, is_state) else: From 08ad527a1f3f553af0c8302d3adfcca34a26981b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 12 Oct 2018 14:07:56 +0200 Subject: [PATCH 160/269] buffer: Remove uneeded print inside a comment. --- matrix/buffer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 880cd07..6254de6 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -836,7 +836,6 @@ class RoomBuffer(object): except KeyError: # No user found, he must have left already in an event that is # yet to come, so do nothing - # W.prnt("", "NOT ADDING USER {}".format(user_id)) return short_name = shorten_sender(user.user_id) From d8362c572bd83129feacc03008b2cba579c7a1b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 13 Oct 2018 12:13:10 +0200 Subject: [PATCH 161/269] commands: Don't show deleted devices in the olm command. --- matrix/commands.py | 6 ++++-- matrix/completion.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index 299e2d0..008a8c5 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -356,7 +356,7 @@ def olm_info_command(server, args): for user_id in sorted(device_store.users): device_strings = [] - for device in device_store[user_id].values(): + for device in device_store.active_user_devices(user_id): if filter_regex: if (not filter_regex.search(user_id) and not filter_regex.search(device.id)): @@ -454,7 +454,9 @@ def olm_action_command(server, args, category, error_category, prefix, action): else: users = [x for x in device_store.users if args.user_filter in x] - user_devices = {user: device_store[user].values() for user in users} + user_devices = { + user: device_store.active_user_devices(user) for user in users + } if args.device_filter and args.device_filter != "*": filtered_user_devices = {} diff --git a/matrix/completion.py b/matrix/completion.py index 67c78b9..fe943dd 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -188,6 +188,17 @@ def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): user = fields[1] + if user not in olm.device_store.users: + return W.WEECHAT_RC_OK + + for device in olm.device_store.active_user_devices(user): + W.hook_completion_list_add( + completion, device.id, 0, W.WEECHAT_LIST_POS_SORT + ) + + return W.WEECHAT_RC_OK + + if user not in olm.device_store.users: return W.WEECHAT_RC_OK From 07d4743db0cc714d95a93b9b50ec02ff446d4352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 13 Oct 2018 12:14:06 +0200 Subject: [PATCH 162/269] commands: Remove duplicate lines. --- matrix/commands.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index 008a8c5..ca0ac3f 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -560,9 +560,6 @@ def matrix_olm_command_cb(data, buffer, args): if not parsed_args: return W.WEECHAT_RC_OK - if not parsed_args: - return W.WEECHAT_RC_OK - if not server.client.olm: W.prnt(server.server_buffer, "{}matrix: Olm account isn't " "loaded.".format(W.prefix("error"))) From 4bf0e4be5cb3539a75e19c8ec7b258916768bdf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 13 Oct 2018 12:14:54 +0200 Subject: [PATCH 163/269] server: Mark a finished key query. --- matrix/server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/matrix/server.py b/matrix/server.py index 8e6c7c2..a4e4690 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -34,6 +34,7 @@ from nio import ( SyncResponse, PartialSyncResponse, ShareGroupSessionResponse, + KeysQueryResponse, KeysClaimResponse, TransportResponse, TransportType, @@ -898,6 +899,9 @@ class MatrixServer(object): elif isinstance(response, RoomMessagesResponse): self.handle_backlog_response(response) + elif isinstance(response, KeysQueryResponse): + self.keys_queried = False + elif isinstance(response, KeysClaimResponse): self.keys_claimed[response.room_id] = False try: From 35f47547d23fc55c80f8347ec60f4ad67d6ba79d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 11:47:11 +0200 Subject: [PATCH 164/269] commands: Add initial device manipulation command. --- main.py | 5 +++-- matrix/commands.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ matrix/completion.py | 30 ++++++++++++++++++++++++++ matrix/server.py | 49 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 9727c9e..b3ad24a 100644 --- a/main.py +++ b/main.py @@ -45,7 +45,7 @@ from matrix.commands import (hook_commands, hook_page_up, matrix_join_command_cb, matrix_kick_command_cb, matrix_me_command_cb, matrix_part_command_cb, matrix_redact_command_cb, matrix_topic_command_cb, - matrix_olm_command_cb) + matrix_olm_command_cb, matrix_devices_command_cb) from matrix.completion import (init_completion, matrix_command_completion_cb, matrix_debug_completion_cb, matrix_message_completion_cb, @@ -53,7 +53,8 @@ from matrix.completion import (init_completion, matrix_command_completion_cb, matrix_olm_user_completion_cb, matrix_server_command_completion_cb, matrix_server_completion_cb, - matrix_user_completion_cb) + matrix_user_completion_cb, + matrix_own_devices_completion_cb) from matrix.config import (MatrixConfig, config_log_category_cb, config_log_level_cb, config_server_buffer_cb, matrix_config_reload_cb, config_pgup_cb) diff --git a/matrix/commands.py b/matrix/commands.py index ca0ac3f..8aefad9 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -90,6 +90,17 @@ class WeechatCommandParser(object): parser.add_argument("room_id", nargs="?") return WeechatCommandParser._run_parser(parser, args) + @staticmethod + def devices(args): + parser = WeechatArgParse(prog="devices") + subparsers = parser.add_subparsers(dest="subcommand") + subparsers.add_parser("list") + + delete_parser = subparsers.add_parser("delete") + delete_parser.add_argument("device_id") + + return WeechatCommandParser._run_parser(parser, args) + @staticmethod def olm(args): parser = WeechatArgParse(prog="olm") @@ -293,6 +304,23 @@ def hook_commands(): "", ) + W.hook_command( + # Command name and short description + "devices", + "list or delete matrix devices", + # Synopsis + ("list ||" + "delete "), + # Description + ("device-id: device id of the device to delete"), + # Completions + ("list ||" + "delete %(matrix_own_devices)"), + # Callback + "matrix_devices_command_cb", + "", + ) + W.hook_command( # Command name and short description "olm", @@ -596,6 +624,29 @@ def matrix_olm_command_cb(data, buffer, args): return W.WEECHAT_RC_OK +@utf8_decode +def matrix_devices_command_cb(data, buffer, args): + for server in SERVERS.values(): + if buffer in server.buffers.values() or buffer == server.server_buffer: + parsed_args = WeechatCommandParser.devices(args) + if not parsed_args: + return W.WEECHAT_RC_OK + + if not parsed_args.subcommand or parsed_args.subcommand == "list": + server.devices() + elif parsed_args.subcommand == "delete": + server.delete_device(parsed_args.delete_device) + + return W.WEECHAT_RC_OK + + W.prnt("", "{prefix}matrix: command \"devices\" must be executed on a " + "matrix buffer (server or channel)".format( + prefix=W.prefix("error") + )) + + return W.WEECHAT_RC_OK + + @utf8_decode def matrix_me_command_cb(data, buffer, args): for server in SERVERS.values(): diff --git a/matrix/completion.py b/matrix/completion.py index fe943dd..0df1362 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -199,6 +199,29 @@ def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): return W.WEECHAT_RC_OK +@utf8_decode +def matrix_own_devices_completion_cb( + data, + completion_item, + buffer, + completion +): + server = server_from_buffer(buffer) + + if not server: + return W.WEECHAT_RC_OK + + olm = server.client.olm + + if not olm: + return W.WEECHAT_RC_OK + + W.hook_completion_list_add( + completion, olm.device_id, 0, W.WEECHAT_LIST_POS_SORT + ) + + user = olm.user_id + if user not in olm.device_store.users: return W.WEECHAT_RC_OK @@ -292,3 +315,10 @@ def init_completion(): "matrix_user_completion_cb", "", ) + + W.hook_completion( + "matrix_own_devices", + "Matrix own devices completion", + "matrix_own_devices_completion_cb", + "", + ) diff --git a/matrix/server.py b/matrix/server.py index a4e4690..f2d75ed 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -36,6 +36,7 @@ from nio import ( ShareGroupSessionResponse, KeysQueryResponse, KeysClaimResponse, + DevicesResponse, TransportResponse, TransportType, RoomMessagesResponse, @@ -582,6 +583,14 @@ class MatrixServer(object): W.prnt(self.server_buffer, msg) + def devices(self): + _, request = self.client.devices() + self.send_or_queue(request) + + def delete_device(self, device_id): + # TODO implement this + return + def room_send_state(self, room_buffer, body, event_type): if room_buffer.room.encrypted: return @@ -731,6 +740,43 @@ class MatrixServer(object): room_buffer.handle_backlog(response) + def handle_devices_response(self, response): + if not response.devices: + m = "{}{}: No devices found for this account".format( + W.prefix("error"), + SCRIPT_NAME) + W.prnt(self.server_buffer, m) + + header = (W.prefix("network") + SCRIPT_NAME + ": devices for " + "server {}{}{}:\n" + " Device ID Device Name " + "Last Seen").format( + W.color("chat_server"), + self.name, + W.color("reset") + ) + W.prnt(self.server_buffer, header) + + lines = [] + for device in response.devices: + last_seen = "{ip} @ {date}".format( + ip=device.last_seen_ip, + date=device.last_seen_date.strftime("%Y/%m/%d %H:%M") + ) + device_color = ("chat_self" if device.id == self.device_id else + W.info_get("nick_color_name", device.id)) + bold = W.color("bold") if device.id == self.device_id else "" + line = " {}{}{:<18}{}{:<29}{:<}".format( + bold, + W.color(device_color), + device.id, + W.color("resetcolor"), + device.display_name, + last_seen + ) + lines.append(line) + W.prnt(self.server_buffer, "\n".join(lines)) + def _handle_login(self, response): self.access_token = response.access_token self.user_id = response.user_id @@ -899,6 +945,9 @@ class MatrixServer(object): elif isinstance(response, RoomMessagesResponse): self.handle_backlog_response(response) + elif isinstance(response, DevicesResponse): + self.handle_devices_response(response) + elif isinstance(response, KeysQueryResponse): self.keys_queried = False From b65346d49c15a023e2a96c9b3e479683a896150d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 11:50:07 +0200 Subject: [PATCH 165/269] server: Add device deletion functionality. --- matrix/commands.py | 2 +- matrix/server.py | 54 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index 8aefad9..322579a 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -635,7 +635,7 @@ def matrix_devices_command_cb(data, buffer, args): if not parsed_args.subcommand or parsed_args.subcommand == "list": server.devices() elif parsed_args.subcommand == "delete": - server.delete_device(parsed_args.delete_device) + server.delete_device(parsed_args.device_id) return W.WEECHAT_RC_OK diff --git a/matrix/server.py b/matrix/server.py index f2d75ed..0bd96c7 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -37,6 +37,8 @@ from nio import ( KeysQueryResponse, KeysClaimResponse, DevicesResponse, + DeleteDevicesAuthResponse, + DeleteDevicesResponse, TransportResponse, TransportType, RoomMessagesResponse, @@ -44,6 +46,9 @@ from nio import ( EncryptionError, GroupEncryptionError, OlmTrustError, + ErrorResponse, + SyncError, + LoginError, ) from . import globals as G @@ -237,6 +242,8 @@ class MatrixServer(object): self.send_buffer = b"" # type: bytes self.device_check_timestamp = None # type: Optional[int] + self.device_deletion_queue = dict() + self.own_message_queue = dict() # type: Dict[str, OwnMessage] self.encryption_queue = defaultdict(deque) \ # type: DefaultDict[str, Deque[EncrytpionQueueItem]] @@ -587,8 +594,10 @@ class MatrixServer(object): _, request = self.client.devices() self.send_or_queue(request) - def delete_device(self, device_id): - # TODO implement this + def delete_device(self, device_id, auth=None): + uuid, request = self.client.delete_devices([device_id], auth) + self.device_deletion_queue[uuid] = device_id + self.send_or_queue(request) return def room_send_state(self, room_buffer, body, event_type): @@ -906,15 +915,30 @@ class MatrixServer(object): W.bar_item_update("buffer_modes") W.bar_item_update("matrix_modes") - def handle_transport_response(self, response): - self.error( - ("Error with response of type type: {}, " "error code {}").format( - response.request_info.type, response.status_code - ) - ) + def handle_delete_device_auth(self, response): + device_id = self.device_deletion_queue.pop(response.uuid, None) - # TODO better error handling. - if response.request_info.type in (RequestType.sync, RequestType.login): + if not device_id: + return + + for flow in response.flows: + if "m.login.password" in flow["stages"]: + session = response.session + auth = { + "type": "m.login.password", + "session": session, + "user": self.client.user_id, + "password": self.config.password + } + self.delete_device(device_id, auth) + return + + self.error("No supported auth method for device deletion found.") + + def handle_error_response(self, response): + self.error("Error: {}".format(str(response))) + + if isinstance(response, (SyncError, LoginError)): self.disconnect() def handle_response(self, response): @@ -930,8 +954,8 @@ class MatrixServer(object): self.lag_done = True W.bar_item_update("lag") - if isinstance(response, TransportResponse): - self.handle_transport_response(response) + if isinstance(response, ErrorResponse): + self.handle_error_response(response) elif isinstance(response, LoginResponse): self._handle_login(response) @@ -948,6 +972,12 @@ class MatrixServer(object): elif isinstance(response, DevicesResponse): self.handle_devices_response(response) + elif isinstance(response, DeleteDevicesAuthResponse): + self.handle_delete_device_auth(response) + + elif isinstance(response, DeleteDevicesResponse): + self.info("Device successfully deleted") + elif isinstance(response, KeysQueryResponse): self.keys_queried = False From f95f33d217a57608bb1da723a8483be002434e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 11:51:03 +0200 Subject: [PATCH 166/269] commands: Allow device renaming. --- matrix/commands.py | 19 +++++++++++++++---- matrix/server.py | 12 ++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index 322579a..0f2be8d 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -99,6 +99,10 @@ class WeechatCommandParser(object): delete_parser = subparsers.add_parser("delete") delete_parser.add_argument("device_id") + name_parser = subparsers.add_parser("set-name") + name_parser.add_argument("device_id") + name_parser.add_argument("device_name", nargs="*") + return WeechatCommandParser._run_parser(parser, args) @staticmethod @@ -307,15 +311,19 @@ def hook_commands(): W.hook_command( # Command name and short description "devices", - "list or delete matrix devices", + "list, delete or rename matrix devices", # Synopsis ("list ||" - "delete "), + "delete ||" + "set-name " + ), # Description - ("device-id: device id of the device to delete"), + ("device-id: device id of the device to delete\n" + " name: new device name to set\n"), # Completions ("list ||" - "delete %(matrix_own_devices)"), + "delete %(matrix_own_devices) ||" + "set-name %(matrix_own_devices)"), # Callback "matrix_devices_command_cb", "", @@ -636,6 +644,9 @@ def matrix_devices_command_cb(data, buffer, args): server.devices() elif parsed_args.subcommand == "delete": server.delete_device(parsed_args.device_id) + elif parsed_args.subcommand == "set-name": + new_name = " ".join(parsed_args.device_name).strip("\"") + server.rename_device(parsed_args.device_id, new_name) return W.WEECHAT_RC_OK diff --git a/matrix/server.py b/matrix/server.py index 0bd96c7..174a658 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -37,6 +37,7 @@ from nio import ( KeysQueryResponse, KeysClaimResponse, DevicesResponse, + UpdateDeviceResponse, DeleteDevicesAuthResponse, DeleteDevicesResponse, TransportResponse, @@ -600,6 +601,14 @@ class MatrixServer(object): self.send_or_queue(request) return + def rename_device(self, device_id, display_name): + content = { + "display_name": display_name + } + + _, request = self.client.update_device(device_id, content) + self.send_or_queue(request) + def room_send_state(self, room_buffer, body, event_type): if room_buffer.room.encrypted: return @@ -972,6 +981,9 @@ class MatrixServer(object): elif isinstance(response, DevicesResponse): self.handle_devices_response(response) + elif isinstance(response, UpdateDeviceResponse): + self.info("Device name successfully updated") + elif isinstance(response, DeleteDevicesAuthResponse): self.handle_delete_device_auth(response) From a99cc24d668f22caf21cb7b373c0366f7d295572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 11:55:02 +0200 Subject: [PATCH 167/269] completion: Don't complete deleted devices in our own devices. --- matrix/completion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matrix/completion.py b/matrix/completion.py index 0df1362..56644e6 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -225,9 +225,9 @@ def matrix_own_devices_completion_cb( if user not in olm.device_store.users: return W.WEECHAT_RC_OK - for device in olm.device_store[user]: + for device in olm.device_store.active_user_devices(user): W.hook_completion_list_add( - completion, device, 0, W.WEECHAT_LIST_POS_SORT + completion, device.id, 0, W.WEECHAT_LIST_POS_SORT ) return W.WEECHAT_RC_OK From 516be65bd3dcb1702544b5a18dc033e6fcffe6ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 11:55:48 +0200 Subject: [PATCH 168/269] server: Do a full key query after an initial sync. --- matrix/server.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 174a658..832f9f5 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -716,8 +716,8 @@ class MatrixServer(object): _, request = self.client.keys_upload() self.send_or_queue(request) - def keys_query(self): - _, request = self.client.keys_query() + def keys_query(self, full=False): + _, request = self.client.keys_query(full) self.keys_queried = True self.send_or_queue(request) @@ -903,7 +903,11 @@ class MatrixServer(object): if self.client.should_upload_keys: self.keys_upload() - if self.client.should_query_keys and not self.keys_queried: + # Query the keys for all users in the encrypted rooms after our + # initial sync + if not self.next_batch and not self.keys_queried: + self.keys_query(True) + elif self.client.should_query_keys and not self.keys_queried: self.keys_query() for room_buffer in self.room_buffers.values(): From c26e30848ffdbde3831b0820f9d8e7180d8f68ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 11:56:07 +0200 Subject: [PATCH 169/269] server: Slight change to the formatting of the device list. --- matrix/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 832f9f5..8b59231 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -765,9 +765,9 @@ class MatrixServer(object): SCRIPT_NAME) W.prnt(self.server_buffer, m) - header = (W.prefix("network") + SCRIPT_NAME + ": devices for " + header = (W.prefix("network") + SCRIPT_NAME + ": Devices for " "server {}{}{}:\n" - " Device ID Device Name " + " Device ID Device Name " "Last Seen").format( W.color("chat_server"), self.name, @@ -784,7 +784,7 @@ class MatrixServer(object): device_color = ("chat_self" if device.id == self.device_id else W.info_get("nick_color_name", device.id)) bold = W.color("bold") if device.id == self.device_id else "" - line = " {}{}{:<18}{}{:<29}{:<}".format( + line = " {}{}{:<18}{}{:<34}{:<}".format( bold, W.color(device_color), device.id, From d66621578b83fa11df637d515e0901ba2cefab4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 11:56:47 +0200 Subject: [PATCH 170/269] buffer: Remove the buffer from the server buffer list if it has been closed. --- matrix/buffer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 6254de6..d012e9d 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -101,7 +101,15 @@ def room_buffer_input_cb(server_name, buffer, input_data): @utf8_decode -def room_buffer_close_cb(data, buffer): +def room_buffer_close_cb(server_name, buffer): + server = SERVERS[server_name] + room_buffer = server.find_room_from_ptr(buffer) + + if room_buffer: + room_id = room_buffer.room.room_id + server.buffers.pop(room_id, None) + server.room_buffers.pop(room_id, None) + return W.WEECHAT_RC_OK From 868a47eca71afa6982eca1928986a91628d1b1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 12:47:47 +0200 Subject: [PATCH 171/269] commands: Misc olm command fixes. --- matrix/commands.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index 0f2be8d..aab4dc9 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -380,7 +380,7 @@ def olm_info_command(server, args): def print_devices( device_store, filter_regex, - device_category="Device", + device_category="All", predicate=None, ): user_strings = [] @@ -424,23 +424,18 @@ def olm_info_command(server, args): W.prnt(server.server_buffer, message) return - W.prnt(server.server_buffer, - "{}matrix: {} devices:\n".format( - W.prefix("network"), - device_category - )) + server.info("{} devices:\n".format(device_category)) W.prnt(server.server_buffer, "\n".join(user_strings)) olm = server.client.olm if args.category == "private": fp_key = partition_key(olm.account.identity_keys["ed25519"]) - message = ("{prefix}matrix: Identity keys:\n" + message = ("Identity keys:\n" " - User: {user_color}{user}{ncolor}\n" " - Device ID: {device_color}{device_id}{ncolor}\n" " - Device key: {key_color}{fp_key}{ncolor}\n" "").format( - prefix=W.prefix("network"), user_color=W.color("chat_self"), ncolor=W.color("reset"), user=olm.user_id, @@ -448,7 +443,7 @@ def olm_info_command(server, args): device_id=olm.device_id, key_color=W.color("chat_server"), fp_key=fp_key) - W.prnt(server.server_buffer, message) + server.info(message) elif args.category == "all": print_devices(olm.device_store, args.filter) From 167bf0c07ac48196cce0f9fecec6c27787c452c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 13:13:18 +0200 Subject: [PATCH 172/269] buffer: Don't mark encryption as unsupported anymore. --- matrix/buffer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index d012e9d..5f0d8c1 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -1067,12 +1067,7 @@ class RoomBuffer(object): elif isinstance(event, PowerLevelsEvent): self._handle_power_level(event) elif isinstance(event, RoomEncryptionEvent): - message = ( - "This room is encrypted, encryption is " - "currently unsuported. Message sending is disabled for " - "this room." - ) - self.weechat_buffer.error(message) + pass def handle_timeline_event(self, event): # TODO this should be done for every messagetype that gets printed in From a97af644cdc009ace649edb3164769f9d884d1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 14:02:37 +0200 Subject: [PATCH 173/269] commands: Use unicode literals. --- matrix/commands.py | 1 + matrix/config.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/commands.py b/matrix/commands.py index aab4dc9..7969fe7 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -14,6 +14,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +from __future__ import unicode_literals import argparse import re from builtins import str diff --git a/matrix/config.py b/matrix/config.py index 0ce38e0..0451e8a 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -14,7 +14,6 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# from __future__ import unicode_literals from builtins import super from collections import namedtuple from enum import Enum, unique From 0a8e491b00c48011ce16f1fb946fca9834eba6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 14 Oct 2018 14:16:51 +0200 Subject: [PATCH 174/269] config: Fix log level and category setting. --- matrix/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/matrix/config.py b/matrix/config.py index 0451e8a..cc04dc7 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -41,6 +41,9 @@ class ServerBufferType(Enum): INDEPENDENT = 2 +nio.logger_group.level = logbook.ERROR + + class Option( namedtuple( "Option", @@ -371,6 +374,7 @@ class MatrixConfig(WeechatConfig): "all", "Debugging category", logbook_category, + config_log_category_cb, ), Option( "debug_buffer", From 8f1c79e640a3be290a0b588630bd4cbdc9470224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 15 Oct 2018 22:14:43 +0200 Subject: [PATCH 175/269] main: Catch the CertificateError exception when doing the ssl handshake. --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index b3ad24a..658d526 100644 --- a/main.py +++ b/main.py @@ -223,7 +223,7 @@ def try_ssl_handshake(server): return False - except (ssl.SSLError, socket.error) as error: + except (ssl.SSLError, ssl.CertificateError, socket.error) as error: try: str_error = error.reason if error.reason else "Unknown error" except AttributeError: From ccb263977a27b50dbe2e014da0ed11163314a82b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 15 Oct 2018 22:15:44 +0200 Subject: [PATCH 176/269] commands: Fix buffer clearing. --- matrix/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matrix/commands.py b/matrix/commands.py index 7969fe7..0395ac8 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -754,8 +754,8 @@ def hook_page_up(): def matrix_command_buf_clear_cb(data, buffer, command): for server in SERVERS.values(): if buffer in server.buffers.values(): - room_id = key_from_value(server.buffers, buffer) - server.rooms[room_id].prev_batch = server.next_batch + room_buffer = server.find_room_from_ptr(buffer) + room_buffer.room.prev_batch = server.next_batch return W.WEECHAT_RC_OK From 53b514809c3cd9123bc4e9cf25ffe50fbc2a172a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 15 Oct 2018 22:18:15 +0200 Subject: [PATCH 177/269] server: Only hook the lazy hook if it isn't already hooked. --- matrix/server.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 8b59231..1057ff7 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -852,8 +852,6 @@ class MatrixServer(object): room_buffer = self.find_room_from_id(room_id) room_buffer.handle_left_room(info) - should_lazy_hook = False - for room_id, info in response.rooms.join.items(): if room_id not in self.buffers: self.create_room_buffer(room_id, info.timeline.prev_batch) @@ -861,7 +859,6 @@ class MatrixServer(object): room_buffer = self.find_room_from_id(room_id) room_buffer.handle_joined_room(info) - def add_unhandled_users(self, rooms, n): # type: (List[RoomBuffer], int) -> bool total_users = 0 @@ -912,9 +909,10 @@ class MatrixServer(object): for room_buffer in self.room_buffers.values(): if room_buffer.unhandled_users: - hook = W.hook_timer(1 * 100, 0, 0, "matrix_load_users_cb", - self.name) - self.lazy_load_hook = hook + if not self.lazy_load_hook: + hook = W.hook_timer(1 * 1000, 0, 0, + "matrix_load_users_cb", self.name) + self.lazy_load_hook = hook break self.next_batch = response.next_batch From fbf56d349141f17e1968a510dd60e94bfbc96395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 19 Oct 2018 18:03:08 +0200 Subject: [PATCH 178/269] main: Close the file descriptor we get from hook_connect(). We create a python socket object with fromfd() which duplicates the file descriptor we get from hook_connect(). Close the unneeded file descriptor instead of leaking it. --- main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/main.py b/main.py index 658d526..3f5fb69 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,7 @@ from __future__ import unicode_literals +import os import socket import ssl import textwrap @@ -144,6 +145,10 @@ def wrap_socket(server, file_descriptor): temp_socket = socket.fromfd(file_descriptor, socket.AF_INET, socket.SOCK_STREAM) + # fromfd() duplicates the file descriptor, we can close the one we got from + # weechat now since we use the one from our socket when calling hook_fd() + os.close(file_descriptor) + # For python 2.7 wrap_socket() doesn't work with sockets created from an # file descriptor because fromfd() doesn't return a wrapped socket, the bug # was fixed for python 3, more info: https://bugs.python.org/issue13942 From a46a50fd7439cddabccc0102f3f0494be2ca607c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 19 Oct 2018 18:05:08 +0200 Subject: [PATCH 179/269] server: Close the socket even if shutdown() raises an exception. --- matrix/server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index 1057ff7..94922a5 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -459,10 +459,14 @@ class MatrixServer(object): if self.socket: try: self.socket.shutdown(socket.SHUT_RDWR) - self.socket.close() except socket.error: pass + try: + self.socket.close() + except OSError: + pass + def disconnect(self, reconnect=True): # type: (bool) -> None if self.fd_hook: From 45d83cc1b12a86c08700a24311e2576d4c74dab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 21 Oct 2018 13:36:01 +0200 Subject: [PATCH 180/269] buffer: Post the encryption error message in the current buffer. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 5f0d8c1..3b588c5 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -94,7 +94,7 @@ def room_buffer_input_cb(server_name, buffer, input_data): server.room_send_message(room_buffer, formatted_data, "m.text") except OlmTrustError as e: m = ("Untrusted devices found in room: {}".format(e)) - server.error(m) + room_buffer.error(m) pass return W.WEECHAT_RC_OK From 60282b4eec1addc4125f5672fd1965786a30f689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 21 Oct 2018 18:56:39 +0200 Subject: [PATCH 181/269] buffer: Support encrypted attachments. Note, the printed url should be passed to a helper program that will download the file and decrypt it. --- matrix/buffer.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index 3b588c5..57ff631 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -33,6 +33,7 @@ from nio import ( RoomMessage, RoomMessageEmote, RoomMessageMedia, + RoomEncryptedMedia, RoomMessageNotice, RoomMessageText, RoomMessageUnknown, @@ -1151,6 +1152,31 @@ class RoomBuffer(object): nick, data, date, self.get_event_tags(event), extra_prefix ) + elif isinstance(event, RoomEncryptedMedia): + nick = self.find_nick(event.sender) + date = server_ts_to_weechat(event.server_timestamp) + http_url = Api.encrypted_mxc_to_plumb( + event.url, + event.key["k"], + event.hashes["sha256"], + event.iv + ) + url = http_url if http_url else event.url + + description = "{}".format(event.body) if event.body else "file" + data = ("{del_color}<{ncolor}{desc}{del_color}>{ncolor} " + "{del_color}[{ncolor}{url}{del_color}]{ncolor}").format( + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + desc=description, url=url) + + extra_prefix = (self.warning_prefix if event.decrypted + and not event.verified else "") + + self.weechat_buffer.message( + nick, data, date, self.get_event_tags(event), extra_prefix + ) + elif isinstance(event, RoomMessageUnknown): nick = self.find_nick(event.sender) date = server_ts_to_weechat(event.server_timestamp) From 3be1b7bfc3107f09b42d5d236ead373327a90380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 30 Oct 2018 16:15:27 +0100 Subject: [PATCH 182/269] server: Use server side lazy user loading. --- matrix/server.py | 70 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 94922a5..a42afac 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -50,6 +50,8 @@ from nio import ( ErrorResponse, SyncError, LoginError, + JoinedMembersResponse, + JoinedMembersError, ) from . import globals as G @@ -250,7 +252,7 @@ class MatrixServer(object): # type: DefaultDict[str, Deque[EncrytpionQueueItem]] self.backlog_queue = dict() # type: Dict[str, str] - self.unhandled_users = dict() # type: Dict[str, List[str]] + self.rooms_with_missing_members = [] # type: List[str] self.lazy_load_hook = None # type: Optional[str] self.partial_sync_hook = None # type: Optional[str] @@ -580,7 +582,12 @@ class MatrixServer(object): ).format(prefix=W.prefix("network"), script_name=SCRIPT_NAME) W.prnt(self.server_buffer, msg) timeout = 0 if self.transport_type == TransportType.HTTP else 30000 - sync_filter = {"room": {"timeline": {"limit": 5000}}} + sync_filter = { + "room": { + "timeline": {"limit": 5000}, + "state": {"lazy_load_members": True} + } + } self.sync(timeout, sync_filter) return @@ -725,6 +732,10 @@ class MatrixServer(object): self.keys_queried = True self.send_or_queue(request) + def get_joined_members(self, room_id): + _, request = self.client.joined_members(room_id) + self.send(request) + def _print_message_error(self, message): server_buffer_prnt( self, @@ -817,7 +828,10 @@ class MatrixServer(object): sync_filter = { "room": { - "timeline": {"limit": G.CONFIG.network.max_initial_sync_events} + "timeline": { + "limit": G.CONFIG.network.max_initial_sync_events + }, + "state": {"lazy_load_members": True} } } self.sync(timeout=0, sync_filter=sync_filter) @@ -891,6 +905,12 @@ class MatrixServer(object): return False + def _hook_lazy_user_adding(self): + if not self.lazy_load_hook: + hook = W.hook_timer(1 * 1000, 0, 0, + "matrix_load_users_cb", self.name) + self.lazy_load_hook = hook + def _handle_sync(self, response): # we got the same batch again, nothing to do if self.next_batch == response.next_batch: @@ -904,23 +924,23 @@ class MatrixServer(object): if self.client.should_upload_keys: self.keys_upload() - # Query the keys for all users in the encrypted rooms after our - # initial sync - if not self.next_batch and not self.keys_queried: - self.keys_query(True) - elif self.client.should_query_keys and not self.keys_queried: + if self.client.should_query_keys and not self.keys_queried: self.keys_query() for room_buffer in self.room_buffers.values(): + if not self.next_batch: + self.rooms_with_missing_members.append( + room_buffer.room.room_id + ) if room_buffer.unhandled_users: - if not self.lazy_load_hook: - hook = W.hook_timer(1 * 1000, 0, 0, - "matrix_load_users_cb", self.name) - self.lazy_load_hook = hook + self._hook_lazy_user_adding() break self.next_batch = response.next_batch self.schedule_sync() + + if self.rooms_with_missing_members: + self.get_joined_members(self.rooms_with_missing_members.pop()) else: if not self.partial_sync_hook: hook = W.hook_timer(1 * 100, 0, 0, "matrix_partial_sync_cb", @@ -955,6 +975,9 @@ class MatrixServer(object): if isinstance(response, (SyncError, LoginError)): self.disconnect() + elif isinstance(response, JoinedMembersError): + self.rooms_with_missing_members.append(response.room_id) + self.get_joined_members(self.rooms_with_missing_members.pop()) def handle_response(self, response): # type: (Response) -> None @@ -999,6 +1022,22 @@ class MatrixServer(object): elif isinstance(response, KeysQueryResponse): self.keys_queried = False + elif isinstance(response, JoinedMembersResponse): + room_buffer = self.room_buffers[response.room_id] + users = [user.user_id for user in response.members] + + # Don't add the users directly use the lazy load hook. + room_buffer.unhandled_users += users + self._hook_lazy_user_adding() + + # Fetch the users for the next room. + if self.rooms_with_missing_members: + self.get_joined_members(self.rooms_with_missing_members.pop()) + # We are done adding all the users, do a full key query now since + # the client knows all the encrypted room members. + else: + self.keys_query(True) + elif isinstance(response, KeysClaimResponse): self.keys_claimed[response.room_id] = False try: @@ -1208,7 +1247,12 @@ def matrix_timer_cb(server_name, remaining_calls): if server.sync_time and current_time > (server.sync_time + 2): timeout = 0 if server.transport_type == TransportType.HTTP else 30000 - sync_filter = {"room": {"timeline": {"limit": 5000}}} + sync_filter = { + "room": { + "timeline": {"limit": 5000}, + "state": {"lazy_load_members": True} + } + } server.sync(timeout, sync_filter) if not server.next_batch: From 0710b7f77ddc8263c237c5cf6d8c133c29405497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 30 Oct 2018 16:15:57 +0100 Subject: [PATCH 183/269] buffer: Don't add already added users to the RoomBuffer. --- matrix/buffer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index 57ff631..b26c97f 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -847,6 +847,10 @@ class RoomBuffer(object): # yet to come, so do nothing return + # User is already added don't add him again. + if user_id in self.displayed_nicks: + return + short_name = shorten_sender(user.user_id) # TODO handle this special case for discord bridge users and From d896962b194af760839d588405ebb5454d7562be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 30 Oct 2018 17:20:44 +0100 Subject: [PATCH 184/269] matrix: Add option to fetch members only when we switch to the buffer. --- main.py | 31 +++++++++++++++++++++++++++++++ matrix/buffer.py | 1 + matrix/config.py | 11 +++++++++++ matrix/server.py | 18 +++++++++++++++--- 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 3f5fb69..2c8be57 100644 --- a/main.py +++ b/main.py @@ -461,6 +461,35 @@ class WeechatHandler(StreamHandler): W.prnt(buf, item) +def lazy_fetch_members_signal(_, _signal, buffer_ptr): + """ Fetch room members on a buffer switch signal. + This function is called every time we switch a buffer. The pointer of + the new buffer is given to us by weechat. If it is one of our room buffers + we check if the members for the room aren't fetched and fetch them now if + they aren't. + """ + for server in SERVERS.values(): + if buffer_ptr == server.server_buffer: + return W.WEECHAT_RC_OK + + if buffer_ptr not in server.buffers.values(): + continue + + room_buffer = server.find_room_from_ptr(buffer_ptr) + if not room_buffer: + continue + + if room_buffer.members_fetched: + return W.WEECHAT_RC_OK + + room_id = room_buffer.room.room_id + server.get_joined_members(room_id) + + break + + return W.WEECHAT_RC_OK + + if __name__ == "__main__": if W.register(WEECHAT_SCRIPT_NAME, WEECHAT_SCRIPT_AUTHOR, WEECHAT_SCRIPT_VERSION, WEECHAT_SCRIPT_LICENSE, @@ -483,6 +512,8 @@ if __name__ == "__main__": init_bar_items() init_completion() + hook = W.hook_signal("buffer_switch", "lazy_fetch_members_signal", "") + if not SERVERS: create_default_server(G.CONFIG) diff --git a/matrix/buffer.py b/matrix/buffer.py index b26c97f..83fb8b8 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -805,6 +805,7 @@ class RoomBuffer(object): self.prev_batch = prev_batch self.joined = True self.leave_event_id = None # type: Optional[str] + self.members_fetched = False self.unhandled_users = [] # type: List[str] buffer_name = "{}.{}".format(server_name, room.room_id) diff --git a/matrix/config.py b/matrix/config.py index cc04dc7..570bab9 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -385,6 +385,17 @@ class MatrixConfig(WeechatConfig): "off", ("Use a separate buffer for debug logs."), ), + Option( + "lazy_load_room_users", + "boolean", + "", + 0, + 0, + "off", + ("If on, room users wont be loaded background proactively " + "they will be loaded when the buffer becomes active. This " + "only affects non encrypted rooms."), + ), ] color_options = [ diff --git a/matrix/server.py b/matrix/server.py index a42afac..bbb8d52 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -252,6 +252,7 @@ class MatrixServer(object): # type: DefaultDict[str, Deque[EncrytpionQueueItem]] self.backlog_queue = dict() # type: Dict[str, str] + self.member_request_list = [] # type: List[str] self.rooms_with_missing_members = [] # type: List[str] self.lazy_load_hook = None # type: Optional[str] self.partial_sync_hook = None # type: Optional[str] @@ -483,6 +484,7 @@ class MatrixServer(object): self.send_buffer = b"" self.transport_type = None + self.member_request_list.clear() if self.client: try: @@ -733,6 +735,10 @@ class MatrixServer(object): self.send_or_queue(request) def get_joined_members(self, room_id): + if room_id in self.member_request_list: + return + + self.member_request_list.append(room_id) _, request = self.client.joined_members(room_id) self.send(request) @@ -928,10 +934,14 @@ class MatrixServer(object): self.keys_query() for room_buffer in self.room_buffers.values(): + # It's our initial sync, we need to fetch room members, so add + # the room to the missing members queue. if not self.next_batch: - self.rooms_with_missing_members.append( - room_buffer.room.room_id - ) + if (not G.CONFIG.network.lazy_load_room_users + or room_buffer.room.encrypted): + self.rooms_with_missing_members.append( + room_buffer.room.room_id + ) if room_buffer.unhandled_users: self._hook_lazy_user_adding() break @@ -1023,12 +1033,14 @@ class MatrixServer(object): self.keys_queried = False elif isinstance(response, JoinedMembersResponse): + self.member_request_list.remove(response.room_id) room_buffer = self.room_buffers[response.room_id] users = [user.user_id for user in response.members] # Don't add the users directly use the lazy load hook. room_buffer.unhandled_users += users self._hook_lazy_user_adding() + room_buffer.members_fetched = True # Fetch the users for the next room. if self.rooms_with_missing_members: From 74efe8ef9b6ad37aebefa26ea168e9e1bb2e1d5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 30 Oct 2018 21:12:41 +0100 Subject: [PATCH 185/269] server: Add a maximum value for the number of nicklist users. --- main.py | 11 +++++--- matrix/bar_items.py | 17 ++++++++++++ matrix/buffer.py | 65 ++++++++++++++++++++++++--------------------- matrix/config.py | 12 +++++++++ matrix/server.py | 52 ++++++++++++++++++++++++++++++++++-- 5 files changed, 122 insertions(+), 35 deletions(-) diff --git a/main.py b/main.py index 2c8be57..3f91cdd 100644 --- a/main.py +++ b/main.py @@ -34,9 +34,14 @@ from logbook import Logger, StreamHandler from nio import RemoteProtocolError, RemoteTransportError, TransportType from matrix import globals as G -from matrix.bar_items import (init_bar_items, matrix_bar_item_buffer_modes, - matrix_bar_item_lag, matrix_bar_item_name, - matrix_bar_item_plugin) +from matrix.bar_items import ( + init_bar_items, + matrix_bar_item_buffer_modes, + matrix_bar_item_lag, + matrix_bar_item_name, + matrix_bar_item_plugin, + matrix_bar_nicklist_count +) from matrix.buffer import room_buffer_close_cb, room_buffer_input_cb # Weechat searches for the registered callbacks in the scope of the main script # file, import the callbacks here so weechat can find them. diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 7d7f111..bfa3ecb 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -117,9 +117,26 @@ def matrix_bar_item_buffer_modes(data, item, window, buffer, extra_info): return "" +@utf8_decode +def matrix_bar_nicklist_count(data, item, window, buffer, extra_info): + # pylint: disable=unused-argument + for server in SERVERS.values(): + if buffer in server.buffers.values(): + room_buffer = server.find_room_from_ptr(buffer) + room = room_buffer.room + return str(len(room.users)) + + return "" + + def init_bar_items(): W.bar_item_new("(extra)buffer_plugin", "matrix_bar_item_plugin", "") W.bar_item_new("(extra)buffer_name", "matrix_bar_item_name", "") W.bar_item_new("(extra)lag", "matrix_bar_item_lag", "") + W.bar_item_new( + "(extra)buffer_nicklist_count", + "matrix_bar_nicklist_count", + "" + ) W.bar_item_new("(extra)buffer_modes", "matrix_bar_item_buffer_modes", "") W.bar_item_new("(extra)matrix_modes", "matrix_bar_item_buffer_modes", "") diff --git a/matrix/buffer.py b/matrix/buffer.py index 83fb8b8..6f684ae 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -692,7 +692,7 @@ class WeechatChannelBuffer(object): message = self._membership_message(user, "invite") self.print_date_tags(message, date, tags + (extra_tags or [])) - def _remove_user_from_nicklist(self, user): + def remove_user_from_nicklist(self, user): # type: (WeechatUser) -> None nick_pointer = W.nicklist_search_nick(self._ptr, "", user.nick) @@ -702,7 +702,7 @@ class WeechatChannelBuffer(object): def _leave(self, nick, date, message, leave_type, extra_tags=None): # type: (str, int, bool, str, List[str]) -> None user = self._get_user(nick) - self._remove_user_from_nicklist(user) + self.remove_user_from_nicklist(user) if len(self.users) <= 2: W.buffer_set(self._ptr, "localvar_set_type", "private") @@ -807,6 +807,7 @@ class RoomBuffer(object): self.leave_event_id = None # type: Optional[str] self.members_fetched = False self.unhandled_users = [] # type: List[str] + self.inactive_users = [] buffer_name = "{}.{}".format(server_name, room.room_id) @@ -840,7 +841,11 @@ class RoomBuffer(object): return user_id - def add_user(self, user_id, date, is_state): + def add_user(self, user_id, date, is_state, force_add=False): + # User is already added don't add him again. + if user_id in self.displayed_nicks: + return + try: user = self.room.users[user_id] except KeyError: @@ -848,9 +853,24 @@ class RoomBuffer(object): # yet to come, so do nothing return - # User is already added don't add him again. - if user_id in self.displayed_nicks: - return + # Adding users to the nicklist is a O(1) + search time + # operation (the nicks are added to a linked list sorted). + # The search time is O(N * min(a,b)) where N is the number + # of nicks already added and a/b are the length of + # the strings that are compared at every itteration. + # Because the search time get's increasingly longer we're + # going to stop adding inactive users, they will be lazily added if + # they become active. + if is_state and not force_add and user.power_level <= 0: + if (len(self.displayed_nicks) >= + G.CONFIG.network.max_nicklist_users): + self.inactive_users.append(user_id) + return + + try: + self.inactive_users.remove(user_id) + except ValueError: + pass short_name = shorten_sender(user.user_id) @@ -882,28 +902,13 @@ class RoomBuffer(object): date = server_ts_to_weechat(event.server_timestamp) if event.content["membership"] == "join": - if event.state_key not in self.displayed_nicks: - # Adding users to the nicklist is a O(1) + search time - # operation (the nicks are added to a linked list sorted). - # The search time is O(N * min(a,b)) where N is the number - # of nicks already added and a/b are the length of - # the strings that are compared at every itteration. - # Because the search time get's increasingly longer we're - # going to add nicks later in a timer hook. - if (len(self.displayed_nicks) > 100 - and is_state): - # Always add users with a high power level - try: - user = self.room.users[event.state_key] - except KeyError: - self.unhandled_users.append(event.state_key) - else: - if user.power_level > 0: - self.add_user(event.state_key, date, is_state) - else: - self.unhandled_users.append(event.state_key) - else: - self.add_user(event.state_key, date, is_state) + if (event.state_key not in self.displayed_nicks + and event.state_key not in self.inactive_users): + if len(self.room.users) > 100: + self.unhandled_users.append(event.state_key) + return + + self.add_user(event.state_key, date, is_state) else: # TODO print out profile changes return @@ -1062,7 +1067,7 @@ class RoomBuffer(object): # There is no way to change the group of a user without # removing him from the nicklist - self.weechat_buffer._remove_user_from_nicklist(user) + self.weechat_buffer.remove_user_from_nicklist(user) self.weechat_buffer._add_user_to_nicklist(user) def handle_state_event(self, event): @@ -1087,7 +1092,7 @@ class RoomBuffer(object): except ValueError: pass - self.add_user(event.sender, 0, True) + self.add_user(event.sender, 0, True, True) if isinstance(event, RoomMemberEvent): self.handle_membership_events(event, False) diff --git a/matrix/config.py b/matrix/config.py index 570bab9..7ad3c27 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -396,6 +396,18 @@ class MatrixConfig(WeechatConfig): "they will be loaded when the buffer becomes active. This " "only affects non encrypted rooms."), ), + Option( + "max_nicklist_users", + "integer", + "", + 100, + 20000, + "5000", + ("Limit the number of users that are added to the nicklist. " + "Active users and users with a higher power level are always." + " Inactive users will be removed from the nicklist after a " + "day of inactivity."), + ), ] color_options = [ diff --git a/matrix/server.py b/matrix/server.py index bbb8d52..f18d743 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -252,6 +252,7 @@ class MatrixServer(object): # type: DefaultDict[str, Deque[EncrytpionQueueItem]] self.backlog_queue = dict() # type: Dict[str, str] + self.user_gc_time = time.time() # type: float self.member_request_list = [] # type: List[str] self.rooms_with_missing_members = [] # type: List[str] self.lazy_load_hook = None # type: Optional[str] @@ -1101,6 +1102,53 @@ class MatrixServer(object): room_buffer = self.room_buffers[room_id] return room_buffer + def garbage_collect_users(self): + """ Remove inactive users. + This tries to keep the number of users added to the nicklist less than + the configuration option matrix.network.max_nicklist_users. It + removes users that have not been active for a day until there are + less than max_nicklist_users or no users are left for removal. + It never removes users that have a bigger power level than the + default one. + This function is run every hour by the server timer callback""" + + now = time.time() + self.user_gc_time = now + + def day_passed(t1, t2): + return (t2 - t1) > 86400 + + for room_buffer in self.room_buffers.values(): + to_remove = max( + (len(room_buffer.displayed_nicks) - + G.CONFIG.network.max_nicklist_users), + 0 + ) + + if not to_remove: + continue + + removed = 0 + removed_user_ids = [] + + for user_id, nick in room_buffer.displayed_nicks.items(): + user = room_buffer.weechat_buffer.users[nick] + + if (not user.speaking_time or + day_passed(user.speaking_time, now)): + room_buffer.weechat_buffer.part(nick, 0, False) + removed_user_ids.append(user_id) + removed += 1 + + if removed >= to_remove: + break + + for user_id in removed_user_ids: + W.prnt("", "Garbage collected {}".format(user_id)) + del room_buffer.displayed_nicks[user_id] + + pass + def buffer_merge(self): if not self.server_buffer: return @@ -1267,8 +1315,8 @@ def matrix_timer_cb(server_name, remaining_calls): } server.sync(timeout, sync_filter) - if not server.next_batch: - return W.WEECHAT_RC_OK + if current_time > (server.user_gc_time + 3600): + server.garbage_collect_users() return W.WEECHAT_RC_OK From c8e2e665376b8649688a3863456962c9c43ca2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 30 Oct 2018 21:13:35 +0100 Subject: [PATCH 186/269] server: Update the bar items when a key query completes. --- matrix/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/matrix/server.py b/matrix/server.py index f18d743..d4de366 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1032,6 +1032,8 @@ class MatrixServer(object): elif isinstance(response, KeysQueryResponse): self.keys_queried = False + W.bar_item_update("buffer_modes") + W.bar_item_update("matrix_modes") elif isinstance(response, JoinedMembersResponse): self.member_request_list.remove(response.room_id) From 441acbadc78d0911f1d30526f772357e89edcbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 30 Oct 2018 21:13:59 +0100 Subject: [PATCH 187/269] server: Print out trust errors in the room buffer. --- matrix/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index d4de366..7015974 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1062,7 +1062,8 @@ class MatrixServer(object): ) except OlmTrustError as e: m = ("Untrusted devices found in room: {}".format(e)) - self.error(m) + room_buffer = self.find_room_from_id(response.room_id) + room_buffer.error(m) return self.send(request) From c29db687e44395adb275fdf3ddfa95963407c0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 30 Oct 2018 22:58:36 +0100 Subject: [PATCH 188/269] server: Send out a hsignal when a device change occurs. --- matrix/server.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/matrix/server.py b/matrix/server.py index 7015974..79414f2 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1035,6 +1035,17 @@ class MatrixServer(object): W.bar_item_update("buffer_modes") W.bar_item_update("matrix_modes") + for user_id, device_dict in response.changed.items(): + for device in device_dict.values(): + message = { + "user_id": user_id, + "device_id": device.id, + "ed25519": device.ed25519, + "curve25519": device.curve25519, + "deleted": str(device.deleted) + } + W.hook_hsignal_send("matrix_device_changed", message) + elif isinstance(response, JoinedMembersResponse): self.member_request_list.remove(response.room_id) room_buffer = self.room_buffers[response.room_id] From 6e27082c1a4994a0e525a8d26dad9ad42bfe4a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 31 Oct 2018 09:27:37 +0100 Subject: [PATCH 189/269] server: Don't use clear() on a list since it's python3 only. --- matrix/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index 79414f2..e9cc585 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -485,7 +485,7 @@ class MatrixServer(object): self.send_buffer = b"" self.transport_type = None - self.member_request_list.clear() + self.member_request_list = [] if self.client: try: From 64bea928ef2bc012c5e2546d6f261115c53c1b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 1 Nov 2018 11:56:43 +0100 Subject: [PATCH 190/269] bar_items: Add a typing notice bar item. --- main.py | 3 ++- matrix/bar_items.py | 41 +++++++++++++++++++++++++++++++++++++++++ matrix/config.py | 18 ++++++++++++++++++ matrix/server.py | 1 + 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 3f91cdd..ef8c8ea 100644 --- a/main.py +++ b/main.py @@ -40,7 +40,8 @@ from matrix.bar_items import ( matrix_bar_item_lag, matrix_bar_item_name, matrix_bar_item_plugin, - matrix_bar_nicklist_count + matrix_bar_nicklist_count, + matrix_bar_typing_notices_cb ) from matrix.buffer import room_buffer_close_cb, room_buffer_input_cb # Weechat searches for the registered callbacks in the scope of the main script diff --git a/matrix/bar_items.py b/matrix/bar_items.py index bfa3ecb..226b80c 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -16,6 +16,7 @@ from __future__ import unicode_literals +from . import globals as G from .globals import SERVERS, W from .utf import utf8_decode @@ -129,6 +130,41 @@ def matrix_bar_nicklist_count(data, item, window, buffer, extra_info): return "" +@utf8_decode +def matrix_bar_typing_notices_cb(data, item, window, buffer, extra_info): + """Update a status bar item showing users currently typing. + This function is called by weechat every time a buffer is switched or + W.bar_item_update() is explicitly called. The bar item shows + currently typing users for the current buffer.""" + # pylint: disable=unused-argument + for server in SERVERS.values(): + if buffer in server.buffers.values(): + room_buffer = server.find_room_from_ptr(buffer) + room = room_buffer.room + + if room.typing_users: + nicks = [] + + for user_id in room.typing_users: + nick = room_buffer.displayed_nicks.get(user_id, user_id) + nicks.append(nick) + + msg = "{}{}".format( + G.CONFIG.look.bar_item_typing_notice_prefix, + ", ".join(sorted(nicks)) + ) + + max_len = G.CONFIG.look.max_typing_notice_item_length + if len(msg) > max_len: + msg[:max_len - 3] + "..." + + return msg + + return "" + + return "" + + def init_bar_items(): W.bar_item_new("(extra)buffer_plugin", "matrix_bar_item_plugin", "") W.bar_item_new("(extra)buffer_name", "matrix_bar_item_name", "") @@ -138,5 +174,10 @@ def init_bar_items(): "matrix_bar_nicklist_count", "" ) + W.bar_item_new( + "(extra)matrix_typing_notice", + "matrix_bar_typing_notices_cb", + "" + ) W.bar_item_new("(extra)buffer_modes", "matrix_bar_item_buffer_modes", "") W.bar_item_new("(extra)matrix_modes", "matrix_bar_item_buffer_modes", "") diff --git a/matrix/config.py b/matrix/config.py index 7ad3c27..c96c4bb 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -322,6 +322,24 @@ class MatrixConfig(WeechatConfig): ServerBufferType, config_server_buffer_cb, ), + Option( + "max_typing_notice_item_length", + "integer", + "", + 10, + 1000, + "50", + ("Limit the length of the typing notice bar item."), + ), + Option( + "bar_item_typing_notice_prefix", + "string", + "", + 0, + 0, + "Typing: ", + ("Prefix for the typing notice bar item."), + ), ] network_options = [ diff --git a/matrix/server.py b/matrix/server.py index e9cc585..2567982 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -949,6 +949,7 @@ class MatrixServer(object): self.next_batch = response.next_batch self.schedule_sync() + W.bar_item_update("matrix_typing_notice") if self.rooms_with_missing_members: self.get_joined_members(self.rooms_with_missing_members.pop()) From ec995e6c8fe428844506c096cf2944c73dfae4ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 1 Nov 2018 12:47:18 +0100 Subject: [PATCH 191/269] main: Use the server error method for hook connect errors. --- main.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/main.py b/main.py index ef8c8ea..b683acd 100644 --- a/main.py +++ b/main.py @@ -363,41 +363,36 @@ def connect_cb(data, status, gnutls_rc, sock, error, ip_address): return W.WEECHAT_RC_OK elif status_value == W.WEECHAT_HOOK_CONNECT_ADDRESS_NOT_FOUND: - W.prnt( - server.server_buffer, - '{address} not found'.format(address=ip_address)) + server.error('{address} not found'.format(address=ip_address)) elif status_value == W.WEECHAT_HOOK_CONNECT_IP_ADDRESS_NOT_FOUND: - W.prnt(server.server_buffer, 'IP address not found') + server.error('IP address not found') elif status_value == W.WEECHAT_HOOK_CONNECT_CONNECTION_REFUSED: - W.prnt(server.server_buffer, 'Connection refused') + server.error('Connection refused') elif status_value == W.WEECHAT_HOOK_CONNECT_PROXY_ERROR: - W.prnt(server.server_buffer, - 'Proxy fails to establish connection to server') + server.error('Proxy fails to establish connection to server') elif status_value == W.WEECHAT_HOOK_CONNECT_LOCAL_HOSTNAME_ERROR: - W.prnt(server.server_buffer, 'Unable to set local hostname') + server.error('Unable to set local hostname') elif status_value == W.WEECHAT_HOOK_CONNECT_GNUTLS_INIT_ERROR: - W.prnt(server.server_buffer, 'TLS init error') + server.error('TLS init error') elif status_value == W.WEECHAT_HOOK_CONNECT_GNUTLS_HANDSHAKE_ERROR: - W.prnt(server.server_buffer, 'TLS Handshake failed') + server.error('TLS Handshake failed') elif status_value == W.WEECHAT_HOOK_CONNECT_MEMORY_ERROR: - W.prnt(server.server_buffer, 'Not enough memory') + server.error('Not enough memory') elif status_value == W.WEECHAT_HOOK_CONNECT_TIMEOUT: - W.prnt(server.server_buffer, 'Timeout') + server.error('Timeout') elif status_value == W.WEECHAT_HOOK_CONNECT_SOCKET_ERROR: - W.prnt(server.server_buffer, 'Unable to create socket') + server.error('Unable to create socket') else: - W.prnt( - server.server_buffer, - 'Unexpected error: {status}'.format(status=status_value)) + server.error('Unexpected error: {status}'.format(status=status_value)) server.disconnect(reconnect=True) return W.WEECHAT_RC_OK From b844a26c41c7a9862e56425c0e2d2ce7a6496dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 1 Nov 2018 12:47:58 +0100 Subject: [PATCH 192/269] server: Clear the encryption queue if we encounter an olm trust error. --- matrix/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/matrix/server.py b/matrix/server.py index 2567982..0feb787 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1076,6 +1076,7 @@ class MatrixServer(object): m = ("Untrusted devices found in room: {}".format(e)) room_buffer = self.find_room_from_id(response.room_id) room_buffer.error(m) + self.encryption_queue[response.room_id].clear() return self.send(request) @@ -1095,6 +1096,7 @@ class MatrixServer(object): self.encryption_queue[room_id].appendleft(message) break except OlmTrustError: + self.encryption_queue[room_id].clear() break def create_room_buffer(self, room_id, prev_batch): From 0bc297ba946eb4022fe93459d10b1a6c57d9e3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 1 Nov 2018 13:33:48 +0100 Subject: [PATCH 193/269] server: Make the lag reconnect time configurable. --- matrix/config.py | 10 ++++++++++ matrix/server.py | 3 +-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/matrix/config.py b/matrix/config.py index c96c4bb..2efeb0f 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -426,6 +426,16 @@ class MatrixConfig(WeechatConfig): " Inactive users will be removed from the nicklist after a " "day of inactivity."), ), + Option( + "lag_reconnect", + "integer", + "", + 5, + 604800, + "90", + ("Reconnect to the server if the lag is greater than this " + "value (in seconds)"), + ), ] color_options = [ diff --git a/matrix/server.py b/matrix/server.py index 0feb787..5bed5e5 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1317,8 +1317,7 @@ def matrix_timer_cb(server_name, remaining_calls): server.lag_done = False W.bar_item_update("lag") - # TODO print out message, make timeout configurable - if server.lag > 300000: + if server.lag > G.CONFIG.network.lag_reconnect * 1000: server.disconnect() return W.WEECHAT_RC_OK From 34b884b8922290261228e85b14e5297ce36d6631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 2 Nov 2018 13:53:20 +0100 Subject: [PATCH 194/269] buffer: Fix the printing of our own action messages. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 6f684ae..142cb5f 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -556,7 +556,7 @@ class WeechatChannelBuffer(object): user.update_speaking_time(date) self.unmask_smart_filtered_nick(nick) - def _print_action(self, user, message, date, tags, extra_prefix): + def _print_action(self, user, message, date, tags, extra_prefix=""): nick_prefix = ( "" if not user.prefix From b8c655b3d42b837a2ef407a6ff7c844bd65af808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 2 Nov 2018 14:04:22 +0100 Subject: [PATCH 195/269] commands: Update the buffer modes bar items after an olm command run. --- matrix/commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/matrix/commands.py b/matrix/commands.py index 0395ac8..7ab5edf 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -612,6 +612,9 @@ def matrix_olm_command_cb(data, buffer, args): prefix=W.prefix("error"))) W.prnt(server.server_buffer, message) + W.bar_item_update("buffer_modes") + W.bar_item_update("matrix_modes") + return W.WEECHAT_RC_OK for server in SERVERS.values(): From e8fa66bbcbf2f607243a6c9c2726700ec1f5d4ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 2 Nov 2018 14:10:08 +0100 Subject: [PATCH 196/269] config: Improve the lazy_load_room_users help message. --- matrix/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/matrix/config.py b/matrix/config.py index 2efeb0f..b78a536 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -410,9 +410,9 @@ class MatrixConfig(WeechatConfig): 0, 0, "off", - ("If on, room users wont be loaded background proactively " - "they will be loaded when the buffer becomes active. This " - "only affects non encrypted rooms."), + ("If on, room users won't be loaded in the background " + "proactively, they will be loaded when the user switches to " + "the room buffer. This only affects non-encrypted rooms."), ), Option( "max_nicklist_users", From 286b15aa86395330d09d1bb6bf8070f0896ff39b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 3 Nov 2018 12:30:20 +0100 Subject: [PATCH 197/269] bar_items: Make the encryption warning sign configurable. --- matrix/bar_items.py | 4 +++- matrix/config.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 226b80c..34569e5 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -105,7 +105,9 @@ def matrix_bar_item_buffer_modes(data, item, window, buffer, extra_info): if (server.client and server.client.room_contains_unverified(room.room_id)): - modes.append("⚠️ ") + modes.append( + G.CONFIG.look.encryption_warning_sign, + ) if not server.connected: modes.append("❌") diff --git a/matrix/config.py b/matrix/config.py index b78a536..c612686 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -178,6 +178,10 @@ def logbook_category(value): return "all" +def eval_cast(string): + return W.string_eval_expression(string, {}, {}, {}) + + class WeechatConfig(object): def __init__(self, sections): self._ptr = W.config_new( @@ -268,6 +272,8 @@ class ConfigSection(object): return bool(W.config_boolean(self._option_ptrs[name])) def str_getter(self): + if cast_func: + return cast_func(W.config_string(self._option_ptrs[name])) return W.config_string(self._option_ptrs[name]) def str_evaluate_getter(self): @@ -340,6 +346,17 @@ class MatrixConfig(WeechatConfig): "Typing: ", ("Prefix for the typing notice bar item."), ), + Option( + "encryption_warning_sign", + "string", + "", + 0, + 0, + "⚠️ ", + ("A sign that is used to signal trust issues in encrypted " + "rooms (note: content is evaluated, see /help eval)"), + eval_cast, + ), ] network_options = [ From fd9053eacdc8128a8b5b2091a44177c823f0cd29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 3 Nov 2018 16:44:40 +0100 Subject: [PATCH 198/269] server: Support printing out sent messages before finishing the request. --- matrix/buffer.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++-- matrix/config.py | 34 ++++++++++++++++++++++++++++- matrix/server.py | 42 ++++++++++++++++++++++++++++++++++-- 3 files changed, 127 insertions(+), 5 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 142cb5f..9fe4506 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -57,6 +57,7 @@ OwnMessages = NamedTuple( ("sender", str), ("age", int), ("event_id", str), + ("uuid", str), ("room_id", str), ("formatted_message", Formatted), ], @@ -1242,7 +1243,10 @@ class RoomBuffer(object): # type: (OwnMessage) -> None nick = self.find_nick(self.room.own_user_id) data = message.formatted_message.to_weechat() - tags = [SCRIPT_NAME + "_id_{}".format(message.event_id)] + if message.event_id: + tags = [SCRIPT_NAME + "_id_{}".format(message.event_id)] + else: + tags = [SCRIPT_NAME + "_uuid_{}".format(message.uuid)] date = message.age self.weechat_buffer.self_message(nick, data, date, tags) @@ -1251,12 +1255,60 @@ class RoomBuffer(object): # type: (OwnMessage) -> None nick = self.find_nick(self.room.own_user_id) date = message.age - tags = [SCRIPT_NAME + "_id_{}".format(message.event_id)] + if message.event_id: + tags = [SCRIPT_NAME + "_id_{}".format(message.event_id)] + else: + tags = [SCRIPT_NAME + "_uuid_{}".format(message.uuid)] self.weechat_buffer.self_action( nick, message.formatted_message.to_weechat(), date, tags ) + @staticmethod + def _find_by_uuid_predicate(uuid, line): + uuid_tag = SCRIPT_NAME + "_uuid_{}".format(uuid) + tags = line.tags + + if uuid_tag in tags: + return True + return False + + def mark_message_as_unsent(self, uuid, _): + """Append to already printed lines that are greyed out an error + message""" + lines = self.weechat_buffer.find_lines( + partial(self._find_by_uuid_predicate, uuid) + ) + last_line = lines[-1] + + message = last_line.message + message += (" {del_color}<{ncolor}{error_color}Error sending " + "message{del_color}>{ncolor}").format( + del_color=W.color("chat_delimiters"), + ncolor=W.color("reset"), + error_color=W.color(G.CONFIG.color.error_message)) + + last_line.message = message + + def replace_printed_line_by_uuid(self, uuid, new_message): + """Replace already printed lines that are greyed out with real ones""" + lines = self.weechat_buffer.find_lines( + partial(self._find_by_uuid_predicate, uuid) + ) + + new_lines = new_message.formatted_message.to_weechat().split("\n") + + for i, line in enumerate(lines): + line.message = new_lines[i] + tags = line.tags + + new_tags = [ + tag for tag in tags + if not tag.startswith(SCRIPT_NAME + "_uuid_") + ] + new_tags.append(SCRIPT_NAME + "_id_ " + new_message.event_id) + line.tags = new_tags + def old_redacted(self, event): tags = [ SCRIPT_NAME + "_message", diff --git a/matrix/config.py b/matrix/config.py index c612686..4d69c76 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -453,6 +453,17 @@ class MatrixConfig(WeechatConfig): ("Reconnect to the server if the lag is greater than this " "value (in seconds)"), ), + Option( + "print_unconfirmed_messages", + "boolean", + "", + 0, + 0, + "on", + ("If off, messages are only printed after the server confirms" + "their receival. If on, messages are immediately printed but " + "colored differently until receival is confirmed."), + ), ] color_options = [ @@ -464,7 +475,28 @@ class MatrixConfig(WeechatConfig): 0, "lightgreen", ("Color for matrix style blockquotes"), - ) + ), + Option( + "error_message", + "color", + "", + 0, + 0, + "darkgray", + ("Color for error messages that appear inside a room buffer (" + "e.g. when a message errors out when sending or when a " + "message is redacted)"), + ), + Option( + "unconfirmed_message", + "color", + "", + 0, + 0, + "darkgray", + ("Color for messages that are printed out but the server " + "hasn't confirmed the that he received them."), + ), ] sections = [ diff --git a/matrix/server.py b/matrix/server.py index 5bed5e5..18b7943 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -61,7 +61,7 @@ from .globals import SCRIPT_NAME, SERVERS, W, MAX_EVENTS from .utf import utf8_decode from .utils import create_server_buffer, key_from_value, server_buffer_prnt -from .colors import Formatted +from .colors import Formatted, FormattedString, DEFAULT_ATRIBUTES try: @@ -248,6 +248,7 @@ class MatrixServer(object): self.device_deletion_queue = dict() self.own_message_queue = dict() # type: Dict[str, OwnMessage] + self.print_before_ack_queue = [] # type: List[UUID] self.encryption_queue = defaultdict(deque) \ # type: DefaultDict[str, Deque[EncrytpionQueueItem]] self.backlog_queue = dict() # type: Dict[str, str] @@ -719,11 +720,32 @@ class MatrixServer(object): message_class = OwnMessage own_message = message_class( - self.user_id, 0, "", room.room_id, formatted + self.user_id, 0, "", uuid, room.room_id, formatted ) self.own_message_queue[uuid] = own_message self.send_or_queue(request) + + if G.CONFIG.network.print_unconfirmed_messages: + self.print_before_ack_queue.append(uuid) + plain_message = formatted.to_weechat() + plain_message = W.string_remove_color(plain_message, "") + attributes = DEFAULT_ATRIBUTES.copy() + attributes["fgcolor"] = G.CONFIG.color.unconfirmed_message + new_formatted = Formatted([FormattedString( + plain_message, + attributes + )]) + + own_message = message_class( + self.user_id, 0, "", uuid, room.room_id, new_formatted + ) + + if isinstance(own_message, OwnAction): + room_buffer.self_action(own_message) + elif isinstance(own_message, OwnMessage): + room_buffer.self_message(own_message) + return True def keys_upload(self): @@ -758,11 +780,25 @@ class MatrixServer(object): server_buffer_prnt(self, pprint.pformat(message.request.payload)) server_buffer_prnt(self, pprint.pformat(message.response.body)) + def handle_own_messages_error(self, response): + message = self.own_message_queue.pop(response.uuid) + if response.uuid not in self.print_before_ack_queue: + return + + room_buffer = self.room_buffers[message.room_id] + room_buffer.mark_message_as_unsent(response.uuid, message) + def handle_own_messages(self, response): message = self.own_message_queue.pop(response.uuid) room_buffer = self.room_buffers[message.room_id] message = message._replace(event_id=response.event_id) + # We already printed the message, just modify it to contain the proper + # colors and formatting. + if response.uuid in self.print_before_ack_queue: + room_buffer.replace_printed_line_by_uuid(response.uuid, message) + return + if isinstance(message, OwnAction): room_buffer.self_action(message) return @@ -990,6 +1026,8 @@ class MatrixServer(object): elif isinstance(response, JoinedMembersError): self.rooms_with_missing_members.append(response.room_id) self.get_joined_members(self.rooms_with_missing_members.pop()) + elif isinstance(response, RoomSendResponse): + self.handle_own_messages_error(response) def handle_response(self, response): # type: (Response) -> None From 59b318bdce793a41688d79f03e1d5be4331692bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 5 Nov 2018 21:59:25 +0100 Subject: [PATCH 199/269] buffer: Handle messages containing a transaction id. --- matrix/buffer.py | 38 ++++++++++++++++++++++++++++++++++++++ matrix/server.py | 22 +++++++++++----------- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 9fe4506..c5564ac 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -21,6 +21,7 @@ import time from builtins import super from functools import partial from typing import Dict, List, NamedTuple, Optional, Set +from uuid import UUID from nio import ( Api, @@ -810,6 +811,9 @@ class RoomBuffer(object): self.unhandled_users = [] # type: List[str] self.inactive_users = [] + self.sent_messages_queue = dict() # type: Dict[UUID, OwnMessage] + self.printed_before_ack_queue = list() # type: List[UUID] + buffer_name = "{}.{}".format(server_name, room.room_id) # This dict remembers the connection from a user_id to the name we @@ -1081,6 +1085,36 @@ class RoomBuffer(object): elif isinstance(event, RoomEncryptionEvent): pass + def handle_own_message_in_timeline(self, event): + """Check if our own message is already printed if not print it. + This function is called for messages that contain a transaction id + indicating that they were sent out using our own client. If we sent out + a message but never got a valid server response (e.g. due to + disconnects) this function prints them out using data from the next + sync response""" + uuid = UUID(event.transaction_id) + message = self.sent_messages_queue.pop(uuid, None) + + # We already got a response to the room_send_message() API call and + # handled the message, no need to print it out again + if not message: + return + + message = message._replace(event_id=event.event_id) + if uuid in self.printed_before_ack_queue: + self.replace_printed_line_by_uuid( + event.transaction_id, + message + ) + self.printed_before_ack_queue.remove(uuid) + return + + if isinstance(message, OwnAction): + self.self_action(message) + elif isinstance(message, OwnMessage): + self.self_message(message) + return + def handle_timeline_event(self, event): # TODO this should be done for every messagetype that gets printed in # the buffer @@ -1095,6 +1129,10 @@ class RoomBuffer(object): self.add_user(event.sender, 0, True, True) + if event.transaction_id: + self.handle_own_message_in_timeline(event) + return + if isinstance(event, RoomMemberEvent): self.handle_membership_events(event, False) diff --git a/matrix/server.py b/matrix/server.py index 18b7943..178da3a 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -247,8 +247,6 @@ class MatrixServer(object): self.device_deletion_queue = dict() - self.own_message_queue = dict() # type: Dict[str, OwnMessage] - self.print_before_ack_queue = [] # type: List[UUID] self.encryption_queue = defaultdict(deque) \ # type: DefaultDict[str, Deque[EncrytpionQueueItem]] self.backlog_queue = dict() # type: Dict[str, str] @@ -723,11 +721,11 @@ class MatrixServer(object): self.user_id, 0, "", uuid, room.room_id, formatted ) - self.own_message_queue[uuid] = own_message + room_buffer.sent_messages_queue[uuid] = own_message self.send_or_queue(request) if G.CONFIG.network.print_unconfirmed_messages: - self.print_before_ack_queue.append(uuid) + room_buffer.printed_before_ack_queue.append(uuid) plain_message = formatted.to_weechat() plain_message = W.string_remove_color(plain_message, "") attributes = DEFAULT_ATRIBUTES.copy() @@ -781,22 +779,24 @@ class MatrixServer(object): server_buffer_prnt(self, pprint.pformat(message.response.body)) def handle_own_messages_error(self, response): - message = self.own_message_queue.pop(response.uuid) - if response.uuid not in self.print_before_ack_queue: + room_buffer = self.room_buffers[response.room_id] + + if response.uuid not in room_buffer.printed_before_ack_queue: return - room_buffer = self.room_buffers[message.room_id] + message = room_buffer.sent_messages_queue.pop(response.uuid) room_buffer.mark_message_as_unsent(response.uuid, message) + room_buffer.printed_before_ack_queue.remove(response.uuid) def handle_own_messages(self, response): - message = self.own_message_queue.pop(response.uuid) - room_buffer = self.room_buffers[message.room_id] + room_buffer = self.room_buffers[response.room_id] + message = room_buffer.sent_messages_queue.pop(response.uuid) message = message._replace(event_id=response.event_id) - # We already printed the message, just modify it to contain the proper # colors and formatting. - if response.uuid in self.print_before_ack_queue: + if response.uuid in room_buffer.printed_before_ack_queue: room_buffer.replace_printed_line_by_uuid(response.uuid, message) + room_buffer.printed_before_ack_queue.remove(response.uuid) return if isinstance(message, OwnAction): From 3f3c5f9fba1401bb25e9d1d127ec8b228adf6075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 6 Nov 2018 13:59:03 +0100 Subject: [PATCH 200/269] buffer: Add a localvar containing the room domain. --- matrix/buffer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index c5564ac..ecbb3a5 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -820,10 +820,21 @@ class RoomBuffer(object): # displayed in the buffer self.displayed_nicks = {} user = shorten_sender(self.room.own_user_id) + self.weechat_buffer = WeechatChannelBuffer( buffer_name, server_name, user ) + try: + _, room_domain = room.room_id.split(":", 1) + W.buffer_set( + self.weechat_buffer._ptr, + "localvar_set_domain", + room_domain + ) + except ValueError: + pass + @property def backlog_pending(self): return self._backlog_pending From 6422381ae50885b8f162184fd98ade5ad55085f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 6 Nov 2018 22:08:33 +0100 Subject: [PATCH 201/269] buffer: Add the room_id to a buffer localvar. --- matrix/buffer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index ecbb3a5..18de907 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -835,6 +835,12 @@ class RoomBuffer(object): except ValueError: pass + W.buffer_set( + self.weechat_buffer._ptr, + "localvar_set_room_id", + room.room_id + ) + @property def backlog_pending(self): return self._backlog_pending From aae57320183d4e4e3830ec3ef47f2ce1310d1504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 6 Nov 2018 22:09:10 +0100 Subject: [PATCH 202/269] buffer: Update the buffer name after a room name event in the state. --- matrix/buffer.py | 11 +++++++---- matrix/server.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 18de907..4d6c319 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -962,6 +962,9 @@ class RoomBuffer(object): self.weechat_buffer.invite(event.state_key, date) return + self.update_buffer_name() + + def update_buffer_name(self): room_name = self.room.display_name() self.weechat_buffer.short_name = room_name @@ -1099,6 +1102,8 @@ class RoomBuffer(object): self._handle_topic(event, True) elif isinstance(event, PowerLevelsEvent): self._handle_power_level(event) + elif isinstance(event, (RoomNameEvent, RoomAliasEvent)): + self.update_buffer_name() elif isinstance(event, RoomEncryptionEvent): pass @@ -1154,8 +1159,7 @@ class RoomBuffer(object): self.handle_membership_events(event, False) elif isinstance(event, (RoomNameEvent, RoomAliasEvent)): - room_name = self.room.display_name() - self.weechat_buffer.short_name = room_name + self.update_buffer_name() elif isinstance(event, RoomTopicEvent): self._handle_topic(event, False) @@ -1498,8 +1502,7 @@ class RoomBuffer(object): # We didn't handle all joined users, the room display name might still # be outdated because of that, update it now. if self.unhandled_users: - room_name = self.room.display_name() - self.weechat_buffer.short_name = room_name + self.update_buffer_name() def handle_left_room(self, info): self.joined = False diff --git a/matrix/server.py b/matrix/server.py index 178da3a..7fadc92 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1094,6 +1094,7 @@ class MatrixServer(object): room_buffer.unhandled_users += users self._hook_lazy_user_adding() room_buffer.members_fetched = True + room_buffer.update_buffer_name() # Fetch the users for the next room. if self.rooms_with_missing_members: From 1a796322f34548a1565908fb313b8817c4587caf Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Tue, 6 Nov 2018 18:25:56 +0100 Subject: [PATCH 203/269] Simplify Formatted.to_weechat logic. --- matrix/colors.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 89b578d..88df27e 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -266,36 +266,33 @@ class Formatted(object): def to_weechat(self): # TODO BG COLOR def add_attribute(string, name, value): - if name == "bold" and value: + if not value: + return string + elif name == "bold": return "{bold_on}{text}{bold_off}".format( bold_on=W.color("bold"), text=string, bold_off=W.color("-bold"), ) - - if name == "italic" and value: + elif name == "italic": return "{italic_on}{text}{italic_off}".format( italic_on=W.color("italic"), text=string, italic_off=W.color("-italic"), ) - - if name == "underline" and value: + elif name == "underline": return "{underline_on}{text}{underline_off}".format( underline_on=W.color("underline"), text=string, underline_off=W.color("-underline"), ) - - if name == "strikethrough" and value: + elif name == "strikethrough": return string_strikethrough(string) - - if name == "quote" and value: + elif name == "quote": return self.textwrapper.fill( W.string_remove_color(string.replace("\n", ""), "") ) - - if name == "code" and value: + elif name == "code": try: lexer = get_lexer_by_name(value) except ClassNotFound: @@ -304,22 +301,20 @@ class Formatted(object): # highlight adds a newline to the end of the string, remove it # from the output return highlight(string, lexer, WeechatFormatter())[:-1] - - if name == "fgcolor" and value: + elif name == "fgcolor": return "{color_on}{text}{color_off}".format( color_on=W.color(value), text=string, color_off=W.color("resetcolor"), ) - - if name == "bgcolor" and value: + elif name == "bgcolor": return "{color_on}{text}{color_off}".format( color_on=W.color("," + value), text=string, color_off=W.color("resetcolor"), ) - - return string + else: + return string def format_string(formatted_string): text = formatted_string.text From 560c8d0c54c7058cfa912ee2652e8cacf3f6b4fd Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 7 Nov 2018 12:05:46 +0100 Subject: [PATCH 204/269] Make pygments style configurable. --- matrix/colors.py | 10 ++++++++-- matrix/config.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 88df27e..41c2600 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -29,7 +29,7 @@ from typing import List import webcolors from pygments import highlight -from pygments.formatter import Formatter +from pygments.formatter import Formatter, get_style_by_name from pygments.lexers import get_lexer_by_name, guess_lexer from pygments.util import ClassNotFound @@ -298,9 +298,15 @@ class Formatted(object): except ClassNotFound: lexer = guess_lexer(string) + try: + style = get_style_by_name(G.CONFIG.look.pygments_style) + except ClassNotFound: + style = "native" + # highlight adds a newline to the end of the string, remove it # from the output - return highlight(string, lexer, WeechatFormatter())[:-1] + return highlight(string, lexer, + WeechatFormatter(style=style))[:-1] elif name == "fgcolor": return "{color_on}{text}{color_off}".format( color_on=W.color(value), diff --git a/matrix/config.py b/matrix/config.py index 4d69c76..b7ac695 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -357,6 +357,16 @@ class MatrixConfig(WeechatConfig): "rooms (note: content is evaluated, see /help eval)"), eval_cast, ), + Option( + "pygments_style", + "string", + "", + 0, + 0, + "native", + "Pygments style to use for highlighting source code blocks", + eval_cast, + ), ] network_options = [ From a74ac888a39cccc8e5e2db7e99304e12cc5ae81b Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 7 Nov 2018 12:57:32 +0100 Subject: [PATCH 205/269] Fix typo: DEFAULT_ATRIBUTES -> DEFAULT_ATTRIBUTES --- matrix/colors.py | 16 ++++++++-------- matrix/server.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 41c2600..261a5bd 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -62,7 +62,7 @@ class Formatted(object): def is_formatted(self): # type: (Formatted) -> bool for string in self.substrings: - if string.attributes != DEFAULT_ATRIBUTES: + if string.attributes != DEFAULT_ATTRIBUTES: return True return False @@ -76,7 +76,7 @@ class Formatted(object): """ text = "" # type: str substrings = [] # type: List[FormattedString] - attributes = DEFAULT_ATRIBUTES.copy() + attributes = DEFAULT_ATTRIBUTES.copy() i = 0 while i < len(line): @@ -165,7 +165,7 @@ class Formatted(object): substrings.append(FormattedString(text, attributes.copy())) text = "" # Reset all the attributes - attributes = DEFAULT_ATRIBUTES.copy() + attributes = DEFAULT_ATTRIBUTES.copy() i = i + 1 # Italic elif line[i] == "\x1D": @@ -371,7 +371,7 @@ class Formatted(object): # TODO this should be a typed dict. -DEFAULT_ATRIBUTES = { +DEFAULT_ATTRIBUTES = { "bold": False, "italic": False, "underline": False, @@ -390,7 +390,7 @@ class MatrixHtmlParser(HTMLParser): HTMLParser.__init__(self) self.text = "" # type: str self.substrings = [] # type: List[FormattedString] - self.attributes = DEFAULT_ATRIBUTES.copy() + self.attributes = DEFAULT_ATTRIBUTES.copy() def unescape(self, text): """Shim to unescape HTML in both Python 2 and 3. @@ -442,13 +442,13 @@ class MatrixHtmlParser(HTMLParser): if self.text: self.add_substring(self.text, self.attributes.copy()) self.text = "\n" - self.add_substring(self.text, DEFAULT_ATRIBUTES.copy()) + self.add_substring(self.text, DEFAULT_ATTRIBUTES.copy()) self.text = "" elif tag == "br": if self.text: self.add_substring(self.text, self.attributes.copy()) self.text = "\n" - self.add_substring(self.text, DEFAULT_ATRIBUTES.copy()) + self.add_substring(self.text, DEFAULT_ATTRIBUTES.copy()) self.text = "" elif tag == "font": for key, value in attrs: @@ -482,7 +482,7 @@ class MatrixHtmlParser(HTMLParser): elif tag == "blockquote": self._toggle_attribute("quote") self.text = "\n" - self.add_substring(self.text, DEFAULT_ATRIBUTES.copy()) + self.add_substring(self.text, DEFAULT_ATTRIBUTES.copy()) self.text = "" elif tag == "font": if self.text: diff --git a/matrix/server.py b/matrix/server.py index 7fadc92..78de798 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -61,7 +61,7 @@ from .globals import SCRIPT_NAME, SERVERS, W, MAX_EVENTS from .utf import utf8_decode from .utils import create_server_buffer, key_from_value, server_buffer_prnt -from .colors import Formatted, FormattedString, DEFAULT_ATRIBUTES +from .colors import Formatted, FormattedString, DEFAULT_ATTRIBUTES try: @@ -728,7 +728,7 @@ class MatrixServer(object): room_buffer.printed_before_ack_queue.append(uuid) plain_message = formatted.to_weechat() plain_message = W.string_remove_color(plain_message, "") - attributes = DEFAULT_ATRIBUTES.copy() + attributes = DEFAULT_ATTRIBUTES.copy() attributes["fgcolor"] = G.CONFIG.color.unconfirmed_message new_formatted = Formatted([FormattedString( plain_message, From caf74ceb1fe407e8f3cc91d552abb6516262a4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 7 Nov 2018 13:16:11 +0100 Subject: [PATCH 206/269] config: Add documentation. --- matrix/config.py | 97 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/matrix/config.py b/matrix/config.py index 4d69c76..34479bd 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -14,6 +14,15 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +"""weechat-matrix Configuration module. + +This module contains abstractions on top of weechats configuration files and +the main script configuration class. + +To add configuration options refer to MatrixConfig. +Server specific configuration options are handled in server.py +""" + from builtins import super from collections import namedtuple from enum import Enum, unique @@ -60,6 +69,12 @@ class Option( ], ) ): + """A class representing a new configuration option. + + An option object is consumed by the ConfigSection class adding + configuration options to weechat. + """ + __slots__ = () def __new__( @@ -74,6 +89,25 @@ class Option( cast=None, change_callback=None, ): + """ + Parameters: + name (str): Name of the configuration option + type (str): Type of the configuration option, can be one of the + supported weechat types: string, boolean, integer, color + string_values: (str): A list of string values that the option can + accept seprated by | + min (int): Minimal value of the option, only used if the type of + the option is integer + max (int): Maximal value of the option, only used if the type of + the option is integer + description (str): Description of the configuration option + cast (callable): A callable function taking a single value and + returning a modified value. Useful to turn the configuration + option into an enum while reading it. + change_callback(callable): A function that will be called + by weechat every time the configuration option is changed. + """ + return super().__new__( cls, name, @@ -94,6 +128,11 @@ def matrix_config_reload_cb(data, config_file): def change_log_level(category, level): + """Change the log level of the underlying nio lib + + Called every time the user changes the log level or log category + configuration option.""" + if category == "all": nio.logger_group.level = level elif category == "http": @@ -110,6 +149,10 @@ def change_log_level(category, level): @utf8_decode def config_server_buffer_cb(data, option): + """Callback for the look.server_buffer option. + Is called when the option is changed and merges/splits the server + buffer""" + for server in SERVERS.values(): server.buffer_merge() return 1 @@ -117,6 +160,7 @@ def config_server_buffer_cb(data, option): @utf8_decode def config_log_level_cb(data, option): + """Callback for the network.debug_level option.""" change_log_level( G.CONFIG.network.debug_category, G.CONFIG.network.debug_level ) @@ -125,6 +169,7 @@ def config_log_level_cb(data, option): @utf8_decode def config_log_category_cb(data, option): + """Callback for the network.debug_category option.""" change_log_level(G.CONFIG.debug_category, logbook.ERROR) G.CONFIG.debug_category = G.CONFIG.network.debug_category change_log_level( @@ -135,6 +180,8 @@ def config_log_category_cb(data, option): @utf8_decode def config_pgup_cb(data, option): + """Callback for the network.fetch_backlog_on_pgup option. + Enables or disables the hook that is run when /window page_up is called""" if G.CONFIG.network.fetch_backlog_on_pgup: if not G.CONFIG.page_up_hook: G.CONFIG.page_up_hook = W.hook_command_run( @@ -179,11 +226,28 @@ def logbook_category(value): def eval_cast(string): + """A function that passes a string to weechat which evaluates it using its + expression evaluation syntax. + Can only be used with strings, useful for passwords or options that contain + a formatted string to e.g. add colors. + More info here: + https://weechat.org/files/doc/stable/weechat_plugin_api.en.html#_string_eval_expression""" + return W.string_eval_expression(string, {}, {}, {}) class WeechatConfig(object): + """A class representing a weechat configuration file + Wraps weechats configuration creation functionality""" + def __init__(self, sections): + """Create a new weechat configuration file, expects the global + SCRIPT_NAME to be defined and a reload callback + + Parameters: + sections (List[Tuple[str, List[Option]]]): List of config sections + that will be created for the configuration file. + """ self._ptr = W.config_new( SCRIPT_NAME, SCRIPT_NAME + "_config_reload_cb", "" ) @@ -194,6 +258,8 @@ class WeechatConfig(object): setattr(self, name, section_class(name, self._ptr, options)) def free(self): + """Free all the config sections and their options as well as the + configuration file. Should be called when the script is unloaded.""" for section in [ getattr(self, a) for a in dir(self) @@ -204,6 +270,7 @@ class WeechatConfig(object): W.config_free(self._ptr) def read(self): + """Read the config file""" return_code = W.config_read(self._ptr) if return_code == W.WEECHAT_CONFIG_READ_OK: return True @@ -215,6 +282,9 @@ class WeechatConfig(object): class ConfigSection(object): + """A class representing a weechat config section. + Should not be used on its own, the WeechatConfig class uses this to build + config sections.""" @classmethod def build(cls, name, options): def constructor(self, name, config_ptr, options): @@ -268,6 +338,12 @@ class ConfigSection(object): @staticmethod def option_property(name, option_type, evaluate=False, cast_func=None): + """Create a property for this class that makes the reading of config + option values pythonic. The option will be available as a property with + the name of the option. + If a cast function was defined for the option the property will pass + the option value to the cast function and return its result.""" + def bool_getter(self): return bool(W.config_boolean(self._option_ptrs[name])) @@ -297,8 +373,27 @@ class ConfigSection(object): class MatrixConfig(WeechatConfig): - def __init__(self): + """Main matrix configuration file. + This class defines all the global matrix configuration options. + New global options should be added to the constructor of this class under + the appropriate section. + There are three main sections defined: + Look: This section is for options that change the way matrix messages + are shown or the way the buffers are shown. + Color: This section should mainly be for color options, options that + change color schemes or themes should go to the look section. + Network: This section is for options that change the way the script + behaves, e.g. the way it communicates with the server, it handles + responses or any other behavioural change that doesn't fit in the + previous sections. + + There is a special section called server defined which contains per server + configuration options. Server options aren't defined here, they need to be + added in server.py + """ + + def __init__(self): self.debug_buffer = "" self.debug_category = "all" self.page_up_hook = None From 9b6ca7886614d78c5a0b9504c32904811678f9dc Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 7 Nov 2018 13:24:22 +0100 Subject: [PATCH 207/269] Don't use eval_cast for pygments_style option. --- matrix/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/matrix/config.py b/matrix/config.py index b7ac695..76afb30 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -365,7 +365,6 @@ class MatrixConfig(WeechatConfig): 0, "native", "Pygments style to use for highlighting source code blocks", - eval_cast, ), ] From ef7b2b7eaf8f6f786133869e0b22cfdda15844a8 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 7 Nov 2018 13:31:33 +0100 Subject: [PATCH 208/269] Fix FormattedString test failure. Initialize FormattedString().attributes to DEFAULT_ATTRIBUTES so all the attributes are always present. FormattedString is now a class, not a namedtuple instance. --- matrix/colors.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/matrix/colors.py b/matrix/colors.py index 261a5bd..96a4c24 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -43,7 +43,13 @@ except ImportError: from html.parser import HTMLParser -FormattedString = namedtuple("FormattedString", ["text", "attributes"]) +class FormattedString: + __slots__ = ("text", "attributes") + + def __init__(self, text, attributes): + self.attributes = DEFAULT_ATTRIBUTES.copy() + self.attributes.update(attributes) + self.text = text class Formatted(object): From 2022151bd0f06f138d8c1bf3dec43f5affe6b5f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 7 Nov 2018 13:50:55 +0100 Subject: [PATCH 209/269] tests: Add initial buffer tests. --- matrix/_weechat.py | 6 ++++++ tests/buffer_test.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/buffer_test.py diff --git a/matrix/_weechat.py b/matrix/_weechat.py index 54cbe37..346b67f 100644 --- a/matrix/_weechat.py +++ b/matrix/_weechat.py @@ -154,6 +154,12 @@ def buffer_set(*_, **__): return +def buffer_get_string(_ptr, property): + if property == "localvar_type": + return "channel" + return "" + + def nicklist_add_group(*_, **__): return diff --git a/tests/buffer_test.py b/tests/buffer_test.py new file mode 100644 index 0000000..e146ac5 --- /dev/null +++ b/tests/buffer_test.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +from matrix.buffer import WeechatChannelBuffer + + +class TestClass(object): + def test_buffer(self): + b = WeechatChannelBuffer("test_buffer_name", "example.org", "alice") + assert b + + def test_buffer_print(self): + b = WeechatChannelBuffer("test_buffer_name", "example.org", "alice") + b.message("alice", "hello world", 0, 0) + assert b From a08400c9753069e441a02103986de523b1b47b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 9 Nov 2018 17:57:26 +0100 Subject: [PATCH 210/269] buffer: Don't put a space in the matrix_id tag. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 4d6c319..9f39ef5 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -1365,7 +1365,7 @@ class RoomBuffer(object): tag for tag in tags if not tag.startswith(SCRIPT_NAME + "_uuid_") ] - new_tags.append(SCRIPT_NAME + "_id_ " + new_message.event_id) + new_tags.append(SCRIPT_NAME + "_id_" + new_message.event_id) line.tags = new_tags def old_redacted(self, event): From 7e15b04f98a251c9a958e7230326bca794652ed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 12 Nov 2018 15:30:28 +0100 Subject: [PATCH 211/269] config: Add the lag_min_show option. --- matrix/bar_items.py | 2 +- matrix/config.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 34569e5..8c09f78 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -74,7 +74,7 @@ def matrix_bar_item_lag(data, item, window, buffer, extra_info): # pylint: disable=unused-argument for server in SERVERS.values(): if buffer in server.buffers.values() or buffer == server.server_buffer: - if server.lag >= 500: + if server.lag >= G.CONFIG.network.lag_min_show: color = W.color("irc.color.item_lag_counting") if server.lag_done: color = W.color("irc.color.item_lag_finished") diff --git a/matrix/config.py b/matrix/config.py index a126311..0c3da6a 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -568,6 +568,15 @@ class MatrixConfig(WeechatConfig): "their receival. If on, messages are immediately printed but " "colored differently until receival is confirmed."), ), + Option( + "lag_min_show", + "integer", + "", + 1, + 604800, + "500", + ("minimum lag to show (in milliseconds)"), + ), ] color_options = [ From 6f29d1154d0d0df99eb28ae6e836b65417baaec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 12 Nov 2018 15:30:56 +0100 Subject: [PATCH 212/269] server: Add some type definitions. --- matrix/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 78de798..661933b 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -245,7 +245,7 @@ class MatrixServer(object): self.send_buffer = b"" # type: bytes self.device_check_timestamp = None # type: Optional[int] - self.device_deletion_queue = dict() + self.device_deletion_queue = dict() # type: Dict[str, str] self.encryption_queue = defaultdict(deque) \ # type: DefaultDict[str, Deque[EncrytpionQueueItem]] @@ -257,8 +257,8 @@ class MatrixServer(object): self.lazy_load_hook = None # type: Optional[str] self.partial_sync_hook = None # type: Optional[str] - self.keys_claimed = defaultdict(bool) - self.group_session_shared = defaultdict(bool) + self.keys_claimed = defaultdict(bool) # type: Dict[str, bool] + self.group_session_shared = defaultdict(bool) # type: Dict[str, bool] self.config = ServerConfig(self.name, config_ptr) self._create_session_dir() From a638ef3e5fab9e2b03c5ce310e3d2da96e5d1408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 12 Nov 2018 15:31:23 +0100 Subject: [PATCH 213/269] server: Remove some debug output. --- matrix/server.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 661933b..14e018d 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1200,11 +1200,8 @@ class MatrixServer(object): break for user_id in removed_user_ids: - W.prnt("", "Garbage collected {}".format(user_id)) del room_buffer.displayed_nicks[user_id] - pass - def buffer_merge(self): if not self.server_buffer: return From 0e3c0724e0d9cad4f1cbebf8a596c2c4992e0981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 12 Nov 2018 15:31:45 +0100 Subject: [PATCH 214/269] globals: Increase MAX_EVENTS. --- matrix/globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/globals.py b/matrix/globals.py index 8f2b6bb..d8b3e66 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -39,4 +39,4 @@ SERVERS = dict() # type: Dict[str, MatrixServer] CONFIG = None # type: Optional[MatrixConfig] ENCRYPTION = True # type: bool SCRIPT_NAME = "matrix" # type: str -MAX_EVENTS = 10 +MAX_EVENTS = 100 From d93184f2ed520e450608b611d8fff9844fc42c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 12 Nov 2018 15:32:59 +0100 Subject: [PATCH 215/269] server: Don't delay consecutive syncs by two seconds. --- matrix/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index 14e018d..23589f3 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1357,7 +1357,7 @@ def matrix_timer_cb(server_name, remaining_calls): server.disconnect() return W.WEECHAT_RC_OK - if server.sync_time and current_time > (server.sync_time + 2): + if server.sync_time and current_time > server.sync_time: timeout = 0 if server.transport_type == TransportType.HTTP else 30000 sync_filter = { "room": { From c1037620facd109a292fdc88abdd5f048869fcc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Nov 2018 20:07:39 +0100 Subject: [PATCH 216/269] bar_items: Make the busy sign configurable. --- matrix/bar_items.py | 4 +++- matrix/config.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 8c09f78..150540b 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -113,7 +113,9 @@ def matrix_bar_item_buffer_modes(data, item, window, buffer, extra_info): modes.append("❌") if room_buffer.backlog_pending or server.busy: - modes.append("⏳") + modes.append( + G.CONFIG.look.busy_sign + ) return "".join(modes) diff --git a/matrix/config.py b/matrix/config.py index 0c3da6a..0764ea4 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -452,6 +452,18 @@ class MatrixConfig(WeechatConfig): "rooms (note: content is evaluated, see /help eval)"), eval_cast, ), + Option( + "busy_sign", + "string", + "", + 0, + 0, + "⏳", + ("A sign that is used to signal that the client is busy e.g. " + "when the room backlog is fetching" + " (note: content is evaluated, see /help eval)"), + eval_cast, + ), Option( "pygments_style", "string", From 14f4da0708920c96e3a7b6a30842c68dccf61626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Nov 2018 22:05:56 +0100 Subject: [PATCH 217/269] bar_items: Make the encrypted room sign configurable. --- matrix/bar_items.py | 4 +++- matrix/config.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 150540b..7260891 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -101,7 +101,9 @@ def matrix_bar_item_buffer_modes(data, item, window, buffer, extra_info): modes = [] if room.encrypted: - modes.append("🔐") + modes.append( + G.CONFIG.look.encrypted_room_sign + ) if (server.client and server.client.room_contains_unverified(room.room_id)): diff --git a/matrix/config.py b/matrix/config.py index 0764ea4..1dd2c4b 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -464,6 +464,18 @@ class MatrixConfig(WeechatConfig): " (note: content is evaluated, see /help eval)"), eval_cast, ), + Option( + "encrypted_room_sign", + "string", + "", + 0, + 0, + "🔐", + ("A sign that is used to show that the current room is " + "encrypted " + "(note: content is evaluated, see /help eval)"), + eval_cast, + ), Option( "pygments_style", "string", From e4377c6d1afb2a6ac9d6b71e8db8f6c30adb5e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 15 Nov 2018 10:35:00 +0100 Subject: [PATCH 218/269] bar_items: Make the disconnect sign configurable. --- matrix/bar_items.py | 14 ++++---------- matrix/config.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 7260891..87d94c6 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -101,23 +101,17 @@ def matrix_bar_item_buffer_modes(data, item, window, buffer, extra_info): modes = [] if room.encrypted: - modes.append( - G.CONFIG.look.encrypted_room_sign - ) + modes.append(G.CONFIG.look.encrypted_room_sign) if (server.client and server.client.room_contains_unverified(room.room_id)): - modes.append( - G.CONFIG.look.encryption_warning_sign, - ) + modes.append(G.CONFIG.look.encryption_warning_sign) if not server.connected: - modes.append("❌") + modes.append(G.CONFIG.look.disconnect_sign) if room_buffer.backlog_pending or server.busy: - modes.append( - G.CONFIG.look.busy_sign - ) + modes.append(G.CONFIG.look.busy_sign) return "".join(modes) diff --git a/matrix/config.py b/matrix/config.py index 1dd2c4b..d842a9c 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -476,6 +476,17 @@ class MatrixConfig(WeechatConfig): "(note: content is evaluated, see /help eval)"), eval_cast, ), + Option( + "disconnect_sign", + "string", + "", + 0, + 0, + "❌", + ("A sign that is used to show that the server is disconnected " + "(note: content is evaluated, see /help eval)"), + eval_cast, + ), Option( "pygments_style", "string", From 58bf19791302b16f5b4b60fe7495f62ecef0e160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 15 Nov 2018 17:18:49 +0100 Subject: [PATCH 219/269] buffer: Fix the editing of actions for already printed lines. --- matrix/buffer.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 9f39ef5..cf2673f 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -558,7 +558,7 @@ class WeechatChannelBuffer(object): user.update_speaking_time(date) self.unmask_smart_filtered_nick(nick) - def _print_action(self, user, message, date, tags, extra_prefix=""): + def _format_action(self, user, message): nick_prefix = ( "" if not user.prefix @@ -570,16 +570,22 @@ class WeechatChannelBuffer(object): ) data = ( - "{extra_prefix}{prefix}{nick_prefix}{nick_color}{author}" + "{nick_prefix}{nick_color}{author}" "{ncolor} {msg}").format( - extra_prefix=extra_prefix, - prefix=W.prefix("action"), nick_prefix=nick_prefix, nick_color=W.color(user.color), author=user.nick, ncolor=W.color("reset"), msg=message, ) + return data + + def _print_action(self, user, message, date, tags, extra_prefix=""): + data = self._format_action(user, message) + data = "{extra_prefix}{prefix}{data}".format( + extra_prefix=extra_prefix, + prefix=W.prefix("action"), + data=data) self.print_date_tags(data, date, tags) @@ -1355,7 +1361,16 @@ class RoomBuffer(object): partial(self._find_by_uuid_predicate, uuid) ) - new_lines = new_message.formatted_message.to_weechat().split("\n") + if isinstance(new_message, OwnAction): + displayed_nick = self.displayed_nicks[self.room.own_user_id] + user = self.weechat_buffer._get_user(displayed_nick) + data = self.weechat_buffer._format_action( + user, + new_message.formatted_message.to_weechat() + ) + new_lines = data.split("\n") + else: + new_lines = new_message.formatted_message.to_weechat().split("\n") for i, line in enumerate(lines): line.message = new_lines[i] From 901a392f5a9ba9ed22833fb6bea250df3b1e4507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 15 Nov 2018 17:19:21 +0100 Subject: [PATCH 220/269] server: Don't try to fetch members if the server is disconnected. --- matrix/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/matrix/server.py b/matrix/server.py index 23589f3..181136a 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -756,6 +756,8 @@ class MatrixServer(object): self.send_or_queue(request) def get_joined_members(self, room_id): + if not self.connected: + return if room_id in self.member_request_list: return From 7bd0c96c578300e1863b15e9409747a52999390e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 15 Nov 2018 17:19:55 +0100 Subject: [PATCH 221/269] server: Remove special case for sync request lag calculation. This is now handled in nio. --- matrix/server.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 181136a..bff5536 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1034,13 +1034,6 @@ class MatrixServer(object): def handle_response(self, response): # type: (Response) -> None self.lag = response.elapsed * 1000 - - # If the response was a sync response and contained a timeout the - # timeout is expected and should be removed from the lag. - # TODO the timeout isn't a constant - if isinstance(response, SyncResponse): - self.lag = max(0, self.lag - (30000)) - self.lag_done = True W.bar_item_update("lag") From b9e60d6ecf9b330973405356483f1cbe66626ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 21 Nov 2018 17:03:38 +0100 Subject: [PATCH 222/269] bar_items: Use the room member_count method for nicklist_count. --- matrix/bar_items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 87d94c6..b6ea602 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -125,7 +125,7 @@ def matrix_bar_nicklist_count(data, item, window, buffer, extra_info): if buffer in server.buffers.values(): room_buffer = server.find_room_from_ptr(buffer) room = room_buffer.room - return str(len(room.users)) + return str(room.member_count) return "" From 47e71ce92577008649fdb100d50d7f2afba35e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 21 Nov 2018 19:17:34 +0100 Subject: [PATCH 223/269] buffer: Add the session id to the line tags. --- matrix/buffer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index cf2673f..bbf42f3 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -1083,6 +1083,10 @@ class RoomBuffer(object): tags = [SCRIPT_NAME + "_id_{}".format(event.event_id)] if event.sender_key: tags.append(SCRIPT_NAME + "_senderkey_{}".format(event.sender_key)) + if event.session_id: + tags.append(SCRIPT_NAME + "_session_id_{}".format( + event.session_id + )) return tags From d802aa30ae69d51a2151b9a688531f2a4e44b0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 21 Nov 2018 19:17:57 +0100 Subject: [PATCH 224/269] server: Send a hsignal when a new room key is received. --- matrix/server.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/matrix/server.py b/matrix/server.py index bff5536..d4e2d22 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -52,6 +52,7 @@ from nio import ( LoginError, JoinedMembersResponse, JoinedMembersError, + RoomKeyEvent ) from . import globals as G @@ -964,6 +965,20 @@ class MatrixServer(object): self._handle_room_info(response) + for event in response.to_device_events: + if not isinstance(event, RoomKeyEvent): + continue + + message = { + "sender": event.sender, + "sender_key": event.sender_key, + "room_id": event.room_id, + "session_id": event.session_id, + "algorithm": event.algorithm, + "server": self.name, + } + W.hook_hsignal_send("matrix_room_key_received", message) + # Full sync response handle everything. if isinstance(response, SyncResponse): if self.client.should_upload_keys: From bdb7f4509e557d86ac3233a5a22f5356c25c7f09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 21 Nov 2018 19:18:20 +0100 Subject: [PATCH 225/269] buffer: Remove dead code. --- matrix/buffer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index bbf42f3..a796c44 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -354,9 +354,6 @@ class WeechatChannelBuffer(object): W.buffer_set(self._ptr, "localvar_set_server", server_name) - # short_name = strip_matrix_server(room_id) - # W.buffer_set(self._ptr, "short_name", short_name) - W.nicklist_add_group( self._ptr, "", "000|o", "weechat.color.nicklist_group", 1 ) From a0199109be258fa5ae1664c212c1949a5163645d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 22 Nov 2018 13:52:09 +0100 Subject: [PATCH 226/269] buffer: Store undecrytped events so we can try to decrypt them later. --- matrix/buffer.py | 41 +++++++++++++++++++++++++++++++++++++++++ matrix/server.py | 26 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index a796c44..b6a5e52 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -20,6 +20,7 @@ from __future__ import unicode_literals import time from builtins import super from functools import partial +from collections import deque from typing import Dict, List, NamedTuple, Optional, Set from uuid import UUID @@ -816,6 +817,7 @@ class RoomBuffer(object): self.sent_messages_queue = dict() # type: Dict[UUID, OwnMessage] self.printed_before_ack_queue = list() # type: List[UUID] + self.undecrypted_events = deque(maxlen=5000) buffer_name = "{}.{}".format(server_name, room.room_id) @@ -1300,6 +1302,8 @@ class RoomBuffer(object): self.get_event_tags(event) + [session_id_tag] ) + self.undecrypted_events.append(event) + else: W.prnt( "", "Unhandled event of type {}.".format(type(event).__name__) @@ -1384,6 +1388,43 @@ class RoomBuffer(object): new_tags.append(SCRIPT_NAME + "_id_" + new_message.event_id) line.tags = new_tags + def replace_undecrypted_line(self, event): + """Find a undecrypted message in the buffer and replace it with the now + decrypted event.""" + # TODO different messages need different formatting + # To implement this, refactor out the different formatting code + # snippets to a Formatter class and reuse them here. + if not isinstance(event, RoomMessageText): + return + + def predicate(event_id, line): + event_tag = SCRIPT_NAME + "_id_{}".format(event_id) + if event_tag in line.tags: + return True + return False + + lines = self.weechat_buffer.find_lines( + partial(predicate, event.event_id) + ) + + if not lines: + return + + formatted = None + if event.formatted_body: + formatted = Formatted.from_html(event.formatted_body) + + data = formatted.to_weechat() if formatted else event.body + # TODO this isn't right if the data has multiple lines, that is + # everything is printed on a signle line and newlines are shown as a + # space. + # Weechat should support deleting lines and printing new ones at an + # arbitrary position. + # To implement this without weechat support either only handle single + # line messages or edit the first line in place, print new ones at the + # bottom and sort the buffer lines. + lines[0].message = data + def old_redacted(self, event): tags = [ SCRIPT_NAME + "_message", diff --git a/matrix/server.py b/matrix/server.py index d4e2d22..2cdd3ee 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -957,6 +957,28 @@ class MatrixServer(object): "matrix_load_users_cb", self.name) self.lazy_load_hook = hook + def decrypt_printed_messages(self, key_event): + """Decrypt already printed messages and send them to the buffer""" + try: + room_buffer = self.find_room_from_id(key_event.room_id) + except KeyError: + return + + decrypted_events = [] + + for undecrypted_event in room_buffer.undecrypted_events: + if undecrypted_event.session_id != key_event.session_id: + continue + + event = self.client.decrypt_event(undecrypted_event) + if event: + decrypted_events.append((undecrypted_event, event)) + + for event_pair in decrypted_events: + undecrypted_event, event = event_pair + room_buffer.undecrypted_events.remove(undecrypted_event) + room_buffer.replace_undecrypted_line(event) + def _handle_sync(self, response): # we got the same batch again, nothing to do if self.next_batch == response.next_batch: @@ -979,6 +1001,10 @@ class MatrixServer(object): } W.hook_hsignal_send("matrix_room_key_received", message) + # TODO try to decrypt some cached undecrypted messages with the + # new key + # self.decrypt_printed_messages(event) + # Full sync response handle everything. if isinstance(response, SyncResponse): if self.client.should_upload_keys: From b392989a5e3d3af2073a4ae762657345b6c803d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 23 Nov 2018 18:36:43 +0100 Subject: [PATCH 227/269] server: Remove unused imports. --- matrix/server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index 2cdd3ee..f2b4614 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -40,10 +40,8 @@ from nio import ( UpdateDeviceResponse, DeleteDevicesAuthResponse, DeleteDevicesResponse, - TransportResponse, TransportType, RoomMessagesResponse, - RequestType, EncryptionError, GroupEncryptionError, OlmTrustError, From c8968020a9ea86a8acbe01f058a3574d6905ae22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 23 Nov 2018 18:39:03 +0100 Subject: [PATCH 228/269] Update gitignore. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6d3c6dc..68aad69 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ .hypothesis/ .mypy_cache/ .pytest_cache/ +.ropeproject +.coverage From 1298a2c910a50355d0d10dcf764ab208e7ded562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 29 Nov 2018 15:54:13 +0100 Subject: [PATCH 229/269] server: Initial typing notice sending support. --- main.py | 22 +++++++++++++++++++++- matrix/buffer.py | 40 +++++++++++++++++++++++++++++++++++++++- matrix/globals.py | 1 + matrix/server.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 105 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index b683acd..e2501e7 100644 --- a/main.py +++ b/main.py @@ -491,6 +491,25 @@ def lazy_fetch_members_signal(_, _signal, buffer_ptr): return W.WEECHAT_RC_OK +def typing_notification_cb(data, signal, buffer_ptr): + """Send out typing notifications if the user is typing. + + This function is called every time the input text is changed. + It checks if we are on a buffer we own, and if we are sends out a typing + notification if the room is configured to send them out. + """ + for server in SERVERS.values(): + room_buffer = server.find_room_from_ptr(buffer_ptr) + if room_buffer: + server.room_send_typing_notice(room_buffer) + return W.WEECHAT_RC_OK + + if buffer_ptr == server.server_buffer: + return W.WEECHAT_RC_OK + + return W.WEECHAT_RC_OK + + if __name__ == "__main__": if W.register(WEECHAT_SCRIPT_NAME, WEECHAT_SCRIPT_AUTHOR, WEECHAT_SCRIPT_VERSION, WEECHAT_SCRIPT_LICENSE, @@ -513,7 +532,8 @@ if __name__ == "__main__": init_bar_items() init_completion() - hook = W.hook_signal("buffer_switch", "lazy_fetch_members_signal", "") + W.hook_signal("buffer_switch", "lazy_fetch_members_signal", "") + W.hook_signal("input_text_changed", "typing_notification_cb", "") if not SERVERS: create_default_server(G.CONFIG) diff --git a/matrix/buffer.py b/matrix/buffer.py index b6a5e52..e3047ae 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -49,7 +49,7 @@ from nio import ( from . import globals as G from .colors import Formatted from .config import RedactType -from .globals import SCRIPT_NAME, SERVERS, W +from .globals import SCRIPT_NAME, SERVERS, W, TYPING_NOTICE_TIMEOUT from .utf import utf8_decode from .utils import server_ts_to_weechat, shorten_sender, string_strikethrough @@ -414,6 +414,12 @@ class WeechatChannelBuffer(object): self.remove_smart_filtered_nick(nick) + @property + def input(self): + # type: () -> str + """Get the bar item input text of the buffer.""" + return W.buffer_get_string(self._ptr, "input") + @property def lines(self): own_lines = W.hdata_pointer(self._hdata, self._ptr, "own_lines") @@ -819,6 +825,9 @@ class RoomBuffer(object): self.printed_before_ack_queue = list() # type: List[UUID] self.undecrypted_events = deque(maxlen=5000) + self.typing_notice_time = None + self._typing = False + buffer_name = "{}.{}".format(server_name, room.room_id) # This dict remembers the connection from a user_id to the name we @@ -860,6 +869,35 @@ class RoomBuffer(object): def warning_prefix(self): return "⚠️ " + @property + def typing(self): + # type: () -> bool + """Return our typing status.""" + return self._typing + + @typing.setter + def typing(self, value): + self._typing = value + if value: + self.typing_notice_time = time.time() + else: + self.typing_notice_time = None + + @property + def typing_notice_expired(self): + # type: () -> bool + """Check if the typing notice has expired. + + Returns true if a new typing notice should be sent. + """ + if not self.typing_notice_time: + return True + + now = time.time() + if (now - self.typing_notice_time) > (TYPING_NOTICE_TIMEOUT / 1000): + return True + return False + def find_nick(self, user_id): # type: (str) -> str """Find a suitable nick from a user_id""" diff --git a/matrix/globals.py b/matrix/globals.py index d8b3e66..3559ab8 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -40,3 +40,4 @@ CONFIG = None # type: Optional[MatrixConfig] ENCRYPTION = True # type: bool SCRIPT_NAME = "matrix" # type: str MAX_EVENTS = 100 +TYPING_NOTICE_TIMEOUT = 4000 # 4 seconds typing notice lifetime diff --git a/matrix/server.py b/matrix/server.py index f2b4614..0b60c77 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -56,7 +56,7 @@ from nio import ( from . import globals as G from .buffer import OwnAction, OwnMessage, RoomBuffer from .config import ConfigSection, Option, ServerBufferType -from .globals import SCRIPT_NAME, SERVERS, W, MAX_EVENTS +from .globals import SCRIPT_NAME, SERVERS, W, MAX_EVENTS, TYPING_NOTICE_TIMEOUT from .utf import utf8_decode from .utils import create_server_buffer, key_from_value, server_buffer_prnt @@ -673,6 +673,49 @@ class MatrixServer(object): self.backlog_queue[uuid] = room_id self.send_or_queue(request) + def room_send_typing_notice(self, room_buffer): + """Send a typing notice for the provided room. + + Args: + room_buffer(RoomBuffer): the room for which the typing notice needs + to be sent. + """ + if not self.connected: + return + + input = room_buffer.weechat_buffer.input + + # Don't send a typing notice if the user is typing in a weechat command + if input.startswith("/") and not input.startswith("//"): + return + + # Don't send a typing notice if we only typed a couple of letters. + elif len(input) < 4 and not room_buffer.typing: + return + + # If we were typing already and our input bar now has no letters or + # only a couple of letters stop the typing notice. + elif len(input) < 4: + _, request = self.client.room_typing( + room_buffer.room.room_id, + typing_state=False) + room_buffer.typing = False + self.send(request) + return + + # Don't send out a typing notice if we already sent one out and it + # didn't expire yet. + if not room_buffer.typing_notice_expired: + return + + _, request = self.client.room_typing( + room_buffer.room.room_id, + typing_state=True, + timeout=TYPING_NOTICE_TIMEOUT) + + room_buffer.typing = True + self.send(request) + def room_send_message( self, room_buffer, # type: RoomBuffer From 45ebb921a94fc66b0a194840ebb438ed5c978084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 29 Nov 2018 15:54:43 +0100 Subject: [PATCH 230/269] bar_items: Dont' display own user as typing. --- matrix/bar_items.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index b6ea602..96d8eba 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -146,9 +146,15 @@ def matrix_bar_typing_notices_cb(data, item, window, buffer, extra_info): nicks = [] for user_id in room.typing_users: + if user_id == room.own_user_id: + continue + nick = room_buffer.displayed_nicks.get(user_id, user_id) nicks.append(nick) + if not nicks: + return "" + msg = "{}{}".format( G.CONFIG.look.bar_item_typing_notice_prefix, ", ".join(sorted(nicks)) From 624ecc4d7708b5feacb11a7c71252b14c5d7e1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 29 Nov 2018 16:48:12 +0100 Subject: [PATCH 231/269] config: Add typing notice conditions. --- matrix/buffer.py | 1 + matrix/config.py | 13 +++++++++++++ matrix/server.py | 10 ++++++++++ 3 files changed, 24 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index e3047ae..da7dca7 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -827,6 +827,7 @@ class RoomBuffer(object): self.typing_notice_time = None self._typing = False + self.typing_enabled = True buffer_name = "{}.{}".format(server_name, room.room_id) diff --git a/matrix/config.py b/matrix/config.py index d842a9c..8f80bb9 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -612,6 +612,19 @@ class MatrixConfig(WeechatConfig): "500", ("minimum lag to show (in milliseconds)"), ), + Option( + "typing_notice_conditions", + "string", + "", + 0, + 0, + "${typing_enabled}", + ("conditions to send typing notifications (note: content is " + "evaluated, see /help eval); besides the buffer and window " + "variables the typing_enabled variable is also expanded; " + "the typing_enabled variable can be manipulated with the " + "/room command, see /help room"), + ), ] color_options = [ diff --git a/matrix/server.py b/matrix/server.py index 0b60c77..657a94b 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -685,6 +685,16 @@ class MatrixServer(object): input = room_buffer.weechat_buffer.input + typing_enabled = bool(int(W.string_eval_expression( + G.CONFIG.network.typing_notice_conditions, + {}, + {"typing_enabled": str(int(room_buffer.typing_enabled))}, + {"type": "condition"} + ))) + + if not typing_enabled: + return + # Don't send a typing notice if the user is typing in a weechat command if input.startswith("/") and not input.startswith("//"): return From 66765f96d8b7792e37cc1b3785680922512a598f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 29 Nov 2018 16:48:51 +0100 Subject: [PATCH 232/269] buffer: Small documentation fix. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index da7dca7..2d0f849 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -901,7 +901,7 @@ class RoomBuffer(object): def find_nick(self, user_id): # type: (str) -> str - """Find a suitable nick from a user_id""" + """Find a suitable nick from a user_id.""" if user_id in self.displayed_nicks: return self.displayed_nicks[user_id] From 199e6de2a7269d597a13ee0ae0286c9489cc9307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 29 Nov 2018 17:09:21 +0100 Subject: [PATCH 233/269] commands: Add initial room command. --- main.py | 3 ++- matrix/commands.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index e2501e7..2cd7bd3 100644 --- a/main.py +++ b/main.py @@ -52,7 +52,8 @@ from matrix.commands import (hook_commands, hook_page_up, matrix_join_command_cb, matrix_kick_command_cb, matrix_me_command_cb, matrix_part_command_cb, matrix_redact_command_cb, matrix_topic_command_cb, - matrix_olm_command_cb, matrix_devices_command_cb) + matrix_olm_command_cb, matrix_devices_command_cb, + matrix_room_command_cb) from matrix.completion import (init_completion, matrix_command_completion_cb, matrix_debug_completion_cb, matrix_message_completion_cb, diff --git a/matrix/commands.py b/matrix/commands.py index 7ab5edf..9350b67 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -141,6 +141,18 @@ class WeechatCommandParser(object): return WeechatCommandParser._run_parser(parser, args) + @staticmethod + def room(args): + parser = WeechatArgParse(prog="room") + subparsers = parser.add_subparsers(dest="subcommand") + typing_notification = subparsers.add_parser("typing-notifications") + typing_notification.add_argument( + "state", + choices=["enable", "disable", "toggle"] + ) + + return WeechatCommandParser._run_parser(parser, args) + def grouper(iterable, n, fillvalue=None): "Collect data into fixed-length chunks or blocks" @@ -359,6 +371,22 @@ def hook_commands(): 'matrix_olm_command_cb', '') + W.hook_command( + # Command name and short description + "room", + "change room state", + # Synopsis + ("typing-notifications " + ), + # Description + ("state: one of enable, disable or toggle\n"), + # Completions + ("typing-notifications enable|disable|toggle"), + # Callback + "matrix_room_command_cb", + "", + ) + W.hook_command_run("/buffer clear", "matrix_command_buf_clear_cb", "") if G.CONFIG.network.fetch_backlog_on_pgup: @@ -857,6 +885,39 @@ def matrix_invite_command_cb(data, buffer, args): return W.WEECHAT_RC_OK +@utf8_decode +def matrix_room_command_cb(data, buffer, args): + parsed_args = WeechatCommandParser.room(args) + if not parsed_args: + return W.WEECHAT_RC_OK + + for server in SERVERS.values(): + if buffer == server.server_buffer: + server.error( + 'command "room" must be ' "executed on a Matrix room buffer" + ) + return W.WEECHAT_RC_OK + + room = server.find_room_from_ptr(buffer) + if not room: + continue + + if not parsed_args.subcommand or parsed_args.subcommand == "list": + server.error("command no subcommand found") + return W.WEECHAT_RC_OK + + if parsed_args.subcommand == "typing-notifications": + if parsed_args.state == "enable": + room.typing_enabled = True + elif parsed_args.state == "disable": + room.typing_enabled = False + elif parsed_args.state == "toggle": + room.typing_enabled = not room.typing_enabled + break + + return W.WEECHAT_RC_OK + + @utf8_decode def matrix_kick_command_cb(data, buffer, args): parsed_args = WeechatCommandParser.kick(args) From be0aae605f1a15625ae7b1b459f1d152cacaf01d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 30 Nov 2018 12:50:39 +0100 Subject: [PATCH 234/269] buffer: Use the display name for slack users as well. --- matrix/buffer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 2d0f849..5ba2f03 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -942,7 +942,8 @@ class RoomBuffer(object): # TODO handle this special case for discord bridge users and # freenode bridge users better - if user.user_id.startswith("@_discord_"): + if (user.user_id.startswith("@_discord_") or + user.user_id.startswith("@_slack_")): if user.display_name: short_name = user.display_name elif user.user_id.startswith("@freenode_"): From 33a96485eebee6a3c2ee1b79f505e6b6c9ad3274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 30 Nov 2018 12:53:01 +0100 Subject: [PATCH 235/269] buffer: Limit the display name to 50 characters. --- matrix/buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 5ba2f03..c70a875 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -945,7 +945,7 @@ class RoomBuffer(object): if (user.user_id.startswith("@_discord_") or user.user_id.startswith("@_slack_")): if user.display_name: - short_name = user.display_name + short_name = user.display_name[0:50] elif user.user_id.startswith("@freenode_"): short_name = shorten_sender(user.user_id[9:]) From 9d3b624733b49202dc48d9b18bfebc35dd4534a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 3 Dec 2018 22:28:49 +0100 Subject: [PATCH 236/269] server: Add support for sending out read markers. --- main.py | 19 +++++++++++++------ matrix/buffer.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ matrix/commands.py | 22 ++++++++++++++++++++-- matrix/config.py | 13 +++++++++++++ matrix/server.py | 20 ++++++++++++++++++++ 5 files changed, 111 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 2cd7bd3..69f420d 100644 --- a/main.py +++ b/main.py @@ -463,12 +463,16 @@ class WeechatHandler(StreamHandler): W.prnt(buf, item) -def lazy_fetch_members_signal(_, _signal, buffer_ptr): - """ Fetch room members on a buffer switch signal. +def buffer_switch_cb(_, _signal, buffer_ptr): + """Do some buffer operations when we switch buffers. + This function is called every time we switch a buffer. The pointer of - the new buffer is given to us by weechat. If it is one of our room buffers - we check if the members for the room aren't fetched and fetch them now if - they aren't. + the new buffer is given to us by weechat. + + If it is one of our room buffers we check if the members for the room + aren't fetched and fetch them now if they aren't. + + Read receipts are send out from here as well. """ for server in SERVERS.values(): if buffer_ptr == server.server_buffer: @@ -481,6 +485,9 @@ def lazy_fetch_members_signal(_, _signal, buffer_ptr): if not room_buffer: continue + if room_buffer.should_send_read_marker: + server.room_send_read_marker(room_buffer) + if room_buffer.members_fetched: return W.WEECHAT_RC_OK @@ -533,7 +540,7 @@ if __name__ == "__main__": init_bar_items() init_completion() - W.hook_signal("buffer_switch", "lazy_fetch_members_signal", "") + W.hook_signal("buffer_switch", "buffer_switch_cb", "") W.hook_signal("input_text_changed", "typing_notification_cb", "") if not SERVERS: diff --git a/matrix/buffer.py b/matrix/buffer.py index c70a875..78b1c9f 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -829,6 +829,9 @@ class RoomBuffer(object): self._typing = False self.typing_enabled = True + self.last_read_event = None + self._read_markers_enabled = True + buffer_name = "{}.{}".format(server_name, room.room_id) # This dict remembers the connection from a user_id to the name we @@ -899,6 +902,48 @@ class RoomBuffer(object): return True return False + @property + def should_send_read_marker(self): + # type () -> bool + """Check if we need to send out a read receipt.""" + if not self.read_markers_enabled: + return False + + if not self.last_read_event: + return True + + if self.last_read_event == self.last_event_id: + return False + + return True + + @property + def last_event_id(self): + # type () -> str + """Get the event id of the last shown matrix event.""" + for line in self.weechat_buffer.lines: + for tag in line.tags: + if tag.startswith("matrix_id"): + event_id = tag[10:] + return event_id + + return "" + + @property + def read_markers_enabled(self): + # type: () -> bool + """Check if read receipts are enabled for this room.""" + return bool(int(W.string_eval_expression( + G.CONFIG.network.read_markers_conditions, + {}, + {"markers_enabled": str(int(self._read_markers_enabled))}, + {"type": "condition"} + ))) + + @read_markers_enabled.setter + def read_markers_enabled(self, value): + self._read_markers_enabled = value + def find_nick(self, user_id): # type: (str) -> str """Find a suitable nick from a user_id.""" diff --git a/matrix/commands.py b/matrix/commands.py index 9350b67..14f2b5b 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -151,6 +151,12 @@ class WeechatCommandParser(object): choices=["enable", "disable", "toggle"] ) + read_markers = subparsers.add_parser("read-markers") + read_markers.add_argument( + "state", + choices=["enable", "disable", "toggle"] + ) + return WeechatCommandParser._run_parser(parser, args) @@ -376,12 +382,15 @@ def hook_commands(): "room", "change room state", # Synopsis - ("typing-notifications " + ("typing-notifications ||" + "read-markers " ), # Description ("state: one of enable, disable or toggle\n"), # Completions - ("typing-notifications enable|disable|toggle"), + ("typing-notifications enable|disable|toggle||" + "read-markers enable|disable|toggle" + ), # Callback "matrix_room_command_cb", "", @@ -915,6 +924,15 @@ def matrix_room_command_cb(data, buffer, args): room.typing_enabled = not room.typing_enabled break + elif parsed_args.subcommand == "read-markers": + if parsed_args.state == "enable": + room.read_markers_enabled = True + elif parsed_args.state == "disable": + room.read_markers_enabled = False + elif parsed_args.state == "toggle": + room.read_markers_enabled = not room.read_markers_enabled + break + return W.WEECHAT_RC_OK diff --git a/matrix/config.py b/matrix/config.py index 8f80bb9..8914938 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -625,6 +625,19 @@ class MatrixConfig(WeechatConfig): "the typing_enabled variable can be manipulated with the " "/room command, see /help room"), ), + Option( + "read_markers_conditions", + "string", + "", + 0, + 0, + "${markers_enabled}", + ("conditions to send read markers (note: content is " + "evaluated, see /help eval); besides the buffer and window " + "variables the markers_enabled variable is also expanded; " + "the markers_enabled variable can be manipulated with the " + "/room command, see /help room"), + ), ] color_options = [ diff --git a/matrix/server.py b/matrix/server.py index 657a94b..ab49e61 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -673,6 +673,26 @@ class MatrixServer(object): self.backlog_queue[uuid] = room_id self.send_or_queue(request) + def room_send_read_marker(self, room_buffer): + """Send read markers for the provided room. + + Args: + room_buffer(RoomBuffer): the room for which the read markers should + be sent. + """ + if not self.connected: + return + + event_id = room_buffer.last_event_id + + _, request = self.client.room_read_markers( + room_buffer.room.room_id, + fully_read_event=event_id, + read_event=event_id) + self.send(request) + + room_buffer.last_read_event = event_id + def room_send_typing_notice(self, room_buffer): """Send a typing notice for the provided room. From 1fe0484b9668a9e612afa8c37c6c6ed4987f54fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 3 Dec 2018 22:39:50 +0100 Subject: [PATCH 237/269] buffer: Ignore unknown events. --- matrix/buffer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 78b1c9f..c3f4378 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -43,7 +43,8 @@ from nio import ( RoomTopicEvent, MegolmEvent, Event, - OlmTrustError + OlmTrustError, + UnknownEvent ) from . import globals as G @@ -1389,6 +1390,9 @@ class RoomBuffer(object): self.undecrypted_events.append(event) + elif isinstance(event, UnknownEvent): + pass + else: W.prnt( "", "Unhandled event of type {}.".format(type(event).__name__) From eabaee6a1ba859b4dcc724969e89a4023a7b8453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 10 Dec 2018 18:55:01 +0100 Subject: [PATCH 238/269] buffer: Handle fully read account data events. --- matrix/buffer.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index c3f4378..ef722a8 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -44,7 +44,8 @@ from nio import ( MegolmEvent, Event, OlmTrustError, - UnknownEvent + UnknownEvent, + FullyReadEvent, ) from . import globals as G @@ -1645,6 +1646,12 @@ class RoomBuffer(object): for event in timeline_events: self.handle_timeline_event(event) + for event in info.account_data: + if isinstance(event, FullyReadEvent): + if event.event_id == self.last_event_id: + W.buffer_set(self.weechat_buffer._ptr, "unread", "") + W.buffer_set(self.weechat_buffer._ptr, "hotlist", "-1") + # We didn't handle all joined users, the room display name might still # be outdated because of that, update it now. if self.unhandled_users: From cf178a32c4e9c7877bf3c40e635ff5691c22de3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 10 Dec 2018 20:10:36 +0100 Subject: [PATCH 239/269] server: Send read markers after sending a message. --- main.py | 5 ++++- matrix/server.py | 21 ++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 69f420d..ce4b090 100644 --- a/main.py +++ b/main.py @@ -486,7 +486,10 @@ def buffer_switch_cb(_, _signal, buffer_ptr): continue if room_buffer.should_send_read_marker: - server.room_send_read_marker(room_buffer) + event_id = room_buffer.last_event_id + server.room_send_read_marker( + room_buffer.room.room_id, event_id) + room_buffer.last_read_event = event_id if room_buffer.members_fetched: return W.WEECHAT_RC_OK diff --git a/matrix/server.py b/matrix/server.py index ab49e61..eef4cab 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -673,26 +673,23 @@ class MatrixServer(object): self.backlog_queue[uuid] = room_id self.send_or_queue(request) - def room_send_read_marker(self, room_buffer): + def room_send_read_marker(self, room_id, event_id): """Send read markers for the provided room. Args: - room_buffer(RoomBuffer): the room for which the read markers should + room_id(str): the room for which the read markers should be sent. + event_id(str): the event id where to set the marker """ if not self.connected: return - event_id = room_buffer.last_event_id - _, request = self.client.room_read_markers( - room_buffer.room.room_id, + room_id, fully_read_event=event_id, read_event=event_id) self.send(request) - room_buffer.last_read_event = event_id - def room_send_typing_notice(self, room_buffer): """Send a typing notice for the provided room. @@ -863,6 +860,13 @@ class MatrixServer(object): room_buffer.printed_before_ack_queue.remove(response.uuid) def handle_own_messages(self, response): + def send_marker(): + if not room_buffer.read_markers_enabled: + return + + self.room_send_read_marker(response.room_id, response.event_id) + room_buffer.last_read_event = response.event_id + room_buffer = self.room_buffers[response.room_id] message = room_buffer.sent_messages_queue.pop(response.uuid) message = message._replace(event_id=response.event_id) @@ -871,13 +875,16 @@ class MatrixServer(object): if response.uuid in room_buffer.printed_before_ack_queue: room_buffer.replace_printed_line_by_uuid(response.uuid, message) room_buffer.printed_before_ack_queue.remove(response.uuid) + send_marker() return if isinstance(message, OwnAction): room_buffer.self_action(message) + send_marker() return if isinstance(message, OwnMessage): room_buffer.self_message(message) + send_marker() return raise NotImplementedError( From 2a68120d0971c1da8e7d902e32f2a92f2cca2332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 10 Dec 2018 20:10:58 +0100 Subject: [PATCH 240/269] buffer: Don't move the unread marker for the current buffer. --- matrix/buffer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/matrix/buffer.py b/matrix/buffer.py index ef722a8..6c9ac18 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -1649,6 +1649,11 @@ class RoomBuffer(object): for event in info.account_data: if isinstance(event, FullyReadEvent): if event.event_id == self.last_event_id: + current_buffer = W.buffer_search("", "") + + if self.weechat_buffer._ptr == current_buffer: + continue + W.buffer_set(self.weechat_buffer._ptr, "unread", "") W.buffer_set(self.weechat_buffer._ptr, "hotlist", "-1") From 98f233defd6d06dbc9a6075b9b9f27d597c28d20 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 9 Jan 2019 15:01:56 +0100 Subject: [PATCH 241/269] Fix collections ABC deprecation warning. --- matrix/utf.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/matrix/utf.py b/matrix/utf.py index 320cb58..4d71987 100644 --- a/matrix/utf.py +++ b/matrix/utf.py @@ -28,9 +28,13 @@ import sys # pylint: disable=redefined-builtin from builtins import bytes, str -from collections import Iterable, Mapping from functools import wraps +if sys.version_info.major == 3 and sys.version_info.minor >= 3: + from collections.abc import Iterable, Mapping +else: + from collections import Iterable, Mapping + # These functions were written by Trygve Aaberge for wee-slack and are under a # MIT License. # More info can be found in the wee-slack repository under the commit: From ab2ba0f9ae1810141e1f93a3ba754733ddedaf46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 12 Dec 2018 13:09:03 +0100 Subject: [PATCH 242/269] globals: Add a logbook based logger. --- matrix/globals.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/matrix/globals.py b/matrix/globals.py index 3559ab8..8958203 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -18,6 +18,7 @@ from __future__ import unicode_literals import sys from typing import Dict, Optional +from logbook import Logger from .utf import WeechatWrapper @@ -41,3 +42,4 @@ ENCRYPTION = True # type: bool SCRIPT_NAME = "matrix" # type: str MAX_EVENTS = 100 TYPING_NOTICE_TIMEOUT = 4000 # 4 seconds typing notice lifetime +LOGGER = Logger("weechat-matrix") From 7e0215702ca4fa97404b3bd325d609d11fd056d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 19 Dec 2018 12:46:23 +0100 Subject: [PATCH 243/269] server: Refactor message sending. --- matrix/buffer.py | 23 ++++++------ matrix/server.py | 95 ++++++++++++++++++++++++++++++++---------------- 2 files changed, 74 insertions(+), 44 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 6c9ac18..1c4e9fe 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -18,6 +18,7 @@ from __future__ import unicode_literals import time +import attr from builtins import super from functools import partial from collections import deque @@ -55,17 +56,15 @@ from .globals import SCRIPT_NAME, SERVERS, W, TYPING_NOTICE_TIMEOUT from .utf import utf8_decode from .utils import server_ts_to_weechat, shorten_sender, string_strikethrough -OwnMessages = NamedTuple( - "OwnMessages", - [ - ("sender", str), - ("age", int), - ("event_id", str), - ("uuid", str), - ("room_id", str), - ("formatted_message", Formatted), - ], -) + +@attr.s +class OwnMessages(object): + sender = attr.ib(type=str) + age = attr.ib(type=int) + event_id = attr.ib(type=str) + uuid = attr.ib(type=str) + room_id = attr.ib(type=str) + formatted_message = attr.ib(type=Formatted) class OwnMessage(OwnMessages): @@ -1218,7 +1217,7 @@ class RoomBuffer(object): if not message: return - message = message._replace(event_id=event.event_id) + message.event_id = event.event_id if uuid in self.printed_before_ack_queue: self.replace_printed_line_by_uuid( event.transaction_id, diff --git a/matrix/server.py b/matrix/server.py index eef4cab..aed054a 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -21,9 +21,12 @@ import pprint import socket import ssl import time +import copy from collections import defaultdict, deque from typing import Any, Deque, Dict, Optional, List, NamedTuple, DefaultDict +from uuid import UUID + from nio import ( HttpClient, LocalProtocolError, @@ -743,6 +746,36 @@ class MatrixServer(object): room_buffer.typing = True self.send(request) + + def _room_send_message( + self, + room_id, # type: str + content, # type: Dict[str, str] + ): + # type: (...) -> UUID + assert self.client + + try: + uuid, request = self.client.room_send( + room_id, "m.room.message", content + ) + self.send(request) + return uuid + except GroupEncryptionError: + try: + if not self.group_session_shared[room_id]: + _, request = self.client.share_group_session(room_id) + self.group_session_shared[room_id] = True + self.send(request) + raise + + except EncryptionError: + if not self.keys_claimed[room_id]: + _, request = self.client.keys_claim(room_id) + self.keys_claimed[room_id] = True + self.send(request) + raise + def room_send_message( self, room_buffer, # type: RoomBuffer @@ -754,31 +787,17 @@ class MatrixServer(object): assert self.client - body = {"msgtype": msgtype, "body": formatted.to_plain()} + content = {"msgtype": msgtype, "body": formatted.to_plain()} if formatted.is_formatted(): - body["format"] = "org.matrix.custom.html" - body["formatted_body"] = formatted.to_html() + content["format"] = "org.matrix.custom.html" + content["formatted_body"] = formatted.to_html() try: - uuid, request = self.client.room_send( - room.room_id, "m.room.message", body - ) - except GroupEncryptionError: - request = None - try: - if not self.group_session_shared[room.room_id]: - _, request = self.client.share_group_session(room.room_id) - self.group_session_shared[room.room_id] = True - except EncryptionError: - if not self.keys_claimed[room.room_id]: - _, request = self.client.keys_claim(room.room_id) - self.keys_claimed[room.room_id] = True - + uuid = self._room_send_message(room.room_id, content) + except (EncryptionError, GroupEncryptionError): message = EncrytpionQueueItem(msgtype, formatted) self.encryption_queue[room.room_id].append(message) - if request: - self.send_or_queue(request) return False if msgtype == "m.emote": @@ -791,11 +810,26 @@ class MatrixServer(object): ) room_buffer.sent_messages_queue[uuid] = own_message - self.send_or_queue(request) + self.print_unconfirmed_message(room_buffer, own_message) + return True + + def print_unconfirmed_message(self, room_buffer, message): + """Print an outoing message before getting a recieve confirmation. + + The message is printed out greyed out and only printed out if the + client is configured to do so. The message needs to be later modified + to contain proper coloring, this is done in the + replace_printed_line_by_uuid() method of the RoomBuffer class. + + Args: + room_buffer(RoomBuffer): the buffer of the room where the message + needs to be printed out + message(OwnMessages): the message that should be printed out + """ if G.CONFIG.network.print_unconfirmed_messages: - room_buffer.printed_before_ack_queue.append(uuid) - plain_message = formatted.to_weechat() + room_buffer.printed_before_ack_queue.append(message.uuid) + plain_message = message.formatted_message.to_weechat() plain_message = W.string_remove_color(plain_message, "") attributes = DEFAULT_ATTRIBUTES.copy() attributes["fgcolor"] = G.CONFIG.color.unconfirmed_message @@ -804,16 +838,13 @@ class MatrixServer(object): attributes )]) - own_message = message_class( - self.user_id, 0, "", uuid, room.room_id, new_formatted - ) + new_message = copy.copy(message) + new_message.formatted_message = new_formatted - if isinstance(own_message, OwnAction): - room_buffer.self_action(own_message) - elif isinstance(own_message, OwnMessage): - room_buffer.self_message(own_message) - - return True + if isinstance(new_message, OwnAction): + room_buffer.self_action(new_message) + elif isinstance(new_message, OwnMessage): + room_buffer.self_message(new_message) def keys_upload(self): _, request = self.client.keys_upload() @@ -869,7 +900,7 @@ class MatrixServer(object): room_buffer = self.room_buffers[response.room_id] message = room_buffer.sent_messages_queue.pop(response.uuid) - message = message._replace(event_id=response.event_id) + message.event_id = response.event_id # We already printed the message, just modify it to contain the proper # colors and formatting. if response.uuid in room_buffer.printed_before_ack_queue: From 4eb1d52d81516e2c65d993f456978b09b44bd05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 19 Dec 2018 17:56:28 +0100 Subject: [PATCH 244/269] matrix: Add initial upload support. --- contrib/matrix_upload | 245 ++++++++++++++++++++++++++++++ main.py | 5 +- matrix/commands.py | 119 ++++++++++++++- matrix/config.py | 1 + matrix/globals.py | 3 + matrix/server.py | 59 +++++++- matrix/uploads.py | 340 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 767 insertions(+), 5 deletions(-) create mode 100755 contrib/matrix_upload create mode 100644 matrix/uploads.py diff --git a/contrib/matrix_upload b/contrib/matrix_upload new file mode 100755 index 0000000..b4fd722 --- /dev/null +++ b/contrib/matrix_upload @@ -0,0 +1,245 @@ +#!/usr/bin/python3 -u +# Copyright © 2018 Damir Jelić +# +# Permission to use, copy, modify, and/or distribute this software for +# any purpose with or without fee is hereby granted, provided that the +# above copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +import os +import json +import magic +import requests +import argparse +from urllib.parse import urlparse +import urllib3 + +from nio import Api, UploadResponse, UploadError +from json.decoder import JSONDecodeError + +urllib3.disable_warnings() + +mime = magic.Magic(mime=True) + + +class Upload(object): + def __init__(self, file, chunksize=1 << 13): + self.file = file + self.filename = os.path.basename(file) + self.chunksize = chunksize + self.totalsize = os.path.getsize(file) + self.mimetype = mime.from_file(file) + self.readsofar = 0 + + def send_progress(self): + message = { + "type": "progress", + "data": self.readsofar + } + to_stdout(message) + + def __iter__(self): + with open(self.filename, 'rb') as file: + while True: + data = file.read(self.chunksize) + + if not data: + break + + self.readsofar += len(data) + self.send_progress() + + yield data + + def __len__(self): + return self.totalsize + + +class IterableToFileAdapter(object): + def __init__(self, iterable): + self.iterator = iter(iterable) + self.length = len(iterable) + + def read(self, size=-1): + return next(self.iterator, b'') + + def __len__(self): + return self.length + + +def to_stdout(message): + print(json.dumps(message), flush=True) + + +def error(e): + message = { + "type": "status", + "status": "error", + "message": str(e) + } + to_stdout(message) + os.sys.exit() + + +def upload_process(args): + file_path = os.path.expanduser(args.file) + + try: + upload = Upload(file_path, 10) + except (FileNotFoundError, OSError, IOError) as e: + error(e) + + try: + url = urlparse(args.homeserver) + except ValueError as e: + error(e) + + upload_url = ("https://{}".format(args.homeserver) + if not url.scheme else args.homeserver) + _, api_path, _ = Api.upload(args.access_token, upload.filename) + upload_url += api_path + + headers = { + "Content-type": upload.mimetype, + } + + proxies = {} + + if args.proxy_address: + user = args.proxy_user or "" + + if args.proxy_password: + user += ":{}".format(args.proxy_password) + + if user: + user += "@" + + proxies = { + "https": "{}://{}{}:{}/".format( + args.proxy_type, + user, + args.proxy_address, + args.proxy_port + ) + } + + message = { + "type": "status", + "status": "started", + "total": upload.totalsize, + "mimetype": upload.mimetype, + "file_name": upload.filename, + } + to_stdout(message) + + session = requests.Session() + session.trust_env = False + + try: + r = session.post( + url=upload_url, + auth=None, + headers=headers, + data=IterableToFileAdapter(upload), + verify=(not args.insecure), + proxies=proxies + ) + except (requests.exceptions.RequestException, OSError) as e: + error(e) + + try: + json_response = json.loads(r.content) + except JSONDecodeError: + error(r.content) + + response = UploadResponse.from_dict(json_response) + + if isinstance(response, UploadError): + error(str(response)) + + message = { + "type": "status", + "status": "done", + "url": response.content_uri + } + + to_stdout(message) + + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Download and decrypt matrix attachments" + ) + parser.add_argument("file", help="the file that will be uploaded") + parser.add_argument( + "homeserver", + type=str, + help="the address of the homeserver" + ) + parser.add_argument( + "access_token", + type=str, + help="the access token to use for the upload" + ) + parser.add_argument( + "--encrypt", + action="store_const", + const=True, + default=False, + help="encrypt the file before uploading it" + ) + parser.add_argument( + "--insecure", + action="store_const", + const=True, + default=False, + help="disable SSL certificate verification" + ) + parser.add_argument( + "--proxy-type", + choices=[ + "http", + "socks4", + "socks5" + ], + default="http", + help="type of the proxy that will be used to establish a connection" + ) + parser.add_argument( + "--proxy-address", + type=str, + help="address of the proxy that will be used to establish a connection" + ) + parser.add_argument( + "--proxy-port", + type=int, + default=8080, + help="port of the proxy that will be used to establish a connection" + ) + parser.add_argument( + "--proxy-user", + type=str, + help="user that will be used for authentication on the proxy" + ) + parser.add_argument( + "--proxy-password", + type=str, + help="password that will be used for authentication on the proxy" + ) + + args = parser.parse_args() + upload_process(args) + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py index ce4b090..e4ec72f 100644 --- a/main.py +++ b/main.py @@ -53,7 +53,8 @@ from matrix.commands import (hook_commands, hook_page_up, matrix_me_command_cb, matrix_part_command_cb, matrix_redact_command_cb, matrix_topic_command_cb, matrix_olm_command_cb, matrix_devices_command_cb, - matrix_room_command_cb) + matrix_room_command_cb, matrix_uploads_command_cb, + matrix_upload_command_cb) from matrix.completion import (init_completion, matrix_command_completion_cb, matrix_debug_completion_cb, matrix_message_completion_cb, @@ -76,6 +77,8 @@ from matrix.server import (MatrixServer, create_default_server, from matrix.utf import utf8_decode from matrix.utils import server_buffer_prnt, server_buffer_set_title +from matrix.uploads import UploadsBuffer, upload_cb + # yapf: disable WEECHAT_SCRIPT_NAME = SCRIPT_NAME WEECHAT_SCRIPT_DESCRIPTION = "matrix chat plugin" # type: str diff --git a/matrix/commands.py b/matrix/commands.py index 14f2b5b..0b6782c 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -23,10 +23,11 @@ from collections import defaultdict from . import globals as G from .colors import Formatted -from .globals import SERVERS, W +from .globals import SERVERS, W, UPLOADS from .server import MatrixServer from .utf import utf8_decode from .utils import key_from_value, tags_from_line_data +from .uploads import UploadsBuffer, Upload class ParseError(Exception): @@ -159,6 +160,23 @@ class WeechatCommandParser(object): return WeechatCommandParser._run_parser(parser, args) + @staticmethod + def uploads(args): + parser = WeechatArgParse(prog="uploads") + subparsers = parser.add_subparsers(dest="subcommand") + subparsers.add_parser("list") + subparsers.add_parser("listfull") + subparsers.add_parser("up") + subparsers.add_parser("down") + + return WeechatCommandParser._run_parser(parser, args) + + @staticmethod + def upload(args): + parser = WeechatArgParse(prog="upload") + parser.add_argument("file") + return WeechatCommandParser._run_parser(parser, args) + def grouper(iterable, n, fillvalue=None): "Collect data into fixed-length chunks or blocks" @@ -396,6 +414,39 @@ def hook_commands(): "", ) + # W.hook_command( + # # Command name and short description + # "uploads", + # "Open the uploads buffer or list uploads in the core buffer", + # # Synopsis + # ("list||" + # "listfull" + # ), + # # Description + # (""), + # # Completions + # ("list ||" + # "listfull"), + # # Callback + # "matrix_uploads_command_cb", + # "", + # ) + + W.hook_command( + # Command name and short description + "upload", + "Upload a file to a room", + # Synopsis + (""), + # Description + (""), + # Completions + ("%(filename)"), + # Callback + "matrix_upload_command_cb", + "", + ) + W.hook_command_run("/buffer clear", "matrix_command_buf_clear_cb", "") if G.CONFIG.network.fetch_backlog_on_pgup: @@ -936,6 +987,72 @@ def matrix_room_command_cb(data, buffer, args): return W.WEECHAT_RC_OK +@utf8_decode +def matrix_uploads_command_cb(data, buffer, args): + if not args: + if not G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer = UploadsBuffer() + G.CONFIG.upload_buffer.display() + return W.WEECHAT_RC_OK + + parsed_args = WeechatCommandParser.uploads(args) + if not parsed_args: + return W.WEECHAT_RC_OK + + if parsed_args.subcommand == "list": + pass + elif parsed_args.subcommand == "listfull": + pass + elif parsed_args.subcommand == "up": + if G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer.move_line_up() + elif parsed_args.subcommand == "down": + if G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer.move_line_down() + + return W.WEECHAT_RC_OK + + +@utf8_decode +def matrix_upload_command_cb(data, buffer, args): + parsed_args = WeechatCommandParser.upload(args) + if not parsed_args: + return W.WEECHAT_RC_OK + + for server in SERVERS.values(): + if buffer == server.server_buffer: + server.error( + 'command "upload" must be ' "executed on a Matrix room buffer" + ) + return W.WEECHAT_RC_OK + + room_buffer = server.find_room_from_ptr(buffer) + if not room_buffer: + continue + + if room_buffer.room.encrypted: + room_buffer.error("Uploading to encrypted rooms is " + "not yet implemented") + return W.WEECHAT_RC_OK + + upload = Upload( + server.name, + server.config.address, + server.client.access_token, + room_buffer.room.room_id, + parsed_args.file, + room_buffer.room.encrypted + ) + UPLOADS[upload.uuid] = upload + + if G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer.render() + + break + + return W.WEECHAT_RC_OK + + @utf8_decode def matrix_kick_command_cb(data, buffer, args): parsed_args = WeechatCommandParser.kick(args) diff --git a/matrix/config.py b/matrix/config.py index 8914938..9d2aeb1 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -395,6 +395,7 @@ class MatrixConfig(WeechatConfig): def __init__(self): self.debug_buffer = "" + self.upload_buffer = "" self.debug_category = "all" self.page_up_hook = None diff --git a/matrix/globals.py b/matrix/globals.py index 8958203..5950d9e 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -19,12 +19,14 @@ from __future__ import unicode_literals import sys from typing import Dict, Optional from logbook import Logger +from collections import OrderedDict from .utf import WeechatWrapper if False: from .server import MatrixServer from .config import MatrixConfig + from .uploads import Upload try: @@ -43,3 +45,4 @@ SCRIPT_NAME = "matrix" # type: str MAX_EVENTS = 100 TYPING_NOTICE_TIMEOUT = 4000 # 4 seconds typing notice lifetime LOGGER = Logger("weechat-matrix") +UPLOADS = OrderedDict() # type: Dict[str, Upload] diff --git a/matrix/server.py b/matrix/server.py index aed054a..016c5bd 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -28,6 +28,7 @@ from typing import Any, Deque, Dict, Optional, List, NamedTuple, DefaultDict from uuid import UUID from nio import ( + Api, HttpClient, LocalProtocolError, LoginResponse, @@ -746,18 +747,70 @@ class MatrixServer(object): room_buffer.typing = True self.send(request) + def room_send_upload( + self, + upload + ): + """Send a room message containing the mxc URI of an upload.""" + try: + room_buffer = self.find_room_from_id(upload.room_id) + except (ValueError, KeyError): + return False - def _room_send_message( + assert self.client + + if room_buffer.room.encrypted: + room_buffer.error("Uploading to encrypted rooms is " + "not yet implemented") + return False + + # TODO the content is different if the room is encrypted. + content = { + "msgtype": Api.mimetype_to_msgtype(upload.mimetype), + "body": upload.file_name, + "url": upload.content_uri, + } + + try: + uuid = self.room_send_event(upload.room_id, content) + except (EncryptionError, GroupEncryptionError): + # TODO put the message in a queue to resend after group sessions + # are shared + # message = EncrytpionQueueItem(msgtype, formatted) + # self.encryption_queue[room.room_id].append(message) + return False + + http_url = Api.mxc_to_http(upload.content_uri) + description = ("/{}".format(upload.file_name) if upload.file_name + else "") + + attributes = DEFAULT_ATTRIBUTES.copy() + formatted = Formatted([FormattedString( + "{url}{desc}".format(url=http_url, desc=description), + attributes + )]) + + own_message = OwnMessage( + self.user_id, 0, "", uuid, upload.room_id, formatted + ) + + room_buffer.sent_messages_queue[uuid] = own_message + self.print_unconfirmed_message(room_buffer, own_message) + + return True + + def room_send_event( self, room_id, # type: str content, # type: Dict[str, str] + event_type="m.room.message" ): # type: (...) -> UUID assert self.client try: uuid, request = self.client.room_send( - room_id, "m.room.message", content + room_id, event_type, content ) self.send(request) return uuid @@ -794,7 +847,7 @@ class MatrixServer(object): content["formatted_body"] = formatted.to_html() try: - uuid = self._room_send_message(room.room_id, content) + uuid = self.room_send_event(room.room_id, content) except (EncryptionError, GroupEncryptionError): message = EncrytpionQueueItem(msgtype, formatted) self.encryption_queue[room.room_id].append(message) diff --git a/matrix/uploads.py b/matrix/uploads.py new file mode 100644 index 0000000..819867c --- /dev/null +++ b/matrix/uploads.py @@ -0,0 +1,340 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2018 Damir Jelić +# +# Permission to use, copy, modify, and/or distribute this software for +# any purpose with or without fee is hereby granted, provided that the +# above copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Module implementing upload functionality.""" + +from __future__ import unicode_literals + +import attr +import time +import json +from uuid import uuid1, UUID +from enum import Enum + +try: + from json.decoder import JSONDecodeError +except ImportError: + JSONDecodeError = ValueError # type: ignore + +from .globals import SCRIPT_NAME, SERVERS, W, UPLOADS +from .utf import utf8_decode +from matrix import globals as G + + +class UploadState(Enum): + created = 0 + active = 1 + finished = 2 + error = 3 + aborted = 4 + + +@attr.s +class Proxy(object): + ptr = attr.ib(type=str) + + @property + def name(self): + return W.infolist_string(self.ptr, "name") + + @property + def address(self): + return W.infolist_string(self.ptr, "address") + + @property + def type(self): + return W.infolist_string(self.ptr, "type_string") + + @property + def port(self): + return str(W.infolist_integer(self.ptr, "port")) + + @property + def user(self): + return W.infolist_string(self.ptr, "username") + + @property + def password(self): + return W.infolist_string(self.ptr, "password") + + +@attr.s +class Upload(object): + """Class representing an upload to a matrix server.""" + + server_name = attr.ib(type=str) + server_address = attr.ib(type=str) + access_token = attr.ib(type=str) + room_id = attr.ib(type=str) + filepath = attr.ib(type=str) + encrypt = attr.ib(type=bool, default=False) + + done = 0 + total = 0 + + uuid = None + buffer = None + upload_hook = None + content_uri = None + file_name = None + mimetype = "?" + state = UploadState.created + + def __attrs_post_init__(self): + self.uuid = uuid1() + self.buffer = "" + + server = SERVERS[self.server_name] + + proxy_name = server.config.proxy + proxy = None + proxies_list = None + + if proxy_name: + proxies_list = W.infolist_get("proxy", "", proxy_name) + if proxies_list: + W.infolist_next(proxies_list) + proxy = Proxy(proxies_list) + + process_args = { + "arg1": self.filepath, + "arg2": self.server_address, + "arg3": self.access_token, + "buffer_flush": "1", + } + + arg_count = 3 + + if self.encrypt: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--encrypt" + + if not server.config.ssl_verify: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--insecure" + + if proxy: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-type" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.type + + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-address" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.address + + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-port" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.port + + if proxy.user: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-user" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.user + + if proxy.password: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-password" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.password + + self.upload_hook = W.hook_process_hashtable( + "matrix_upload", + process_args, + 0, + "upload_cb", + str(self.uuid) + ) + + if proxies_list: + W.infolist_free(proxies_list) + + def abort(self): + pass + + +@attr.s +class UploadsBuffer(object): + """Weechat buffer showing the uploads for a server.""" + + _ptr = "" # type: str + _selected_line = 0 # type: int + uploads = UPLOADS + + def __attrs_post_init__(self): + self._ptr = W.buffer_new( + SCRIPT_NAME + ".uploads", + "", + "", + "", + "", + ) + W.buffer_set(self._ptr, "type", "free") + W.buffer_set(self._ptr, "title", "Upload list") + W.buffer_set(self._ptr, "key_bind_meta2-A", "/uploads up") + W.buffer_set(self._ptr, "key_bind_meta2-B", "/uploads down") + W.buffer_set(self._ptr, "localvar_set_type", "uploads") + + self.render() + + def move_line_up(self): + self._selected_line = max(self._selected_line - 1, 0) + self.render() + + def move_line_down(self): + self._selected_line = min( + self._selected_line + 1, + len(self.uploads) - 1 + ) + self.render() + + def display(self): + """Display the buffer.""" + W.buffer_set(self._ptr, "display", "1") + + def render(self): + """Render the new state of the upload buffer.""" + # This function is under the MIT license. + # Copyright (c) 2016 Vladimir Ignatev + def progress(count, total): + bar_len = 60 + + if total == 0: + bar = '-' * bar_len + return "[{}] {}%".format(bar, "?") + + filled_len = int(round(bar_len * count / float(total))) + percents = round(100.0 * count / float(total), 1) + bar = '=' * filled_len + '-' * (bar_len - filled_len) + + return "[{}] {}%".format(bar, percents) + + W.buffer_clear(self._ptr) + header = "{}{}{}{}{}{}{}{}".format( + W.color("green"), + "Actions (letter+enter):", + W.color("lightgreen"), + " [A] Accept", + " [C] Cancel", + " [R] Remove", + " [P] Purge finished", + " [Q] Close this buffer" + ) + W.prnt_y(self._ptr, 0, header) + + for line_number, upload in enumerate(self.uploads.values()): + line_color = "{},{}".format( + "white" if line_number == self._selected_line else "default", + "blue" if line_number == self._selected_line else "default", + ) + first_line = ("%s%s %-24s %s%s%s %s (%s.%s)" % ( + W.color(line_color), + "*** " if line_number == self._selected_line else " ", + upload.room_id, + "\"", + upload.filepath, + "\"", + upload.mimetype, + SCRIPT_NAME, + upload.server_name, + )) + W.prnt_y(self._ptr, (line_number * 2) + 2, first_line) + + status_color = "{},{}".format("green", "blue") + status = "{}{}{}".format( + W.color(status_color), + upload.state.name, + W.color(line_color) + ) + + second_line = ("{color}{prefix} {status} {progressbar} " + "{done} / {total}").format( + color=W.color(line_color), + prefix="*** " if line_number == self._selected_line else " ", + status=status, + progressbar=progress(upload.done, upload.total), + done=W.string_format_size(upload.done), + total=W.string_format_size(upload.total)) + + W.prnt_y(self._ptr, (line_number * 2) + 3, second_line) + + +def find_upload(uuid): + return UPLOADS.get(uuid, None) + + +def handle_child_message(upload, message): + if message["type"] == "progress": + upload.done = message["data"] + + elif message["type"] == "status": + if message["status"] == "started": + upload.state = UploadState.active + upload.total = message["total"] + upload.mimetype = message["mimetype"] + upload.file_name = message["file_name"] + + elif message["status"] == "done": + upload.state = UploadState.finished + upload.content_uri = message["url"] + + server = SERVERS.get(upload.server_name, None) + + if not server: + return + + server.room_send_upload(upload) + + elif message["status"] == "error": + upload.state = UploadState.error + + if G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer.render() + + +@utf8_decode +def upload_cb(data, command, return_code, out, err): + upload = find_upload(UUID(data)) + + if not upload: + return W.WEECHAT_RC_OK + + if return_code == W.WEECHAT_HOOK_PROCESS_ERROR: + W.prnt("", "Error with command '%s'" % command) + return W.WEECHAT_RC_OK + + if err != "": + W.prnt("", "Error with command '%s'" % err) + upload.state = UploadState.error + + if out != "": + upload.buffer += out + messages = upload.buffer.split("\n") + upload.buffer = "" + + for m in messages: + try: + message = json.loads(m) + except (JSONDecodeError, TypeError): + upload.buffer += m + continue + + handle_child_message(upload, message) + + return W.WEECHAT_RC_OK From a75ce9c636b84110a1fa81a4ef914cc8963c128f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 13 Jan 2019 13:41:32 +0100 Subject: [PATCH 245/269] buffer: Fix buffer lines updating if the new string is an empty string. This should fix the buffer sorting bug where the prefix isn't properly updated. --- matrix/buffer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 1c4e9fe..24382f4 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -314,17 +314,17 @@ class WeechatChannelBuffer(object): ): new_data = {} - if date: + if date is not None: new_data["date"] = str(date) - if date_printed: + if date_printed is not None: new_data["date_printed"] = str(date_printed) - if tags: + if tags is not None: new_data["tags_array"] = ",".join(tags) - if prefix: + if prefix is not None: new_data["prefix"] = prefix - if message: + if message is not None: new_data["message"] = message - if highlight: + if highlight is not None: new_data["highlight"] = highlight if new_data: From 6203a84553389fb933eddd2f5d0a90959972518a Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 7 Nov 2018 14:00:35 +0100 Subject: [PATCH 246/269] Use uniform spelling: colour -> color. --- matrix/colors.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 96a4c24..aa4a452 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -624,20 +624,20 @@ def color_line_to_weechat(color_string): return line_colors[color_string] -# The functions colour_dist_sq(), colour_to_6cube(), and colour_find_rgb +# The functions color_dist_sq(), color_to_6cube(), and color_find_rgb # are python ports of the same named functions from the tmux # source, they are under the copyright of Nicholas Marriott, and Avi Halachmi # under the ISC license. # More info: https://github.com/tmux/tmux/blob/master/colour.c -def colour_dist_sq(R, G, B, r, g, b): +def color_dist_sq(R, G, B, r, g, b): # pylint: disable=invalid-name,too-many-arguments # type: (int, int, int, int, int, int) -> int return (R - r) * (R - r) + (G - g) * (G - g) + (B - b) * (B - b) -def colour_to_6cube(v): +def color_to_6cube(v): # pylint: disable=invalid-name # type: (int) -> int if v < 48: @@ -647,15 +647,15 @@ def colour_to_6cube(v): return (v - 35) // 40 -def colour_find_rgb(r, g, b): +def color_find_rgb(r, g, b): # type: (int, int, int) -> int - """Convert an RGB triplet to the xterm(1) 256 colour palette. + """Convert an RGB triplet to the xterm(1) 256 color palette. - xterm provides a 6x6x6 colour cube (16 - 231) and 24 greys (232 - 255). - We map our RGB colour to the closest in the cube, also work out the + xterm provides a 6x6x6 color cube (16 - 231) and 24 greys (232 - 255). + We map our RGB color to the closest in the cube, also work out the closest grey, and use the nearest of the two. - Note that the xterm has much lower resolution for darker colours (they + Note that the xterm has much lower resolution for darker colors (they are not evenly spread out), so our 6 levels are not evenly spread: 0x0, 0x5f (95), 0x87 (135), 0xaf (175), 0xd7 (215) and 0xff (255). Greys are more evenly spread (8, 18, 28 ... 238). @@ -664,15 +664,15 @@ def colour_find_rgb(r, g, b): q2c = [0x00, 0x5f, 0x87, 0xaf, 0xd7, 0xff] # Map RGB to 6x6x6 cube. - qr = colour_to_6cube(r) - qg = colour_to_6cube(g) - qb = colour_to_6cube(b) + qr = color_to_6cube(r) + qg = color_to_6cube(g) + qb = color_to_6cube(b) cr = q2c[qr] cg = q2c[qg] cb = q2c[qb] - # If we have hit the colour exactly, return early. + # If we have hit the color exactly, return early. if cr == r and cg == g and cb == b: return 16 + (36 * qr) + (6 * qg) + qb @@ -686,10 +686,10 @@ def colour_find_rgb(r, g, b): grey = 8 + (10 * grey_idx) - # Is grey or 6x6x6 colour closest? - d = colour_dist_sq(cr, cg, cb, r, g, b) + # Is grey or 6x6x6 color closest? + d = color_dist_sq(cr, cg, cb, r, g, b) - if colour_dist_sq(grey, grey, grey, r, g, b) < d: + if color_dist_sq(grey, grey, grey, r, g, b) < d: idx = 232 + grey_idx else: idx = 16 + (36 * qr) + (6 * qg) + qb @@ -728,7 +728,7 @@ def color_html_to_weechat(color): if rgb_color in weechat_basic_colors: return weechat_basic_colors[rgb_color] - return str(colour_find_rgb(*rgb_color)) + return str(color_find_rgb(*rgb_color)) def color_weechat_to_html(color): From 96e8f5246bca926bf16f1b7eb5263bef377540f6 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Mon, 12 Nov 2018 19:25:30 +0100 Subject: [PATCH 247/269] Improve handling of code blocks. * Don't try to guess the language of code blocks without a language specifier ("untagged" blocks). * Colour untagged and inline code blocks (and make the colour configurable). * Reflow inline code blocks. --- matrix/colors.py | 25 +++++++++++++++++++------ matrix/config.py | 10 ++++++++++ matrix/utils.py | 18 ++++++++++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index aa4a452..eff7697 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -30,12 +30,12 @@ from typing import List import webcolors from pygments import highlight from pygments.formatter import Formatter, get_style_by_name -from pygments.lexers import get_lexer_by_name, guess_lexer +from pygments.lexers import get_lexer_by_name from pygments.util import ClassNotFound from . import globals as G from .globals import W -from .utils import string_strikethrough +from .utils import string_strikethrough, string_color_and_reset try: from HTMLParser import HTMLParser @@ -302,7 +302,8 @@ class Formatted(object): try: lexer = get_lexer_by_name(value) except ClassNotFound: - lexer = guess_lexer(string) + return string_color_and_reset(string, + G.CONFIG.color.untagged_code) try: style = get_style_by_name(G.CONFIG.look.pygments_style) @@ -345,14 +346,21 @@ class Formatted(object): return prefix + text.replace("\n", "\n{}".format(prefix)) for key, value in attributes.items(): - # Don't use textwrap to quote the code - if attributes["code"] and key == "quote" and value: + if not value: continue + # Don't use textwrap to quote the code + if key == "quote" and attributes["code"]: + continue + + # Reflow inline code blocks + if key == "code" and not attributes["preformatted"]: + text = text.strip().replace('\n', ' ') + text = add_attribute(text, key, value) # If we're quoted code add quotation marks now. - if attributes["quote"] and key == "code" and value: + if key == "code" and attributes["quote"]: text = indent( text, "{}>{} ".format( @@ -382,6 +390,7 @@ DEFAULT_ATTRIBUTES = { "italic": False, "underline": False, "strikethrough": False, + "preformatted": False, "quote": False, "code": None, "fgcolor": None, @@ -430,6 +439,8 @@ class MatrixHtmlParser(HTMLParser): self._toggle_attribute("strikethrough") elif tag == "blockquote": self._toggle_attribute("quote") + elif tag == "pre": + self._toggle_attribute("preformatted") elif tag == "code": lang = None @@ -480,6 +491,8 @@ class MatrixHtmlParser(HTMLParser): self._toggle_attribute("underline") elif tag == "del": self._toggle_attribute("strikethrough") + elif tag == "pre": + self._toggle_attribute("preformatted") elif tag == "code": if self.text: self.add_substring(self.text, self.attributes.copy()) diff --git a/matrix/config.py b/matrix/config.py index 8914938..c8655f7 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -671,6 +671,16 @@ class MatrixConfig(WeechatConfig): ("Color for messages that are printed out but the server " "hasn't confirmed the that he received them."), ), + Option( + "untagged_code", + "color", + "", + 0, + 0, + "blue", + ("Color for code without a language specifier. Also used for " + "`inline code`."), + ), ] sections = [ diff --git a/matrix/utils.py b/matrix/utils.py index 060fd14..e711953 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -105,3 +105,21 @@ def shorten_sender(sender): def string_strikethrough(string): return "".join(["{}\u0336".format(c) for c in string]) + + +def string_color_and_reset(string, color): + """Color string with color, then reset all attributes.""" + + lines = string.split('\n') + lines = ("{}{}{}".format(W.color(color), line, W.color("reset")) + for line in lines) + return "\n".join(lines) + + +def string_color(string, color): + """Color string with color, then reset the color attribute.""" + + lines = string.split('\n') + lines = ("{}{}{}".format(W.color(color), line, W.color("resetcolor")) + for line in lines) + return "\n".join(lines) From 041c15e811ff48677e32f40c39d59be0330b5da4 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Mon, 12 Nov 2018 19:31:24 +0100 Subject: [PATCH 248/269] Be more careful when stripping newline added by pygments. Calls rstrip() instead of cutting of the last character blindly, so that: 1. Potentially multiple trailing whitespace of any kind will get stripped. 2. We don't chop off a non-whitespace character, in case pygments' changes. --- matrix/colors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/colors.py b/matrix/colors.py index eff7697..eb6dece 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -313,7 +313,7 @@ class Formatted(object): # highlight adds a newline to the end of the string, remove it # from the output return highlight(string, lexer, - WeechatFormatter(style=style))[:-1] + WeechatFormatter(style=style)).rstrip() elif name == "fgcolor": return "{color_on}{text}{color_off}".format( color_on=W.color(value), From 141814bb848e04fb3f1ec64c8e6f10f760e39789 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Mon, 12 Nov 2018 22:42:44 +0100 Subject: [PATCH 249/269] Support full color pair (fg/bg) for each color. --- matrix/buffer.py | 11 +++++++-- matrix/colors.py | 22 +++++++++++------- matrix/config.py | 60 ++++++++++++++++++++++++++++++++++++++---------- matrix/server.py | 3 ++- matrix/utils.py | 9 ++++++++ 5 files changed, 82 insertions(+), 23 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 6c9ac18..cb10db5 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -53,7 +53,12 @@ from .colors import Formatted from .config import RedactType from .globals import SCRIPT_NAME, SERVERS, W, TYPING_NOTICE_TIMEOUT from .utf import utf8_decode -from .utils import server_ts_to_weechat, shorten_sender, string_strikethrough +from .utils import ( + server_ts_to_weechat, + shorten_sender, + string_strikethrough, + color_pair, +) OwnMessages = NamedTuple( "OwnMessages", @@ -1446,7 +1451,9 @@ class RoomBuffer(object): "message{del_color}>{ncolor}").format( del_color=W.color("chat_delimiters"), ncolor=W.color("reset"), - error_color=W.color(G.CONFIG.color.error_message)) + error_color=W.color(color_pair( + G.CONFIG.color.error_message_fg, + G.CONFIG.color.error_message_bg))) last_line.message = message diff --git a/matrix/colors.py b/matrix/colors.py index eb6dece..a8a7376 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -35,7 +35,9 @@ from pygments.util import ClassNotFound from . import globals as G from .globals import W -from .utils import string_strikethrough, string_color_and_reset +from .utils import (string_strikethrough, + string_color_and_reset, + color_pair) try: from HTMLParser import HTMLParser @@ -59,10 +61,12 @@ class Formatted(object): @property def textwrapper(self): + quote_pair = color_pair(G.CONFIG.color.quote_fg, + G.CONFIG.color.quote_bg) return textwrap.TextWrapper( width=67, - initial_indent="{}> ".format(W.color(G.CONFIG.color.quote)), - subsequent_indent="{}> ".format(W.color(G.CONFIG.color.quote)), + initial_indent="{}> ".format(W.color(quote_pair)), + subsequent_indent="{}> ".format(W.color(quote_pair)), ) def is_formatted(self): @@ -302,8 +306,10 @@ class Formatted(object): try: lexer = get_lexer_by_name(value) except ClassNotFound: - return string_color_and_reset(string, - G.CONFIG.color.untagged_code) + return string_color_and_reset( + string, + color_pair(G.CONFIG.color.untagged_code_fg, + G.CONFIG.color.untagged_code_bg)) try: style = get_style_by_name(G.CONFIG.look.pygments_style) @@ -361,11 +367,11 @@ class Formatted(object): # If we're quoted code add quotation marks now. if key == "code" and attributes["quote"]: + fg = G.CONFIG.color.quote_fg + bg = G.CONFIG.color.quote_bg text = indent( text, - "{}>{} ".format( - W.color(G.CONFIG.color.quote), W.color("reset") - ), + string_color_and_reset(">", color_pair(fg, bg)) + " ", ) # If we're code don't remove multiple newlines blindly diff --git a/matrix/config.py b/matrix/config.py index c8655f7..98f51c2 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -642,44 +642,80 @@ class MatrixConfig(WeechatConfig): color_options = [ Option( - "quote", + "quote_fg", "color", "", 0, 0, "lightgreen", - ("Color for matrix style blockquotes"), + "Foreground color for matrix style blockquotes", ), Option( - "error_message", + "quote_bg", + "color", + "", + 0, + 0, + "default", + "Background counterpart of quote_fg", + ), + Option( + "error_message_fg", "color", "", 0, 0, "darkgray", - ("Color for error messages that appear inside a room buffer (" - "e.g. when a message errors out when sending or when a " - "message is redacted)"), + ("Foreground color for error messages that appear inside a " + "room buffer (e.g. when a message errors out when sending or " + "when a message is redacted)"), ), Option( - "unconfirmed_message", + "error_message_bg", + "color", + "", + 0, + 0, + "default", + "Background counterpart of error_message_fg.", + ), + Option( + "unconfirmed_message_fg", "color", "", 0, 0, "darkgray", - ("Color for messages that are printed out but the server " - "hasn't confirmed the that he received them."), + ("Foreground color for messages that are printed out but the " + "server hasn't confirmed the that he received them."), ), Option( - "untagged_code", + "unconfirmed_message_bg", + "color", + "", + 0, + 0, + "default", + "Background counterpart of unconfirmed_message_fg." + ), + Option( + "untagged_code_fg", "color", "", 0, 0, "blue", - ("Color for code without a language specifier. Also used for " - "`inline code`."), + ("Foreground color for code without a language specifier. " + "Also used for `inline code`."), + ), + Option( + "untagged_code_bg", + "color", + "", + 0, + 0, + "default", + "Background counterpart of untagged_code_fg", ), ] diff --git a/matrix/server.py b/matrix/server.py index eef4cab..5546ad1 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -798,7 +798,8 @@ class MatrixServer(object): plain_message = formatted.to_weechat() plain_message = W.string_remove_color(plain_message, "") attributes = DEFAULT_ATTRIBUTES.copy() - attributes["fgcolor"] = G.CONFIG.color.unconfirmed_message + attributes["fgcolor"] = G.CONFIG.color.unconfirmed_message_fg + attributes["bgcolor"] = G.CONFIG.color.unconfirmed_message_bg new_formatted = Formatted([FormattedString( plain_message, attributes diff --git a/matrix/utils.py b/matrix/utils.py index e711953..d77b6f6 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -123,3 +123,12 @@ def string_color(string, color): lines = ("{}{}{}".format(W.color(color), line, W.color("resetcolor")) for line in lines) return "\n".join(lines) + + +def color_pair(color_fg, color_bg): + """Make a color pair from a pair of colors.""" + + if color_bg: + return "{},{}".format(color_fg, color_bg) + else: + return color_fg From a6cf2e15272a554474ee13a571fce58e5c6af4c9 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Tue, 13 Nov 2018 10:28:44 +0100 Subject: [PATCH 250/269] Stylize code blocks as actual boxes with margins. --- matrix/colors.py | 60 +++++++++++++++++++++++++++++++++--------------- matrix/utils.py | 35 +++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 20 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index a8a7376..218cfe2 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -37,7 +37,9 @@ from . import globals as G from .globals import W from .utils import (string_strikethrough, string_color_and_reset, - color_pair) + color_pair, + text_block, + colored_text_block) try: from HTMLParser import HTMLParser @@ -275,7 +277,7 @@ class Formatted(object): def to_weechat(self): # TODO BG COLOR - def add_attribute(string, name, value): + def add_attribute(string, name, value, attributes): if not value: return string elif name == "bold": @@ -303,23 +305,40 @@ class Formatted(object): W.string_remove_color(string.replace("\n", ""), "") ) elif name == "code": - try: - lexer = get_lexer_by_name(value) - except ClassNotFound: - return string_color_and_reset( - string, - color_pair(G.CONFIG.color.untagged_code_fg, - G.CONFIG.color.untagged_code_bg)) + code_color_pair = color_pair( + G.CONFIG.color.untagged_code_fg, + G.CONFIG.color.untagged_code_bg + ) - try: - style = get_style_by_name(G.CONFIG.look.pygments_style) - except ClassNotFound: - style = "native" + if attributes["preformatted"]: + # code block - # highlight adds a newline to the end of the string, remove it - # from the output - return highlight(string, lexer, - WeechatFormatter(style=style)).rstrip() + try: + lexer = get_lexer_by_name(value) + except ClassNotFound: + return colored_text_block( + string, + margin=2, + color_pair=code_color_pair) + + try: + style = get_style_by_name(G.CONFIG.look.pygments_style) + except ClassNotFound: + style = "native" + + code_block = text_block(string, margin=2) + + # highlight adds a newline to the end of the string, remove it + # from the output + highlighted_code = highlight( + code_block, + lexer, + WeechatFormatter(style=style) + ).rstrip() + + return highlighted_code + else: + return string_color_and_reset(string, code_color_pair) elif name == "fgcolor": return "{color_on}{text}{color_off}".format( color_on=W.color(value), @@ -344,7 +363,10 @@ class Formatted(object): # terminal, but doing it the other way around results in garbage. if "strikethrough" in attributes: text = add_attribute( - text, "strikethrough", attributes["strikethrough"] + text, + "strikethrough", + attributes["strikethrough"], + attributes ) attributes.pop("strikethrough") @@ -363,7 +385,7 @@ class Formatted(object): if key == "code" and not attributes["preformatted"]: text = text.strip().replace('\n', ' ') - text = add_attribute(text, key, value) + text = add_attribute(text, key, value, attributes) # If we're quoted code add quotation marks now. if key == "code" and attributes["quote"]: diff --git a/matrix/utils.py b/matrix/utils.py index d77b6f6..91d122d 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -14,7 +14,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import unicode_literals +from __future__ import unicode_literals, division import time from typing import Any, Dict, List @@ -132,3 +132,36 @@ def color_pair(color_fg, color_bg): return "{},{}".format(color_fg, color_bg) else: return color_fg + + +def text_block(text, margin=0): + """ + Pad block of text with whitespace to form a regular block, optionally + adding a margin. + """ + + # add vertical margin + vertical_margin = margin // 2 + text = "{}{}{}".format( + "\n" * vertical_margin, + text, + "\n" * vertical_margin + ) + + lines = text.split("\n") + longest_len = max(len(l) for l in lines) + margin + + # pad block and add horizontal margin + text = "\n".join( + "{pre}{line}{post}".format( + pre=" " * margin, + line=l, + post=" " * (longest_len - len(l))) + for l in lines) + + return text + + +def colored_text_block(text, margin=0, color_pair=""): + """ Like text_block, but also colors it.""" + return string_color_and_reset(text_block(text, margin=margin), color_pair) From 26f7dae9908a008749b1e5b9a765a82140e5a108 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 9 Jan 2019 14:14:16 +0100 Subject: [PATCH 251/269] Make code block margin configurable. --- matrix/colors.py | 10 ++++++---- matrix/config.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 218cfe2..2f51936 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -310,6 +310,8 @@ class Formatted(object): G.CONFIG.color.untagged_code_bg ) + margin = G.CONFIG.look.code_block_margin + if attributes["preformatted"]: # code block @@ -318,7 +320,7 @@ class Formatted(object): except ClassNotFound: return colored_text_block( string, - margin=2, + margin=margin, color_pair=code_color_pair) try: @@ -326,10 +328,10 @@ class Formatted(object): except ClassNotFound: style = "native" - code_block = text_block(string, margin=2) + code_block = text_block(string, margin=margin) - # highlight adds a newline to the end of the string, remove it - # from the output + # highlight adds a newline to the end of the string, remove + # it from the output highlighted_code = highlight( code_block, lexer, diff --git a/matrix/config.py b/matrix/config.py index 98f51c2..d16d456 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -496,6 +496,16 @@ class MatrixConfig(WeechatConfig): "native", "Pygments style to use for highlighting source code blocks", ), + Option( + "code_block_margin", + "integer", + "", + 0, + 100, + "2", + ("Number of spaces to add as a margin around around a code " + "block"), + ), ] network_options = [ From dcc5a3e5304aebad0b61873ecafefc6f410a6954 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 9 Jan 2019 14:44:37 +0100 Subject: [PATCH 252/269] Make displaying preformatted code as blocks configurable. --- matrix/colors.py | 17 ++++++++++++----- matrix/config.py | 11 +++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 2f51936..3eeabb8 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -318,17 +318,24 @@ class Formatted(object): try: lexer = get_lexer_by_name(value) except ClassNotFound: - return colored_text_block( - string, - margin=margin, - color_pair=code_color_pair) + if G.CONFIG.look.code_blocks: + return colored_text_block( + string, + margin=margin, + color_pair=code_color_pair) + else: + return string_color_and_reset(string, + code_color_pair) try: style = get_style_by_name(G.CONFIG.look.pygments_style) except ClassNotFound: style = "native" - code_block = text_block(string, margin=margin) + if G.CONFIG.look.code_blocks: + code_block = text_block(string, margin=margin) + else: + code_block = string # highlight adds a newline to the end of the string, remove # it from the output diff --git a/matrix/config.py b/matrix/config.py index d16d456..4f28963 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -496,6 +496,17 @@ class MatrixConfig(WeechatConfig): "native", "Pygments style to use for highlighting source code blocks", ), + Option( + "code_blocks", + "boolean", + "", + 0, + 0, + "on", + ("Display preformatted code blocks as rectangular areas by " + "padding them with whitespace up to the length of the longest " + "line (with optional margin)"), + ), Option( "code_block_margin", "integer", From 11b960e22c5b191670acc8fee1b36a2ad7c307b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 14 Jan 2019 17:12:49 +0100 Subject: [PATCH 253/269] server: Fix device printing if some fields are None. --- matrix/server.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index f6cfd2f..94ff6e2 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1001,9 +1001,11 @@ class MatrixServer(object): lines = [] for device in response.devices: + last_seen_date = ("?" if not device.last_seen_date else + device.last_seen_date.strftime("%Y/%m/%d %H:%M")) last_seen = "{ip} @ {date}".format( - ip=device.last_seen_ip, - date=device.last_seen_date.strftime("%Y/%m/%d %H:%M") + ip=device.last_seen_ip or "?", + date=last_seen_date ) device_color = ("chat_self" if device.id == self.device_id else W.info_get("nick_color_name", device.id)) @@ -1013,7 +1015,7 @@ class MatrixServer(object): W.color(device_color), device.id, W.color("resetcolor"), - device.display_name, + device.display_name or "", last_seen ) lines.append(line) From 3efd8660522f93173c7ab1d1f55a6db7878c1533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 15 Jan 2019 13:51:20 +0100 Subject: [PATCH 254/269] buffer: Limit the number of lines we search when replacing lines. --- matrix/buffer.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/matrix/buffer.py b/matrix/buffer.py index 0091ae6..7414268 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -807,11 +807,15 @@ class WeechatChannelBuffer(object): def short_name(self, name): W.buffer_set(self._ptr, "short_name", name) - def find_lines(self, predicate): + def find_lines(self, predicate, max_lines=None): lines = [] + count = 0 for line in self.lines: if predicate(line): lines.append(line) + count += 1 + if max_lines is not None and count == max_lines: + return lines return lines @@ -1457,11 +1461,7 @@ class RoomBuffer(object): last_line.message = message def replace_printed_line_by_uuid(self, uuid, new_message): - """Replace already printed lines that are greyed out with real ones""" - lines = self.weechat_buffer.find_lines( - partial(self._find_by_uuid_predicate, uuid) - ) - + """Replace already printed lines that are greyed out with real ones.""" if isinstance(new_message, OwnAction): displayed_nick = self.displayed_nicks[self.room.own_user_id] user = self.weechat_buffer._get_user(displayed_nick) @@ -1473,6 +1473,12 @@ class RoomBuffer(object): else: new_lines = new_message.formatted_message.to_weechat().split("\n") + line_count = len(new_lines) + + lines = self.weechat_buffer.find_lines( + partial(self._find_by_uuid_predicate, uuid), line_count + ) + for i, line in enumerate(lines): line.message = new_lines[i] tags = line.tags From a8ea6504e9b79aa431fcb09123fe6e50c4f24c97 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Fri, 11 Jan 2019 22:30:32 +0100 Subject: [PATCH 255/269] Fix parsing of unclosed Markdown delimiters. Always append the last segment of the message with default attributes. This prevents unclosed Markdown delimiters from starting Markdown effects (for instance, `*foobar` won't display "foobar" as emphasized/italicised). Markdown parsing is still half-broken as it isn't a proper Markdown parser, but this improves the situation somewhat. --- matrix/colors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/colors.py b/matrix/colors.py index 3eeabb8..17e541f 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -200,7 +200,7 @@ class Formatted(object): text = text + line[i] i = i + 1 - substrings.append(FormattedString(text, attributes)) + substrings.append(FormattedString(text, DEFAULT_ATTRIBUTES.copy())) return cls(substrings) @classmethod From ae2f70a7237151c8c526a491213ad48c0aef20f4 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 16 Jan 2019 14:12:10 +0100 Subject: [PATCH 256/269] Reword comment: bold -> emphasis --- matrix/colors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matrix/colors.py b/matrix/colors.py index 17e541f..1b27125 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -100,7 +100,7 @@ class Formatted(object): attributes["bold"] = not attributes["bold"] i = i + 1 - # Markdown bold + # Markdown emphasis elif line[i] == "*": if attributes["italic"] and not line[i - 1].isspace(): if text: @@ -179,6 +179,7 @@ class Formatted(object): # Reset all the attributes attributes = DEFAULT_ATTRIBUTES.copy() i = i + 1 + # Italic elif line[i] == "\x1D": if text: From b235a890e677f43334a18c4483696b1c639b08dd Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 16 Jan 2019 14:14:29 +0100 Subject: [PATCH 257/269] Implement Markdown inline `code`. --- matrix/colors.py | 49 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/matrix/colors.py b/matrix/colors.py index 1b27125..062259d 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -93,15 +93,29 @@ class Formatted(object): i = 0 while i < len(line): # Bold - if line[i] == "\x02": + if line[i] == "\x02" and not attributes["code"]: if text: substrings.append(FormattedString(text, attributes.copy())) text = "" attributes["bold"] = not attributes["bold"] i = i + 1 + # Markdown inline code + elif line[i] == "`": + if text: + # strip leading and trailing spaces from inline code blocks + if attributes["code"]: + text = text.strip() + + substrings.append( + FormattedString(text, attributes.copy()) + ) + text = "" + attributes["code"] = not attributes["code"] + i = i + 1 + # Markdown emphasis - elif line[i] == "*": + elif line[i] == "*" and not attributes["code"]: if attributes["italic"] and not line[i - 1].isspace(): if text: substrings.append( @@ -134,7 +148,7 @@ class Formatted(object): i = i + 1 # Color - elif line[i] == "\x03": + elif line[i] == "\x03" and not attributes["code"]: if text: substrings.append(FormattedString(text, attributes.copy())) text = "" @@ -172,7 +186,7 @@ class Formatted(object): else: attributes["bgcolor"] = None # Reset - elif line[i] == "\x0F": + elif line[i] == "\x0F" and not attributes["code"]: if text: substrings.append(FormattedString(text, attributes.copy())) text = "" @@ -181,7 +195,7 @@ class Formatted(object): i = i + 1 # Italic - elif line[i] == "\x1D": + elif line[i] == "\x1D" and not attributes["code"]: if text: substrings.append(FormattedString(text, attributes.copy())) text = "" @@ -189,7 +203,7 @@ class Formatted(object): i = i + 1 # Underline - elif line[i] == "\x1F": + elif line[i] == "\x1F" and not attributes["code"]: if text: substrings.append(FormattedString(text, attributes.copy())) text = "" @@ -236,6 +250,10 @@ class Formatted(object): text=string, quote_off="", ) + if name == "code" and value: + return "{code_on}{text}{code_off}".format( + code_on="", text=string, code_off="" + ) if name == "fgcolor" and value: return "{color_on}{text}{color_off}".format( color_on="".format( @@ -249,10 +267,23 @@ class Formatted(object): def format_string(formatted_string): text = formatted_string.text - attributes = formatted_string.attributes + attributes = formatted_string.attributes.copy() + + if attributes["code"]: + if attributes["preformatted"]: + # XXX: This can't really happen since there's no way of + # creating preformatted code blocks in weechat (because + # there is not multiline input), but I'm creating this + # branch as a note that it should be handled once we do + # implement them. + pass + else: + text = add_attribute(text, "code", True) + attributes.pop("code") + else: + for key, value in attributes.items(): + text = add_attribute(text, key, value) - for key, value in attributes.items(): - text = add_attribute(text, key, value) return text html_string = map(format_string, self.substrings) From d6a415e54a699fd4cee165460cf2373c9b16b0c3 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 16 Jan 2019 14:39:56 +0100 Subject: [PATCH 258/269] Update copyright notices. --- matrix/bar_items.py | 2 +- matrix/buffer.py | 2 +- matrix/colors.py | 3 ++- matrix/commands.py | 3 ++- matrix/completion.py | 2 +- matrix/config.py | 3 ++- matrix/globals.py | 2 +- matrix/server.py | 2 +- matrix/uploads.py | 2 +- matrix/utils.py | 3 ++- 10 files changed, 14 insertions(+), 10 deletions(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index 96d8eba..cfe3699 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/matrix/buffer.py b/matrix/buffer.py index 7414268..07d3284 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Weechat Matrix Protocol Script -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/matrix/colors.py b/matrix/colors.py index 062259d..ba16bdd 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -2,7 +2,8 @@ # Copyright © 2008 Nicholas Marriott # Copyright © 2016 Avi Halachmi -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić +# Copyright © 2018, 2019 Denis Kasak # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/matrix/commands.py b/matrix/commands.py index 0b6782c..5063c01 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić +# Copyright © 2018, 2019 Denis Kasak # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/matrix/completion.py b/matrix/completion.py index 56644e6..3792c97 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/matrix/config.py b/matrix/config.py index f90b8b0..102e16e 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić +# Copyright © 2018, 2019 Denis Kasak # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/matrix/globals.py b/matrix/globals.py index 5950d9e..7d34248 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/matrix/server.py b/matrix/server.py index 94ff6e2..2701c07 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/matrix/uploads.py b/matrix/uploads.py index 819867c..a7b4625 100644 --- a/matrix/uploads.py +++ b/matrix/uploads.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the diff --git a/matrix/utils.py b/matrix/utils.py index 91d122d..5b27a49 100644 --- a/matrix/utils.py +++ b/matrix/utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright © 2018 Damir Jelić +# Copyright © 2018, 2019 Damir Jelić +# Copyright © 2018, 2019 Denis Kasak # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the From dbf5b1d6bc113b044a29f43e2a7022688a26ae36 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 16 Jan 2019 15:42:33 +0100 Subject: [PATCH 259/269] Compress consecutive spaces in inline code blocks. --- matrix/colors.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/matrix/colors.py b/matrix/colors.py index ba16bdd..f789f56 100644 --- a/matrix/colors.py +++ b/matrix/colors.py @@ -104,9 +104,11 @@ class Formatted(object): # Markdown inline code elif line[i] == "`": if text: - # strip leading and trailing spaces from inline code blocks + # strip leading and trailing spaces and compress consecutive + # spaces in inline code blocks if attributes["code"]: text = text.strip() + text = re.sub(r"\s+", " ", text) substrings.append( FormattedString(text, attributes.copy()) From 8bc83842eeb31736dab769a0eebfce5f495c02ba Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 16 Jan 2019 16:33:08 +0100 Subject: [PATCH 260/269] Implement MockConfig object. To be used from tests that use code that expects `G.CONFIG` to be defined, e.g. G.CONFIG = MockConfig() --- matrix/_weechat.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/matrix/_weechat.py b/matrix/_weechat.py index 346b67f..d8b8356 100644 --- a/matrix/_weechat.py +++ b/matrix/_weechat.py @@ -22,6 +22,74 @@ WEECHAT_BASE_COLORS = { } +class MockObject(object): + pass + +class MockConfig(object): + config_template = { + 'debug_buffer': None, + 'debug_category': None, + '_ptr': None, + 'read': None, + 'free': None, + 'page_up_hook': None, + 'color': { + 'error_message_bg': "", + 'error_message_fg': "", + 'quote_bg': "", + 'quote_fg': "", + 'unconfirmed_message_bg': "", + 'unconfirmed_message_fg': "", + 'untagged_code_bg': "", + 'untagged_code_fg': "", + }, + 'upload_buffer': { + 'display': None, + 'move_line_down': None, + 'move_line_up': None, + 'render': None, + }, + 'look': { + 'bar_item_typing_notice_prefix': None, + 'busy_sign': None, + 'code_block_margin': None, + 'code_blocks': None, + 'disconnect_sign': None, + 'encrypted_room_sign': None, + 'encryption_warning_sign': None, + 'max_typing_notice_item_length': None, + 'pygments_style': None, + 'redactions': None, + 'server_buffer': None, + }, + 'network': { + 'debug_buffer': None, + 'debug_category': None, + 'debug_level': None, + 'fetch_backlog_on_pgup': None, + 'lag_min_show': None, + 'lag_reconnect': None, + 'lazy_load_room_users': None, + 'max_initial_sync_events': None, + 'max_nicklist_users': None, + 'print_unconfirmed_messages': None, + 'read_markers_conditions': None, + 'typing_notice_conditions': None, + }, + } + + def __init__(self): + for category, options in MockConfig.config_template.items(): + if options: + category_object = MockObject() + for option, value in options.items(): + setattr(category_object, option, value) + else: + category_object = options + + setattr(self, category, category_object) + + def color(color_name): # type: (str) -> str # yapf: disable From e39b6a82b1f3294169c1dfaff85c0893d9e3aac8 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 16 Jan 2019 15:47:41 +0100 Subject: [PATCH 261/269] Add test for normalization of spaces in inline code. --- tests/color_test.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/color_test.py b/tests/color_test.py index 6b290ff..1c10405 100644 --- a/tests/color_test.py +++ b/tests/color_test.py @@ -7,8 +7,11 @@ from collections import OrderedDict from hypothesis import given from hypothesis.strategies import sampled_from -from matrix.colors import (Formatted, FormattedString, +from matrix.colors import (G, Formatted, FormattedString, color_html_to_weechat, color_weechat_to_html) +from matrix._weechat import MockConfig + +G.CONFIG = MockConfig() html_prism = ("Test") @@ -42,3 +45,14 @@ def test_handle_strikethrough_first(): assert f1.to_weechat() == valid_result assert f2.to_weechat() == valid_result + + +def test_normalize_spaces_in_inline_code(): + """Normalize spaces in inline code blocks. + + Strips leading and trailing spaces and compress consecutive infix spaces. + """ + valid_result = '\x1b[0m* a *\x1b[00m' + + formatted = Formatted.from_input_line('` * a * `') + assert formatted.to_weechat() == valid_result From 9644ae5b65fe0aae23a63599ac185a460c99c1fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 16 Jan 2019 21:21:50 +0100 Subject: [PATCH 262/269] matrix: Don't send out read markers with an empty event id. --- main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index e4ec72f..1790b75 100644 --- a/main.py +++ b/main.py @@ -490,9 +490,13 @@ def buffer_switch_cb(_, _signal, buffer_ptr): if room_buffer.should_send_read_marker: event_id = room_buffer.last_event_id - server.room_send_read_marker( - room_buffer.room.room_id, event_id) - room_buffer.last_read_event = event_id + + # A buffer may not have any events, in that case no event id is + # here returned + if event_id: + server.room_send_read_marker( + room_buffer.room.room_id, event_id) + room_buffer.last_read_event = event_id if room_buffer.members_fetched: return W.WEECHAT_RC_OK From c9ddcf7e785ee208e36c144989fea5ea6b512505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 19 Jan 2019 12:37:05 +0100 Subject: [PATCH 263/269] matrix: Room display name calculation method is now a property. --- matrix/bar_items.py | 2 +- matrix/buffer.py | 2 +- matrix/server.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/matrix/bar_items.py b/matrix/bar_items.py index cfe3699..d95f447 100644 --- a/matrix/bar_items.py +++ b/matrix/bar_items.py @@ -50,7 +50,7 @@ def matrix_bar_item_name(data, item, window, buffer, extra_info): room = room_buffer.room return "{color}{name}".format( - color=W.color(color), name=room.display_name() + color=W.color(color), name=room.display_name ) if buffer == server.server_buffer: diff --git a/matrix/buffer.py b/matrix/buffer.py index 07d3284..8225bd3 100644 --- a/matrix/buffer.py +++ b/matrix/buffer.py @@ -1065,7 +1065,7 @@ class RoomBuffer(object): self.update_buffer_name() def update_buffer_name(self): - room_name = self.room.display_name() + room_name = self.room.display_name self.weechat_buffer.short_name = room_name def _redact_line(self, event): diff --git a/matrix/server.py b/matrix/server.py index 2701c07..5f333bf 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -1062,7 +1062,7 @@ class MatrixServer(object): self.info( "You have been invited to {} {}({}{}{}){}" "{}".format( - room.display_name(), + room.display_name, W.color("chat_delimiters"), W.color("chat_channel"), room_id, From 5476d8e5003e37c57a5c006cdac08308ce26a2a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 19 Jan 2019 15:45:51 +0100 Subject: [PATCH 264/269] config: Change the import for the encryption logger. --- matrix/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/config.py b/matrix/config.py index 102e16e..281058e 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -145,7 +145,7 @@ def change_log_level(category, level): elif category == "responses": nio.responses.logger.level = level elif category == "encryption": - nio.encryption.logger.level = level + nio.crypto.logger.level = level @utf8_decode From f57b1b32c01efbc5fb0d6528945cf339074c931b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sat, 19 Jan 2019 16:11:28 +0100 Subject: [PATCH 265/269] completion: Don't use the Olm class directly. --- matrix/completion.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/matrix/completion.py b/matrix/completion.py index 3792c97..d8939aa 100644 --- a/matrix/completion.py +++ b/matrix/completion.py @@ -19,6 +19,7 @@ from __future__ import unicode_literals from matrix.globals import SERVERS, W from matrix.utf import utf8_decode from matrix.utils import tags_from_line_data +from nio import LocalProtocolError def add_servers_to_completion(completion): @@ -154,12 +155,12 @@ def matrix_olm_user_completion_cb(data, completion_item, buffer, completion): if not server: return W.WEECHAT_RC_OK - olm = server.client.olm - - if not olm: + try: + device_store = server.client.device_store + except LocalProtocolError: return W.WEECHAT_RC_OK - for user in olm.device_store.users: + for user in device_store.users: W.hook_completion_list_add( completion, user, 0, W.WEECHAT_LIST_POS_SORT ) @@ -174,9 +175,9 @@ def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): if not server: return W.WEECHAT_RC_OK - olm = server.client.olm - - if not olm: + try: + device_store = server.client.device_store + except LocalProtocolError: return W.WEECHAT_RC_OK args = W.hook_completion_get_string(completion, "args") @@ -188,10 +189,10 @@ def matrix_olm_device_completion_cb(data, completion_item, buffer, completion): user = fields[1] - if user not in olm.device_store.users: + if user not in device_store.users: return W.WEECHAT_RC_OK - for device in olm.device_store.active_user_devices(user): + for device in device_store.active_user_devices(user): W.hook_completion_list_add( completion, device.id, 0, W.WEECHAT_LIST_POS_SORT ) From a34d1e68b00549645280d6f23eeaac78f5925b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 21 Jan 2019 00:27:04 +0100 Subject: [PATCH 266/269] server: Handle case where we receive a sent message first in a sync. --- matrix/server.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index 5f333bf..a0f5bdf 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -953,7 +953,14 @@ class MatrixServer(object): room_buffer.last_read_event = response.event_id room_buffer = self.room_buffers[response.room_id] - message = room_buffer.sent_messages_queue.pop(response.uuid) + + message = room_buffer.sent_messages_queue.pop(response.uuid, None) + + # The message might have been returned in a sync response before we got + # a room send response. + if not message: + return + message.event_id = response.event_id # We already printed the message, just modify it to contain the proper # colors and formatting. From 58705661f8769519eab5702b5f0107ce800934dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 21 Jan 2019 16:05:44 +0100 Subject: [PATCH 267/269] server: Don't do full key queries anymore. --- matrix/server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/matrix/server.py b/matrix/server.py index a0f5bdf..781c731 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -904,8 +904,8 @@ class MatrixServer(object): _, request = self.client.keys_upload() self.send_or_queue(request) - def keys_query(self, full=False): - _, request = self.client.keys_query(full) + def keys_query(self): + _, request = self.client.keys_query() self.keys_queried = True self.send_or_queue(request) @@ -1310,7 +1310,8 @@ class MatrixServer(object): # We are done adding all the users, do a full key query now since # the client knows all the encrypted room members. else: - self.keys_query(True) + if self.client.should_query_keys and not self.keys_queried: + self.keys_query() elif isinstance(response, KeysClaimResponse): self.keys_claimed[response.room_id] = False From fffb43fda0800fdfbacd128a60d881fede2e120c Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 23 Jan 2019 13:45:56 +0100 Subject: [PATCH 268/269] Make reconnect delay configurable, per server. --- matrix/server.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index 2701c07..3b9724a 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -156,6 +156,15 @@ class ServerConfig(ConfigSection): "Weechat Matrix", "Device name to use while logging in to the matrix server", ), + Option( + "autoreconnect_delay", + "integer", + "", + 0, + 86400, + "10", + ("Delay (in seconds) before trying to reconnect to server"), + ), ] section = W.config_search_section(config_ptr, "server") @@ -193,6 +202,7 @@ class ServerConfig(ConfigSection): ssl_verify = ConfigSection.option_property("ssl_verify", "boolean") username = ConfigSection.option_property("username", "string") device_name = ConfigSection.option_property("device_name", "string") + reconnect_delay = ConfigSection.option_property("autoreconnect_delay", "integer") password = ConfigSection.option_property( "password", "string", evaluate=True ) @@ -452,7 +462,7 @@ class MatrixServer(object): if self.reconnect_delay: self.reconnect_delay = self.reconnect_delay * 2 else: - self.reconnect_delay = 10 + self.reconnect_delay = self.config.reconnect_delay message = ( "{prefix}matrix: reconnecting to server in {t} " "seconds" From 5228e50e2edb481af553f90faa4508df2d534d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 23 Jan 2019 16:37:26 +0100 Subject: [PATCH 269/269] server: Use atomic_write for the device id file. --- matrix/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matrix/server.py b/matrix/server.py index 4626271..404c571 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -23,6 +23,7 @@ import ssl import time import copy from collections import defaultdict, deque +from atomicwrites import atomic_write from typing import Any, Deque, Dict, Optional, List, NamedTuple, DefaultDict from uuid import UUID @@ -305,7 +306,7 @@ class MatrixServer(object): file_name = "{}{}".format(self.config.username, ".device_id") path = os.path.join(self.get_session_path(), file_name) - with open(path, "w") as device_file: + with atomic_write(path, overwrite=True) as device_file: device_file.write(self.device_id) def _change_client(self):