diff --git a/contrib/matrix_upload b/contrib/matrix_upload new file mode 100755 index 0000000..b4fd722 --- /dev/null +++ b/contrib/matrix_upload @@ -0,0 +1,245 @@ +#!/usr/bin/python3 -u +# Copyright © 2018 Damir Jelić +# +# Permission to use, copy, modify, and/or distribute this software for +# any purpose with or without fee is hereby granted, provided that the +# above copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +import os +import json +import magic +import requests +import argparse +from urllib.parse import urlparse +import urllib3 + +from nio import Api, UploadResponse, UploadError +from json.decoder import JSONDecodeError + +urllib3.disable_warnings() + +mime = magic.Magic(mime=True) + + +class Upload(object): + def __init__(self, file, chunksize=1 << 13): + self.file = file + self.filename = os.path.basename(file) + self.chunksize = chunksize + self.totalsize = os.path.getsize(file) + self.mimetype = mime.from_file(file) + self.readsofar = 0 + + def send_progress(self): + message = { + "type": "progress", + "data": self.readsofar + } + to_stdout(message) + + def __iter__(self): + with open(self.filename, 'rb') as file: + while True: + data = file.read(self.chunksize) + + if not data: + break + + self.readsofar += len(data) + self.send_progress() + + yield data + + def __len__(self): + return self.totalsize + + +class IterableToFileAdapter(object): + def __init__(self, iterable): + self.iterator = iter(iterable) + self.length = len(iterable) + + def read(self, size=-1): + return next(self.iterator, b'') + + def __len__(self): + return self.length + + +def to_stdout(message): + print(json.dumps(message), flush=True) + + +def error(e): + message = { + "type": "status", + "status": "error", + "message": str(e) + } + to_stdout(message) + os.sys.exit() + + +def upload_process(args): + file_path = os.path.expanduser(args.file) + + try: + upload = Upload(file_path, 10) + except (FileNotFoundError, OSError, IOError) as e: + error(e) + + try: + url = urlparse(args.homeserver) + except ValueError as e: + error(e) + + upload_url = ("https://{}".format(args.homeserver) + if not url.scheme else args.homeserver) + _, api_path, _ = Api.upload(args.access_token, upload.filename) + upload_url += api_path + + headers = { + "Content-type": upload.mimetype, + } + + proxies = {} + + if args.proxy_address: + user = args.proxy_user or "" + + if args.proxy_password: + user += ":{}".format(args.proxy_password) + + if user: + user += "@" + + proxies = { + "https": "{}://{}{}:{}/".format( + args.proxy_type, + user, + args.proxy_address, + args.proxy_port + ) + } + + message = { + "type": "status", + "status": "started", + "total": upload.totalsize, + "mimetype": upload.mimetype, + "file_name": upload.filename, + } + to_stdout(message) + + session = requests.Session() + session.trust_env = False + + try: + r = session.post( + url=upload_url, + auth=None, + headers=headers, + data=IterableToFileAdapter(upload), + verify=(not args.insecure), + proxies=proxies + ) + except (requests.exceptions.RequestException, OSError) as e: + error(e) + + try: + json_response = json.loads(r.content) + except JSONDecodeError: + error(r.content) + + response = UploadResponse.from_dict(json_response) + + if isinstance(response, UploadError): + error(str(response)) + + message = { + "type": "status", + "status": "done", + "url": response.content_uri + } + + to_stdout(message) + + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Download and decrypt matrix attachments" + ) + parser.add_argument("file", help="the file that will be uploaded") + parser.add_argument( + "homeserver", + type=str, + help="the address of the homeserver" + ) + parser.add_argument( + "access_token", + type=str, + help="the access token to use for the upload" + ) + parser.add_argument( + "--encrypt", + action="store_const", + const=True, + default=False, + help="encrypt the file before uploading it" + ) + parser.add_argument( + "--insecure", + action="store_const", + const=True, + default=False, + help="disable SSL certificate verification" + ) + parser.add_argument( + "--proxy-type", + choices=[ + "http", + "socks4", + "socks5" + ], + default="http", + help="type of the proxy that will be used to establish a connection" + ) + parser.add_argument( + "--proxy-address", + type=str, + help="address of the proxy that will be used to establish a connection" + ) + parser.add_argument( + "--proxy-port", + type=int, + default=8080, + help="port of the proxy that will be used to establish a connection" + ) + parser.add_argument( + "--proxy-user", + type=str, + help="user that will be used for authentication on the proxy" + ) + parser.add_argument( + "--proxy-password", + type=str, + help="password that will be used for authentication on the proxy" + ) + + args = parser.parse_args() + upload_process(args) + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py index ce4b090..e4ec72f 100644 --- a/main.py +++ b/main.py @@ -53,7 +53,8 @@ from matrix.commands import (hook_commands, hook_page_up, matrix_me_command_cb, matrix_part_command_cb, matrix_redact_command_cb, matrix_topic_command_cb, matrix_olm_command_cb, matrix_devices_command_cb, - matrix_room_command_cb) + matrix_room_command_cb, matrix_uploads_command_cb, + matrix_upload_command_cb) from matrix.completion import (init_completion, matrix_command_completion_cb, matrix_debug_completion_cb, matrix_message_completion_cb, @@ -76,6 +77,8 @@ from matrix.server import (MatrixServer, create_default_server, from matrix.utf import utf8_decode from matrix.utils import server_buffer_prnt, server_buffer_set_title +from matrix.uploads import UploadsBuffer, upload_cb + # yapf: disable WEECHAT_SCRIPT_NAME = SCRIPT_NAME WEECHAT_SCRIPT_DESCRIPTION = "matrix chat plugin" # type: str diff --git a/matrix/commands.py b/matrix/commands.py index 14f2b5b..0b6782c 100644 --- a/matrix/commands.py +++ b/matrix/commands.py @@ -23,10 +23,11 @@ from collections import defaultdict from . import globals as G from .colors import Formatted -from .globals import SERVERS, W +from .globals import SERVERS, W, UPLOADS from .server import MatrixServer from .utf import utf8_decode from .utils import key_from_value, tags_from_line_data +from .uploads import UploadsBuffer, Upload class ParseError(Exception): @@ -159,6 +160,23 @@ class WeechatCommandParser(object): return WeechatCommandParser._run_parser(parser, args) + @staticmethod + def uploads(args): + parser = WeechatArgParse(prog="uploads") + subparsers = parser.add_subparsers(dest="subcommand") + subparsers.add_parser("list") + subparsers.add_parser("listfull") + subparsers.add_parser("up") + subparsers.add_parser("down") + + return WeechatCommandParser._run_parser(parser, args) + + @staticmethod + def upload(args): + parser = WeechatArgParse(prog="upload") + parser.add_argument("file") + return WeechatCommandParser._run_parser(parser, args) + def grouper(iterable, n, fillvalue=None): "Collect data into fixed-length chunks or blocks" @@ -396,6 +414,39 @@ def hook_commands(): "", ) + # W.hook_command( + # # Command name and short description + # "uploads", + # "Open the uploads buffer or list uploads in the core buffer", + # # Synopsis + # ("list||" + # "listfull" + # ), + # # Description + # (""), + # # Completions + # ("list ||" + # "listfull"), + # # Callback + # "matrix_uploads_command_cb", + # "", + # ) + + W.hook_command( + # Command name and short description + "upload", + "Upload a file to a room", + # Synopsis + (""), + # Description + (""), + # Completions + ("%(filename)"), + # Callback + "matrix_upload_command_cb", + "", + ) + W.hook_command_run("/buffer clear", "matrix_command_buf_clear_cb", "") if G.CONFIG.network.fetch_backlog_on_pgup: @@ -936,6 +987,72 @@ def matrix_room_command_cb(data, buffer, args): return W.WEECHAT_RC_OK +@utf8_decode +def matrix_uploads_command_cb(data, buffer, args): + if not args: + if not G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer = UploadsBuffer() + G.CONFIG.upload_buffer.display() + return W.WEECHAT_RC_OK + + parsed_args = WeechatCommandParser.uploads(args) + if not parsed_args: + return W.WEECHAT_RC_OK + + if parsed_args.subcommand == "list": + pass + elif parsed_args.subcommand == "listfull": + pass + elif parsed_args.subcommand == "up": + if G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer.move_line_up() + elif parsed_args.subcommand == "down": + if G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer.move_line_down() + + return W.WEECHAT_RC_OK + + +@utf8_decode +def matrix_upload_command_cb(data, buffer, args): + parsed_args = WeechatCommandParser.upload(args) + if not parsed_args: + return W.WEECHAT_RC_OK + + for server in SERVERS.values(): + if buffer == server.server_buffer: + server.error( + 'command "upload" must be ' "executed on a Matrix room buffer" + ) + return W.WEECHAT_RC_OK + + room_buffer = server.find_room_from_ptr(buffer) + if not room_buffer: + continue + + if room_buffer.room.encrypted: + room_buffer.error("Uploading to encrypted rooms is " + "not yet implemented") + return W.WEECHAT_RC_OK + + upload = Upload( + server.name, + server.config.address, + server.client.access_token, + room_buffer.room.room_id, + parsed_args.file, + room_buffer.room.encrypted + ) + UPLOADS[upload.uuid] = upload + + if G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer.render() + + break + + return W.WEECHAT_RC_OK + + @utf8_decode def matrix_kick_command_cb(data, buffer, args): parsed_args = WeechatCommandParser.kick(args) diff --git a/matrix/config.py b/matrix/config.py index 8914938..9d2aeb1 100644 --- a/matrix/config.py +++ b/matrix/config.py @@ -395,6 +395,7 @@ class MatrixConfig(WeechatConfig): def __init__(self): self.debug_buffer = "" + self.upload_buffer = "" self.debug_category = "all" self.page_up_hook = None diff --git a/matrix/globals.py b/matrix/globals.py index 8958203..5950d9e 100644 --- a/matrix/globals.py +++ b/matrix/globals.py @@ -19,12 +19,14 @@ from __future__ import unicode_literals import sys from typing import Dict, Optional from logbook import Logger +from collections import OrderedDict from .utf import WeechatWrapper if False: from .server import MatrixServer from .config import MatrixConfig + from .uploads import Upload try: @@ -43,3 +45,4 @@ SCRIPT_NAME = "matrix" # type: str MAX_EVENTS = 100 TYPING_NOTICE_TIMEOUT = 4000 # 4 seconds typing notice lifetime LOGGER = Logger("weechat-matrix") +UPLOADS = OrderedDict() # type: Dict[str, Upload] diff --git a/matrix/server.py b/matrix/server.py index aed054a..016c5bd 100644 --- a/matrix/server.py +++ b/matrix/server.py @@ -28,6 +28,7 @@ from typing import Any, Deque, Dict, Optional, List, NamedTuple, DefaultDict from uuid import UUID from nio import ( + Api, HttpClient, LocalProtocolError, LoginResponse, @@ -746,18 +747,70 @@ class MatrixServer(object): room_buffer.typing = True self.send(request) + def room_send_upload( + self, + upload + ): + """Send a room message containing the mxc URI of an upload.""" + try: + room_buffer = self.find_room_from_id(upload.room_id) + except (ValueError, KeyError): + return False - def _room_send_message( + assert self.client + + if room_buffer.room.encrypted: + room_buffer.error("Uploading to encrypted rooms is " + "not yet implemented") + return False + + # TODO the content is different if the room is encrypted. + content = { + "msgtype": Api.mimetype_to_msgtype(upload.mimetype), + "body": upload.file_name, + "url": upload.content_uri, + } + + try: + uuid = self.room_send_event(upload.room_id, content) + except (EncryptionError, GroupEncryptionError): + # TODO put the message in a queue to resend after group sessions + # are shared + # message = EncrytpionQueueItem(msgtype, formatted) + # self.encryption_queue[room.room_id].append(message) + return False + + http_url = Api.mxc_to_http(upload.content_uri) + description = ("/{}".format(upload.file_name) if upload.file_name + else "") + + attributes = DEFAULT_ATTRIBUTES.copy() + formatted = Formatted([FormattedString( + "{url}{desc}".format(url=http_url, desc=description), + attributes + )]) + + own_message = OwnMessage( + self.user_id, 0, "", uuid, upload.room_id, formatted + ) + + room_buffer.sent_messages_queue[uuid] = own_message + self.print_unconfirmed_message(room_buffer, own_message) + + return True + + def room_send_event( self, room_id, # type: str content, # type: Dict[str, str] + event_type="m.room.message" ): # type: (...) -> UUID assert self.client try: uuid, request = self.client.room_send( - room_id, "m.room.message", content + room_id, event_type, content ) self.send(request) return uuid @@ -794,7 +847,7 @@ class MatrixServer(object): content["formatted_body"] = formatted.to_html() try: - uuid = self._room_send_message(room.room_id, content) + uuid = self.room_send_event(room.room_id, content) except (EncryptionError, GroupEncryptionError): message = EncrytpionQueueItem(msgtype, formatted) self.encryption_queue[room.room_id].append(message) diff --git a/matrix/uploads.py b/matrix/uploads.py new file mode 100644 index 0000000..819867c --- /dev/null +++ b/matrix/uploads.py @@ -0,0 +1,340 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2018 Damir Jelić +# +# Permission to use, copy, modify, and/or distribute this software for +# any purpose with or without fee is hereby granted, provided that the +# above copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Module implementing upload functionality.""" + +from __future__ import unicode_literals + +import attr +import time +import json +from uuid import uuid1, UUID +from enum import Enum + +try: + from json.decoder import JSONDecodeError +except ImportError: + JSONDecodeError = ValueError # type: ignore + +from .globals import SCRIPT_NAME, SERVERS, W, UPLOADS +from .utf import utf8_decode +from matrix import globals as G + + +class UploadState(Enum): + created = 0 + active = 1 + finished = 2 + error = 3 + aborted = 4 + + +@attr.s +class Proxy(object): + ptr = attr.ib(type=str) + + @property + def name(self): + return W.infolist_string(self.ptr, "name") + + @property + def address(self): + return W.infolist_string(self.ptr, "address") + + @property + def type(self): + return W.infolist_string(self.ptr, "type_string") + + @property + def port(self): + return str(W.infolist_integer(self.ptr, "port")) + + @property + def user(self): + return W.infolist_string(self.ptr, "username") + + @property + def password(self): + return W.infolist_string(self.ptr, "password") + + +@attr.s +class Upload(object): + """Class representing an upload to a matrix server.""" + + server_name = attr.ib(type=str) + server_address = attr.ib(type=str) + access_token = attr.ib(type=str) + room_id = attr.ib(type=str) + filepath = attr.ib(type=str) + encrypt = attr.ib(type=bool, default=False) + + done = 0 + total = 0 + + uuid = None + buffer = None + upload_hook = None + content_uri = None + file_name = None + mimetype = "?" + state = UploadState.created + + def __attrs_post_init__(self): + self.uuid = uuid1() + self.buffer = "" + + server = SERVERS[self.server_name] + + proxy_name = server.config.proxy + proxy = None + proxies_list = None + + if proxy_name: + proxies_list = W.infolist_get("proxy", "", proxy_name) + if proxies_list: + W.infolist_next(proxies_list) + proxy = Proxy(proxies_list) + + process_args = { + "arg1": self.filepath, + "arg2": self.server_address, + "arg3": self.access_token, + "buffer_flush": "1", + } + + arg_count = 3 + + if self.encrypt: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--encrypt" + + if not server.config.ssl_verify: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--insecure" + + if proxy: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-type" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.type + + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-address" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.address + + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-port" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.port + + if proxy.user: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-user" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.user + + if proxy.password: + arg_count += 1 + process_args["arg{}".format(arg_count)] = "--proxy-password" + arg_count += 1 + process_args["arg{}".format(arg_count)] = proxy.password + + self.upload_hook = W.hook_process_hashtable( + "matrix_upload", + process_args, + 0, + "upload_cb", + str(self.uuid) + ) + + if proxies_list: + W.infolist_free(proxies_list) + + def abort(self): + pass + + +@attr.s +class UploadsBuffer(object): + """Weechat buffer showing the uploads for a server.""" + + _ptr = "" # type: str + _selected_line = 0 # type: int + uploads = UPLOADS + + def __attrs_post_init__(self): + self._ptr = W.buffer_new( + SCRIPT_NAME + ".uploads", + "", + "", + "", + "", + ) + W.buffer_set(self._ptr, "type", "free") + W.buffer_set(self._ptr, "title", "Upload list") + W.buffer_set(self._ptr, "key_bind_meta2-A", "/uploads up") + W.buffer_set(self._ptr, "key_bind_meta2-B", "/uploads down") + W.buffer_set(self._ptr, "localvar_set_type", "uploads") + + self.render() + + def move_line_up(self): + self._selected_line = max(self._selected_line - 1, 0) + self.render() + + def move_line_down(self): + self._selected_line = min( + self._selected_line + 1, + len(self.uploads) - 1 + ) + self.render() + + def display(self): + """Display the buffer.""" + W.buffer_set(self._ptr, "display", "1") + + def render(self): + """Render the new state of the upload buffer.""" + # This function is under the MIT license. + # Copyright (c) 2016 Vladimir Ignatev + def progress(count, total): + bar_len = 60 + + if total == 0: + bar = '-' * bar_len + return "[{}] {}%".format(bar, "?") + + filled_len = int(round(bar_len * count / float(total))) + percents = round(100.0 * count / float(total), 1) + bar = '=' * filled_len + '-' * (bar_len - filled_len) + + return "[{}] {}%".format(bar, percents) + + W.buffer_clear(self._ptr) + header = "{}{}{}{}{}{}{}{}".format( + W.color("green"), + "Actions (letter+enter):", + W.color("lightgreen"), + " [A] Accept", + " [C] Cancel", + " [R] Remove", + " [P] Purge finished", + " [Q] Close this buffer" + ) + W.prnt_y(self._ptr, 0, header) + + for line_number, upload in enumerate(self.uploads.values()): + line_color = "{},{}".format( + "white" if line_number == self._selected_line else "default", + "blue" if line_number == self._selected_line else "default", + ) + first_line = ("%s%s %-24s %s%s%s %s (%s.%s)" % ( + W.color(line_color), + "*** " if line_number == self._selected_line else " ", + upload.room_id, + "\"", + upload.filepath, + "\"", + upload.mimetype, + SCRIPT_NAME, + upload.server_name, + )) + W.prnt_y(self._ptr, (line_number * 2) + 2, first_line) + + status_color = "{},{}".format("green", "blue") + status = "{}{}{}".format( + W.color(status_color), + upload.state.name, + W.color(line_color) + ) + + second_line = ("{color}{prefix} {status} {progressbar} " + "{done} / {total}").format( + color=W.color(line_color), + prefix="*** " if line_number == self._selected_line else " ", + status=status, + progressbar=progress(upload.done, upload.total), + done=W.string_format_size(upload.done), + total=W.string_format_size(upload.total)) + + W.prnt_y(self._ptr, (line_number * 2) + 3, second_line) + + +def find_upload(uuid): + return UPLOADS.get(uuid, None) + + +def handle_child_message(upload, message): + if message["type"] == "progress": + upload.done = message["data"] + + elif message["type"] == "status": + if message["status"] == "started": + upload.state = UploadState.active + upload.total = message["total"] + upload.mimetype = message["mimetype"] + upload.file_name = message["file_name"] + + elif message["status"] == "done": + upload.state = UploadState.finished + upload.content_uri = message["url"] + + server = SERVERS.get(upload.server_name, None) + + if not server: + return + + server.room_send_upload(upload) + + elif message["status"] == "error": + upload.state = UploadState.error + + if G.CONFIG.upload_buffer: + G.CONFIG.upload_buffer.render() + + +@utf8_decode +def upload_cb(data, command, return_code, out, err): + upload = find_upload(UUID(data)) + + if not upload: + return W.WEECHAT_RC_OK + + if return_code == W.WEECHAT_HOOK_PROCESS_ERROR: + W.prnt("", "Error with command '%s'" % command) + return W.WEECHAT_RC_OK + + if err != "": + W.prnt("", "Error with command '%s'" % err) + upload.state = UploadState.error + + if out != "": + upload.buffer += out + messages = upload.buffer.split("\n") + upload.buffer = "" + + for m in messages: + try: + message = json.loads(m) + except (JSONDecodeError, TypeError): + upload.buffer += m + continue + + handle_child_message(upload, message) + + return W.WEECHAT_RC_OK