matrix: Add initial upload support.

This commit is contained in:
Damir Jelić 2018-12-19 17:56:28 +01:00
parent 7e0215702c
commit 4eb1d52d81
7 changed files with 767 additions and 5 deletions

245
contrib/matrix_upload Executable file
View file

@ -0,0 +1,245 @@
#!/usr/bin/python3 -u
# Copyright © 2018 Damir Jelić <poljar@termina.org.uk>
#
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
import os
import json
import magic
import requests
import argparse
from urllib.parse import urlparse
import urllib3
from nio import Api, UploadResponse, UploadError
from json.decoder import JSONDecodeError
urllib3.disable_warnings()
mime = magic.Magic(mime=True)
class Upload(object):
def __init__(self, file, chunksize=1 << 13):
self.file = file
self.filename = os.path.basename(file)
self.chunksize = chunksize
self.totalsize = os.path.getsize(file)
self.mimetype = mime.from_file(file)
self.readsofar = 0
def send_progress(self):
message = {
"type": "progress",
"data": self.readsofar
}
to_stdout(message)
def __iter__(self):
with open(self.filename, 'rb') as file:
while True:
data = file.read(self.chunksize)
if not data:
break
self.readsofar += len(data)
self.send_progress()
yield data
def __len__(self):
return self.totalsize
class IterableToFileAdapter(object):
def __init__(self, iterable):
self.iterator = iter(iterable)
self.length = len(iterable)
def read(self, size=-1):
return next(self.iterator, b'')
def __len__(self):
return self.length
def to_stdout(message):
print(json.dumps(message), flush=True)
def error(e):
message = {
"type": "status",
"status": "error",
"message": str(e)
}
to_stdout(message)
os.sys.exit()
def upload_process(args):
file_path = os.path.expanduser(args.file)
try:
upload = Upload(file_path, 10)
except (FileNotFoundError, OSError, IOError) as e:
error(e)
try:
url = urlparse(args.homeserver)
except ValueError as e:
error(e)
upload_url = ("https://{}".format(args.homeserver)
if not url.scheme else args.homeserver)
_, api_path, _ = Api.upload(args.access_token, upload.filename)
upload_url += api_path
headers = {
"Content-type": upload.mimetype,
}
proxies = {}
if args.proxy_address:
user = args.proxy_user or ""
if args.proxy_password:
user += ":{}".format(args.proxy_password)
if user:
user += "@"
proxies = {
"https": "{}://{}{}:{}/".format(
args.proxy_type,
user,
args.proxy_address,
args.proxy_port
)
}
message = {
"type": "status",
"status": "started",
"total": upload.totalsize,
"mimetype": upload.mimetype,
"file_name": upload.filename,
}
to_stdout(message)
session = requests.Session()
session.trust_env = False
try:
r = session.post(
url=upload_url,
auth=None,
headers=headers,
data=IterableToFileAdapter(upload),
verify=(not args.insecure),
proxies=proxies
)
except (requests.exceptions.RequestException, OSError) as e:
error(e)
try:
json_response = json.loads(r.content)
except JSONDecodeError:
error(r.content)
response = UploadResponse.from_dict(json_response)
if isinstance(response, UploadError):
error(str(response))
message = {
"type": "status",
"status": "done",
"url": response.content_uri
}
to_stdout(message)
return 0
def main():
parser = argparse.ArgumentParser(
description="Download and decrypt matrix attachments"
)
parser.add_argument("file", help="the file that will be uploaded")
parser.add_argument(
"homeserver",
type=str,
help="the address of the homeserver"
)
parser.add_argument(
"access_token",
type=str,
help="the access token to use for the upload"
)
parser.add_argument(
"--encrypt",
action="store_const",
const=True,
default=False,
help="encrypt the file before uploading it"
)
parser.add_argument(
"--insecure",
action="store_const",
const=True,
default=False,
help="disable SSL certificate verification"
)
parser.add_argument(
"--proxy-type",
choices=[
"http",
"socks4",
"socks5"
],
default="http",
help="type of the proxy that will be used to establish a connection"
)
parser.add_argument(
"--proxy-address",
type=str,
help="address of the proxy that will be used to establish a connection"
)
parser.add_argument(
"--proxy-port",
type=int,
default=8080,
help="port of the proxy that will be used to establish a connection"
)
parser.add_argument(
"--proxy-user",
type=str,
help="user that will be used for authentication on the proxy"
)
parser.add_argument(
"--proxy-password",
type=str,
help="password that will be used for authentication on the proxy"
)
args = parser.parse_args()
upload_process(args)
if __name__ == "__main__":
main()

View file

@ -53,7 +53,8 @@ from matrix.commands import (hook_commands, hook_page_up,
matrix_me_command_cb, matrix_part_command_cb,
matrix_redact_command_cb, matrix_topic_command_cb,
matrix_olm_command_cb, matrix_devices_command_cb,
matrix_room_command_cb)
matrix_room_command_cb, matrix_uploads_command_cb,
matrix_upload_command_cb)
from matrix.completion import (init_completion, matrix_command_completion_cb,
matrix_debug_completion_cb,
matrix_message_completion_cb,
@ -76,6 +77,8 @@ from matrix.server import (MatrixServer, create_default_server,
from matrix.utf import utf8_decode
from matrix.utils import server_buffer_prnt, server_buffer_set_title
from matrix.uploads import UploadsBuffer, upload_cb
# yapf: disable
WEECHAT_SCRIPT_NAME = SCRIPT_NAME
WEECHAT_SCRIPT_DESCRIPTION = "matrix chat plugin" # type: str

View file

@ -23,10 +23,11 @@ from collections import defaultdict
from . import globals as G
from .colors import Formatted
from .globals import SERVERS, W
from .globals import SERVERS, W, UPLOADS
from .server import MatrixServer
from .utf import utf8_decode
from .utils import key_from_value, tags_from_line_data
from .uploads import UploadsBuffer, Upload
class ParseError(Exception):
@ -159,6 +160,23 @@ class WeechatCommandParser(object):
return WeechatCommandParser._run_parser(parser, args)
@staticmethod
def uploads(args):
parser = WeechatArgParse(prog="uploads")
subparsers = parser.add_subparsers(dest="subcommand")
subparsers.add_parser("list")
subparsers.add_parser("listfull")
subparsers.add_parser("up")
subparsers.add_parser("down")
return WeechatCommandParser._run_parser(parser, args)
@staticmethod
def upload(args):
parser = WeechatArgParse(prog="upload")
parser.add_argument("file")
return WeechatCommandParser._run_parser(parser, args)
def grouper(iterable, n, fillvalue=None):
"Collect data into fixed-length chunks or blocks"
@ -396,6 +414,39 @@ def hook_commands():
"",
)
# W.hook_command(
# # Command name and short description
# "uploads",
# "Open the uploads buffer or list uploads in the core buffer",
# # Synopsis
# ("list||"
# "listfull"
# ),
# # Description
# (""),
# # Completions
# ("list ||"
# "listfull"),
# # Callback
# "matrix_uploads_command_cb",
# "",
# )
W.hook_command(
# Command name and short description
"upload",
"Upload a file to a room",
# Synopsis
("<file>"),
# Description
(""),
# Completions
("%(filename)"),
# Callback
"matrix_upload_command_cb",
"",
)
W.hook_command_run("/buffer clear", "matrix_command_buf_clear_cb", "")
if G.CONFIG.network.fetch_backlog_on_pgup:
@ -936,6 +987,72 @@ def matrix_room_command_cb(data, buffer, args):
return W.WEECHAT_RC_OK
@utf8_decode
def matrix_uploads_command_cb(data, buffer, args):
if not args:
if not G.CONFIG.upload_buffer:
G.CONFIG.upload_buffer = UploadsBuffer()
G.CONFIG.upload_buffer.display()
return W.WEECHAT_RC_OK
parsed_args = WeechatCommandParser.uploads(args)
if not parsed_args:
return W.WEECHAT_RC_OK
if parsed_args.subcommand == "list":
pass
elif parsed_args.subcommand == "listfull":
pass
elif parsed_args.subcommand == "up":
if G.CONFIG.upload_buffer:
G.CONFIG.upload_buffer.move_line_up()
elif parsed_args.subcommand == "down":
if G.CONFIG.upload_buffer:
G.CONFIG.upload_buffer.move_line_down()
return W.WEECHAT_RC_OK
@utf8_decode
def matrix_upload_command_cb(data, buffer, args):
parsed_args = WeechatCommandParser.upload(args)
if not parsed_args:
return W.WEECHAT_RC_OK
for server in SERVERS.values():
if buffer == server.server_buffer:
server.error(
'command "upload" must be ' "executed on a Matrix room buffer"
)
return W.WEECHAT_RC_OK
room_buffer = server.find_room_from_ptr(buffer)
if not room_buffer:
continue
if room_buffer.room.encrypted:
room_buffer.error("Uploading to encrypted rooms is "
"not yet implemented")
return W.WEECHAT_RC_OK
upload = Upload(
server.name,
server.config.address,
server.client.access_token,
room_buffer.room.room_id,
parsed_args.file,
room_buffer.room.encrypted
)
UPLOADS[upload.uuid] = upload
if G.CONFIG.upload_buffer:
G.CONFIG.upload_buffer.render()
break
return W.WEECHAT_RC_OK
@utf8_decode
def matrix_kick_command_cb(data, buffer, args):
parsed_args = WeechatCommandParser.kick(args)

View file

@ -395,6 +395,7 @@ class MatrixConfig(WeechatConfig):
def __init__(self):
self.debug_buffer = ""
self.upload_buffer = ""
self.debug_category = "all"
self.page_up_hook = None

View file

@ -19,12 +19,14 @@ from __future__ import unicode_literals
import sys
from typing import Dict, Optional
from logbook import Logger
from collections import OrderedDict
from .utf import WeechatWrapper
if False:
from .server import MatrixServer
from .config import MatrixConfig
from .uploads import Upload
try:
@ -43,3 +45,4 @@ SCRIPT_NAME = "matrix" # type: str
MAX_EVENTS = 100
TYPING_NOTICE_TIMEOUT = 4000 # 4 seconds typing notice lifetime
LOGGER = Logger("weechat-matrix")
UPLOADS = OrderedDict() # type: Dict[str, Upload]

View file

@ -28,6 +28,7 @@ from typing import Any, Deque, Dict, Optional, List, NamedTuple, DefaultDict
from uuid import UUID
from nio import (
Api,
HttpClient,
LocalProtocolError,
LoginResponse,
@ -746,18 +747,70 @@ class MatrixServer(object):
room_buffer.typing = True
self.send(request)
def room_send_upload(
self,
upload
):
"""Send a room message containing the mxc URI of an upload."""
try:
room_buffer = self.find_room_from_id(upload.room_id)
except (ValueError, KeyError):
return False
def _room_send_message(
assert self.client
if room_buffer.room.encrypted:
room_buffer.error("Uploading to encrypted rooms is "
"not yet implemented")
return False
# TODO the content is different if the room is encrypted.
content = {
"msgtype": Api.mimetype_to_msgtype(upload.mimetype),
"body": upload.file_name,
"url": upload.content_uri,
}
try:
uuid = self.room_send_event(upload.room_id, content)
except (EncryptionError, GroupEncryptionError):
# TODO put the message in a queue to resend after group sessions
# are shared
# message = EncrytpionQueueItem(msgtype, formatted)
# self.encryption_queue[room.room_id].append(message)
return False
http_url = Api.mxc_to_http(upload.content_uri)
description = ("/{}".format(upload.file_name) if upload.file_name
else "")
attributes = DEFAULT_ATTRIBUTES.copy()
formatted = Formatted([FormattedString(
"{url}{desc}".format(url=http_url, desc=description),
attributes
)])
own_message = OwnMessage(
self.user_id, 0, "", uuid, upload.room_id, formatted
)
room_buffer.sent_messages_queue[uuid] = own_message
self.print_unconfirmed_message(room_buffer, own_message)
return True
def room_send_event(
self,
room_id, # type: str
content, # type: Dict[str, str]
event_type="m.room.message"
):
# type: (...) -> UUID
assert self.client
try:
uuid, request = self.client.room_send(
room_id, "m.room.message", content
room_id, event_type, content
)
self.send(request)
return uuid
@ -794,7 +847,7 @@ class MatrixServer(object):
content["formatted_body"] = formatted.to_html()
try:
uuid = self._room_send_message(room.room_id, content)
uuid = self.room_send_event(room.room_id, content)
except (EncryptionError, GroupEncryptionError):
message = EncrytpionQueueItem(msgtype, formatted)
self.encryption_queue[room.room_id].append(message)

340
matrix/uploads.py Normal file
View file

@ -0,0 +1,340 @@
# -*- coding: utf-8 -*-
# Copyright © 2018 Damir Jelić <poljar@termina.org.uk>
#
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
"""Module implementing upload functionality."""
from __future__ import unicode_literals
import attr
import time
import json
from uuid import uuid1, UUID
from enum import Enum
try:
from json.decoder import JSONDecodeError
except ImportError:
JSONDecodeError = ValueError # type: ignore
from .globals import SCRIPT_NAME, SERVERS, W, UPLOADS
from .utf import utf8_decode
from matrix import globals as G
class UploadState(Enum):
created = 0
active = 1
finished = 2
error = 3
aborted = 4
@attr.s
class Proxy(object):
ptr = attr.ib(type=str)
@property
def name(self):
return W.infolist_string(self.ptr, "name")
@property
def address(self):
return W.infolist_string(self.ptr, "address")
@property
def type(self):
return W.infolist_string(self.ptr, "type_string")
@property
def port(self):
return str(W.infolist_integer(self.ptr, "port"))
@property
def user(self):
return W.infolist_string(self.ptr, "username")
@property
def password(self):
return W.infolist_string(self.ptr, "password")
@attr.s
class Upload(object):
"""Class representing an upload to a matrix server."""
server_name = attr.ib(type=str)
server_address = attr.ib(type=str)
access_token = attr.ib(type=str)
room_id = attr.ib(type=str)
filepath = attr.ib(type=str)
encrypt = attr.ib(type=bool, default=False)
done = 0
total = 0
uuid = None
buffer = None
upload_hook = None
content_uri = None
file_name = None
mimetype = "?"
state = UploadState.created
def __attrs_post_init__(self):
self.uuid = uuid1()
self.buffer = ""
server = SERVERS[self.server_name]
proxy_name = server.config.proxy
proxy = None
proxies_list = None
if proxy_name:
proxies_list = W.infolist_get("proxy", "", proxy_name)
if proxies_list:
W.infolist_next(proxies_list)
proxy = Proxy(proxies_list)
process_args = {
"arg1": self.filepath,
"arg2": self.server_address,
"arg3": self.access_token,
"buffer_flush": "1",
}
arg_count = 3
if self.encrypt:
arg_count += 1
process_args["arg{}".format(arg_count)] = "--encrypt"
if not server.config.ssl_verify:
arg_count += 1
process_args["arg{}".format(arg_count)] = "--insecure"
if proxy:
arg_count += 1
process_args["arg{}".format(arg_count)] = "--proxy-type"
arg_count += 1
process_args["arg{}".format(arg_count)] = proxy.type
arg_count += 1
process_args["arg{}".format(arg_count)] = "--proxy-address"
arg_count += 1
process_args["arg{}".format(arg_count)] = proxy.address
arg_count += 1
process_args["arg{}".format(arg_count)] = "--proxy-port"
arg_count += 1
process_args["arg{}".format(arg_count)] = proxy.port
if proxy.user:
arg_count += 1
process_args["arg{}".format(arg_count)] = "--proxy-user"
arg_count += 1
process_args["arg{}".format(arg_count)] = proxy.user
if proxy.password:
arg_count += 1
process_args["arg{}".format(arg_count)] = "--proxy-password"
arg_count += 1
process_args["arg{}".format(arg_count)] = proxy.password
self.upload_hook = W.hook_process_hashtable(
"matrix_upload",
process_args,
0,
"upload_cb",
str(self.uuid)
)
if proxies_list:
W.infolist_free(proxies_list)
def abort(self):
pass
@attr.s
class UploadsBuffer(object):
"""Weechat buffer showing the uploads for a server."""
_ptr = "" # type: str
_selected_line = 0 # type: int
uploads = UPLOADS
def __attrs_post_init__(self):
self._ptr = W.buffer_new(
SCRIPT_NAME + ".uploads",
"",
"",
"",
"",
)
W.buffer_set(self._ptr, "type", "free")
W.buffer_set(self._ptr, "title", "Upload list")
W.buffer_set(self._ptr, "key_bind_meta2-A", "/uploads up")
W.buffer_set(self._ptr, "key_bind_meta2-B", "/uploads down")
W.buffer_set(self._ptr, "localvar_set_type", "uploads")
self.render()
def move_line_up(self):
self._selected_line = max(self._selected_line - 1, 0)
self.render()
def move_line_down(self):
self._selected_line = min(
self._selected_line + 1,
len(self.uploads) - 1
)
self.render()
def display(self):
"""Display the buffer."""
W.buffer_set(self._ptr, "display", "1")
def render(self):
"""Render the new state of the upload buffer."""
# This function is under the MIT license.
# Copyright (c) 2016 Vladimir Ignatev
def progress(count, total):
bar_len = 60
if total == 0:
bar = '-' * bar_len
return "[{}] {}%".format(bar, "?")
filled_len = int(round(bar_len * count / float(total)))
percents = round(100.0 * count / float(total), 1)
bar = '=' * filled_len + '-' * (bar_len - filled_len)
return "[{}] {}%".format(bar, percents)
W.buffer_clear(self._ptr)
header = "{}{}{}{}{}{}{}{}".format(
W.color("green"),
"Actions (letter+enter):",
W.color("lightgreen"),
" [A] Accept",
" [C] Cancel",
" [R] Remove",
" [P] Purge finished",
" [Q] Close this buffer"
)
W.prnt_y(self._ptr, 0, header)
for line_number, upload in enumerate(self.uploads.values()):
line_color = "{},{}".format(
"white" if line_number == self._selected_line else "default",
"blue" if line_number == self._selected_line else "default",
)
first_line = ("%s%s %-24s %s%s%s %s (%s.%s)" % (
W.color(line_color),
"*** " if line_number == self._selected_line else " ",
upload.room_id,
"\"",
upload.filepath,
"\"",
upload.mimetype,
SCRIPT_NAME,
upload.server_name,
))
W.prnt_y(self._ptr, (line_number * 2) + 2, first_line)
status_color = "{},{}".format("green", "blue")
status = "{}{}{}".format(
W.color(status_color),
upload.state.name,
W.color(line_color)
)
second_line = ("{color}{prefix} {status} {progressbar} "
"{done} / {total}").format(
color=W.color(line_color),
prefix="*** " if line_number == self._selected_line else " ",
status=status,
progressbar=progress(upload.done, upload.total),
done=W.string_format_size(upload.done),
total=W.string_format_size(upload.total))
W.prnt_y(self._ptr, (line_number * 2) + 3, second_line)
def find_upload(uuid):
return UPLOADS.get(uuid, None)
def handle_child_message(upload, message):
if message["type"] == "progress":
upload.done = message["data"]
elif message["type"] == "status":
if message["status"] == "started":
upload.state = UploadState.active
upload.total = message["total"]
upload.mimetype = message["mimetype"]
upload.file_name = message["file_name"]
elif message["status"] == "done":
upload.state = UploadState.finished
upload.content_uri = message["url"]
server = SERVERS.get(upload.server_name, None)
if not server:
return
server.room_send_upload(upload)
elif message["status"] == "error":
upload.state = UploadState.error
if G.CONFIG.upload_buffer:
G.CONFIG.upload_buffer.render()
@utf8_decode
def upload_cb(data, command, return_code, out, err):
upload = find_upload(UUID(data))
if not upload:
return W.WEECHAT_RC_OK
if return_code == W.WEECHAT_HOOK_PROCESS_ERROR:
W.prnt("", "Error with command '%s'" % command)
return W.WEECHAT_RC_OK
if err != "":
W.prnt("", "Error with command '%s'" % err)
upload.state = UploadState.error
if out != "":
upload.buffer += out
messages = upload.buffer.split("\n")
upload.buffer = ""
for m in messages:
try:
message = json.loads(m)
except (JSONDecodeError, TypeError):
upload.buffer += m
continue
handle_child_message(upload, message)
return W.WEECHAT_RC_OK