From f39c58e15a487153a3323235434d3f0bea62b8cf Mon Sep 17 00:00:00 2001 From: JC Brand Date: Sun, 13 Jan 2019 09:43:33 +0000 Subject: [PATCH 1/2] Add the ability to apply a quota to file uploads. --- README.rst | 16 ++++++++++++++-- xhu.py | 48 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 3365417..e58df50 100644 --- a/README.rst +++ b/README.rst @@ -44,10 +44,22 @@ The configuration file must contain the following keys: Allow cross-origin access to all endpoints unconditionally. This is needed to allow web clients to use the upload feature. +Setting an upload quota +----------------------- + +A quota will be enforced if it's received from the XMPP server in the PUT +request as a `q` URL parameter. + +The quota is enforced on the parent directory of the UUID directory containing +the file and metadata. + +If this parent directory is global to all users, then the quota is also global, +otherwise if this parent directory is specific to each user, then the quota is +also per user. + Issues, Bugs, Limitations ========================= -* This service **does not handle any kind of quota**. * The format in which the files are stored is **not** compatible with ``mod_http_upload`` -- so you'll lose all uploaded files when switching. * This blindly trusts the clients Content-Type. I don't think this is a major issue, because we also tell the browser to blindly trust the clients MIME type. This, in addition with forcing all but a white list of MIME types to be downloaded instead of shown inline, should provide safety against any type of XSS attacks. * I have no idea about web security. The headers I set may be subtly wrong and circumvent all security measures I intend this to have. Please double-check for yourself and report if you find anything amiss. @@ -87,4 +99,4 @@ Enable systemd service:: Configure your webserver: As final step you need to point your external webserver to your xmpp-http-upload flask app. -Check the ``contrib`` directory, there is an example for nginx there. \ No newline at end of file +Check the ``contrib`` directory, there is an example for nginx there. diff --git a/xhu.py b/xhu.py index a28c5a8..37c698f 100644 --- a/xhu.py +++ b/xhu.py @@ -22,10 +22,13 @@ import contextlib import errno import fnmatch -import json import hashlib import hmac +import json +import os import pathlib +import shutil +import stat import typing import flask @@ -34,6 +37,7 @@ app.config.from_envvar("XMPP_HTTP_UPLOAD_CONFIG") application = app + if app.config['ENABLE_CORS']: from flask_cors import CORS CORS(app) @@ -49,7 +53,6 @@ def sanitized_join(path: str, root: pathlib.Path) -> pathlib.Path: def get_paths(base_path: pathlib.Path): data_file = pathlib.Path(str(base_path) + ".data") metadata_file = pathlib.Path(str(base_path) + ".meta") - return data_file, metadata_file @@ -65,12 +68,41 @@ def get_info(path: str, root: pathlib.Path) -> typing.Tuple[ path, pathlib.Path(app.config["DATA_ROOT"]), ) - data_file, metadata_file = get_paths(dest_path) - return data_file, load_metadata(metadata_file) +def apply_quota(root: pathlib.Path, quota: int): + """ Get the files, sorted by last modification date and the sum of their + sizes. + """ + if not quota: + return + + file_list = [] + total_size = 0 + # We assume a file structure whereby files are are stored inside + # uuid() directories inside the root dir and that there aren't any files in + # the root dir itself. + for uuid_dir in os.listdir(root): + for path, dirnames, filenames in os.walk(root/uuid_dir): + for name in [n for n in filenames if n.endswith('.data')]: + fp = os.path.join(path, name) + size = os.path.getsize(fp) + total_size += size + modified = os.stat(fp)[stat.ST_MTIME] + file_list.append((modified, path, name, size)) + + bytes = total_size - quota + if (bytes > 0): + # Remove files (oldest first) until we're under our quota + file_list.sort(key=lambda a: a[0]) + while (bytes >= 0): + modified, path, name, size = file_list.pop() + shutil.rmtree(path) + bytes -= size + + @contextlib.contextmanager def write_file(at: pathlib.Path): with at.open("xb") as f: @@ -135,6 +167,11 @@ def put_file(path): ) dest_path.parent.mkdir(parents=True, exist_ok=True, mode=0o770) + + quota = flask.request.args.get("q", "") + if (quota): + apply_quota(dest_path.parent.parent, int(quota)) + data_file, metadata_file = get_paths(dest_path) try: @@ -183,7 +220,8 @@ def generate_headers(response_headers, metadata_headers): response_headers["X-Content-Type-Options"] = "nosniff" response_headers["X-Frame-Options"] = "DENY" - response_headers["Content-Security-Policy"] = "default-src 'none'; frame-ancestors 'none'; sandbox" + response_headers["Content-Security-Policy"] = \ + "default-src 'none'; frame-ancestors 'none'; sandbox" @app.route("/", methods=["HEAD"]) From 6cb35559c211f80ba68c398647e1c828682a14ee Mon Sep 17 00:00:00 2001 From: JC Brand Date: Sun, 13 Jan 2019 11:53:14 +0000 Subject: [PATCH 2/2] Add the ability to log to a file --- xhu.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/xhu.py b/xhu.py index 37c698f..fa10bb3 100644 --- a/xhu.py +++ b/xhu.py @@ -25,6 +25,7 @@ import hashlib import hmac import json +import logging import os import pathlib import shutil @@ -37,6 +38,13 @@ app.config.from_envvar("XMPP_HTTP_UPLOAD_CONFIG") application = app +if app.config['LOGDIR']: + logging.basicConfig( + filename='{}/xhu.log'.format(app.config['LOGDIR']), + filemode='a', + level=logging.INFO + ) +logger = logging.getLogger(__name__) if app.config['ENABLE_CORS']: from flask_cors import CORS @@ -99,6 +107,10 @@ def apply_quota(root: pathlib.Path, quota: int): file_list.sort(key=lambda a: a[0]) while (bytes >= 0): modified, path, name, size = file_list.pop() + logger.info( + "Removing file {}/{} to maintain quota of {}" + .format(path, name, quota) + ) shutil.rmtree(path) bytes -= size