diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 6ed4d539e..39b6bed8c 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -50,7 +50,8 @@ ARG EXTRACTED_FILE_CAPA_VERBOSE=false ARG EXTRACTED_FILE_HTTP_SERVER_DEBUG=false ARG EXTRACTED_FILE_HTTP_SERVER_ENABLE=false ARG EXTRACTED_FILE_HTTP_SERVER_ENCRYPT=false -ARG EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined +ARG EXTRACTED_FILE_HTTP_SERVER_ZIP=false +ARG EXTRACTED_FILE_HTTP_SERVER_KEY=infected ARG EXTRACTED_FILE_HTTP_SERVER_PORT=8440 ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH @@ -90,6 +91,7 @@ ENV CAPA_BIN "${CAPA_DIR}/capa" ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT +ENV EXTRACTED_FILE_HTTP_SERVER_ZIP $EXTRACTED_FILE_HTTP_SERVER_ZIP ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT @@ -137,7 +139,16 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour python3-requests \ python3-zmq \ rsync && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir clamd supervisor yara-python python-magic psutil pycryptodome watchdog && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir \ + clamd \ + psutil \ + pycryptodome \ + pyminizip \ + python-magic \ + stream-zip \ + supervisor \ + watchdog \ + yara-python && \ curl -fsSLO "$SUPERCRONIC_URL" && \ echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \ chmod +x "$SUPERCRONIC" && \ diff --git a/config/zeek-secret.env.example b/config/zeek-secret.env.example index 8ce2739c6..ad56a4987 100644 --- a/config/zeek-secret.env.example +++ b/config/zeek-secret.env.example @@ -1,5 +1,7 @@ # A VirusTotal Public API v.20 used to submit hashes of Zeek-extracted files VTOT_API2_KEY=0 -# Specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files served over HTTP -EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined +# Specifies the password for encrypted Zeek-extracted files served over HTTP +# If EXTRACTED_FILE_HTTP_SERVER_ZIP is true this is the password for the Zip file, +# otherwise it is the AES-256-CBC decryption password +EXTRACTED_FILE_HTTP_SERVER_KEY=infected K8S_SECRET=True \ No newline at end of file diff --git a/config/zeek.env.example b/config/zeek.env.example index 7e0e64249..a70fab119 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -49,6 +49,8 @@ EXTRACTED_FILE_UPDATE_RULES=false EXTRACTED_FILE_PIPELINE_VERBOSITY= # Whether or not to serve the directory containing Zeek-extracted over HTTP at ./extracted-files/ EXTRACTED_FILE_HTTP_SERVER_ENABLE=false +# Whether or not Zeek-extracted files served over HTTP will be archived in a Zip file +EXTRACTED_FILE_HTTP_SERVER_ZIP=false # Whether or not Zeek-extracted files served over HTTP will be AES-256-CBC-encrypted EXTRACTED_FILE_HTTP_SERVER_ENCRYPT=true # Environment variables for tweaking Zeek at runtime (see local.zeek) diff --git a/docs/file-scanning.md b/docs/file-scanning.md index e7c3908c5..f373c7862 100644 --- a/docs/file-scanning.md +++ b/docs/file-scanning.md @@ -25,4 +25,4 @@ The `EXTRACTED_FILE_PRESERVATION` [environment variable in `zeek.env`](malcolm-c * `all`: preserve flagged files in `./zeek-logs/extract_files/quarantine` and all other extracted files in `./zeek-logs/extract_files/preserved` * `none`: preserve no extracted files -The `EXTRACTED_FILE_HTTP_SERVER_…` [environment variables in `zeek.env`](malcolm-config.md#MalcolmConfigEnvVars) configure access to the Zeek-extracted files path through the means of a simple HTTPS directory server. Beware that Zeek-extracted files may contain malware. As such, these files may be optionally encrypted upon download (and decrypted using `openssl`, e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`) +The `EXTRACTED_FILE_HTTP_SERVER_…` [environment variables in `zeek.env`](malcolm-config.md#MalcolmConfigEnvVars) configure access to the Zeek-extracted files path through the means of a simple HTTPS directory server. Beware that Zeek-extracted files may contain malware. As such, these files may be optionally ZIP archived (with or without a password) or encrypted (to be decrypted using `openssl`, e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`) upon download. diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 2c232ba4a..38e14b911 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -410,7 +410,9 @@ Select file preservation behavior (quarantined): 1 Expose web interface for downloading preserved files? (y / N): y -Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted): quarantined +ZIP downloaded preserved files? (y / N): y + +Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected): infected Scan extracted files with ClamAV? (Y / n): y diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 94ee44959..6f4df8ee4 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -81,8 +81,9 @@ Although the configuration script automates many of the following configuration - `EXTRACTED_FILE_ENABLE_CLAMAV` – if set to `true`, [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) will be scanned with [ClamAV](https://www.clamav.net/) - `EXTRACTED_FILE_ENABLE_YARA` – if set to `true`, [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) will be scanned with [Yara](https://github.com/VirusTotal/yara) - `EXTRACTED_FILE_HTTP_SERVER_ENABLE` – if set to `true`, the directory containing [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) will be served over HTTP at `./extracted-files/` (e.g., **https://localhost/extracted-files/** if connecting locally) - - `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` – if to `true`, the Zeek-extracted files will be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`) - - `EXTRACTED_FILE_HTTP_SERVER_KEY` – specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files; used in conjunction with `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` + - `EXTRACTED_FILE_HTTP_SERVER_ZIP` – if to `true`, the Zeek-extracted files will be archived in a ZIP file upon download + - `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` – if to `true`, the Zeek-extracted files will be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`) upon download + - `EXTRACTED_FILE_HTTP_SERVER_KEY` – specifies the password for the ZIP archive if `EXTRACTED_FILE_HTTP_SERVER_ZIP` is `true`; otherwise, this specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files if `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` is `true` - `EXTRACTED_FILE_IGNORE_EXISTING` – if set to `true`, files extant in `./zeek-logs/extract_files/` directory will be ignored on startup rather than scanned - `EXTRACTED_FILE_PRESERVATION` – determines behavior for preservation of [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) - `EXTRACTED_FILE_UPDATE_RULES` – if set to `true`, file scanner engines (e.g., ClamAV, Capa, Yara) will periodically update their rule definitions (default `false`) diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index cc756826a..61c6822bb 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -235,8 +235,10 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest + `none`: preserve no extracted files * **Expose web interface for downloading preserved files?** - Answering **Y** enables access to the Zeek-extracted files path through the means of a simple HTTPS directory server at **https:///extracted-files/**. Beware that Zeek-extracted files may contain malware. -* **Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)** - - If a password is specified here, Zeek-extracted files downloaded as described under the previous question will be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`). +* **ZIP downloaded preserved files?** + - Answering **Y** will cause that Zeek-extracted files downloaded as described under the previous question will be archived using the ZIP file format. +* **Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)** and **Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)** + - A non-blank value will be used as either the ZIP archive file password (if the previous question was answered **Y**) or as the encryption key for the file to be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`). * **Scan extracted files with ClamAV?** - Answer **Y** to scan extracted files with [ClamAV](https://www.clamav.net/), an antivirus engine. * **Scan extracted files with Yara?** diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 4e2d0c652..c7fe7ea91 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -169,7 +169,9 @@ Select file preservation behavior (quarantined): 1 Expose web interface for downloading preserved files? (y / N): y -Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted): decryptme +ZIP downloaded preserved files? (y / N): y + +Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected): infected Scan extracted files with ClamAV? (y / N): y diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf index 78cf4d79b..979612f2b 100644 --- a/file-monitor/supervisord.conf +++ b/file-monitor/supervisord.conf @@ -152,6 +152,7 @@ redirect_stderr=true [program:fileserve] command=/usr/local/bin/zeek_carved_http_server.py --port %(ENV_EXTRACTED_FILE_HTTP_SERVER_PORT)s + --zip %(ENV_EXTRACTED_FILE_HTTP_SERVER_ZIP)s --encrypt %(ENV_EXTRACTED_FILE_HTTP_SERVER_ENCRYPT)s --directory /zeek/extract_files autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s diff --git a/scripts/install.py b/scripts/install.py index 0b341bcae..d7cda08cf 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1197,6 +1197,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): clamAvScan = False fileScanRuleUpdate = False fileCarveHttpServer = False + fileCarveHttpServerZip = False fileCarveHttpServeEncryptKey = '' if InstallerYesOrNo('Enable file extraction with Zeek?', default=bool(fileCarveModeDefault)): @@ -1229,8 +1230,13 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'Expose web interface for downloading preserved files?', default=args.fileCarveHttpServer ) if fileCarveHttpServer: + fileCarveHttpServerZip = InstallerYesOrNo( + 'ZIP downloaded preserved files?', default=args.fileCarveHttpServerZip + ) fileCarveHttpServeEncryptKey = InstallerAskForString( - 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)', + 'Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)' + if fileCarveHttpServerZip + else 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)', default=args.fileCarveHttpServeEncryptKey, ) if fileCarveMode is not None: @@ -1774,11 +1780,19 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'EXTRACTED_FILE_HTTP_SERVER_ENABLE', TrueOrFalseNoQuote(fileCarveHttpServer), ), + # ZIP HTTP server for extracted files + EnvValue( + os.path.join(args.configDir, 'zeek.env'), + 'EXTRACTED_FILE_HTTP_SERVER_ZIP', + TrueOrFalseNoQuote(fileCarveHttpServerZip), + ), # encrypt HTTP server for extracted files EnvValue( os.path.join(args.configDir, 'zeek.env'), 'EXTRACTED_FILE_HTTP_SERVER_ENCRYPT', - TrueOrFalseNoQuote(fileCarveHttpServer and (len(fileCarveHttpServeEncryptKey) > 0)), + TrueOrFalseNoQuote( + fileCarveHttpServer and (len(fileCarveHttpServeEncryptKey) > 0) and (not fileCarveHttpServerZip) + ), ), # key for encrypted HTTP-served extracted files (' -> '' for escaping in YAML) EnvValue( @@ -3703,6 +3717,16 @@ def main(): default=False, help='Expose web interface for downloading preserved files', ) + fileCarveArgGroup.add_argument( + '--extracted-file-server-zip', + dest='fileCarveHttpServerZip', + type=str2bool, + metavar="true|false", + nargs='?', + const=True, + default=False, + help='ZIP downloaded preserved files', + ) fileCarveArgGroup.add_argument( '--extracted-file-server-password', dest='fileCarveHttpServeEncryptKey', @@ -3710,7 +3734,7 @@ def main(): metavar='', type=str, default='', - help='AES-256-CBC encryption password for downloaded preserved files (blank for unencrypted)', + help='ZIP archive or AES-256-CBC encryption password for downloaded preserved files (blank for unencrypted)', ) fileCarveArgGroup.add_argument( '--extracted-file-clamav', diff --git a/shared/bin/zeek_carved_http_server.py b/shared/bin/zeek_carved_http_server.py index 8b385c9de..2c0707a09 100755 --- a/shared/bin/zeek_carved_http_server.py +++ b/shared/bin/zeek_carved_http_server.py @@ -9,14 +9,26 @@ import argparse import hashlib import os +import pyminizip import sys -from threading import Thread -from socketserver import ThreadingMixIn -from http.server import HTTPServer, SimpleHTTPRequestHandler from Crypto.Cipher import AES +from datetime import datetime +from http.server import HTTPServer, SimpleHTTPRequestHandler +from socketserver import ThreadingMixIn +from stat import S_IFREG +from stream_zip import ZIP_32, stream_zip +from threading import Thread -from malcolm_utils import str2bool, eprint, EVP_KEY_SIZE, PKCS5_SALT_LEN, OPENSSL_ENC_MAGIC, EVP_BytesToKey +from malcolm_utils import ( + str2bool, + eprint, + temporary_filename, + EVP_KEY_SIZE, + PKCS5_SALT_LEN, + OPENSSL_ENC_MAGIC, + EVP_BytesToKey, +) ################################################################################################### args = None @@ -26,6 +38,19 @@ orig_path = os.getcwd() +################################################################################################### +# +def LocalFilesForZip(names): + now = datetime.now() + + def contents(name): + with open(name, 'rb') as f: + while chunk := f.read(65536): + yield chunk + + return ((os.path.join('.', os.path.basename(name)), now, S_IFREG | 0o600, ZIP_32, contents(name)) for name in names) + + ################################################################################################### # class HTTPHandler(SimpleHTTPRequestHandler): @@ -43,13 +68,33 @@ def do_GET(self): fullpath = self.translate_path(self.path) - if (not args.encrypt) or os.path.isdir(fullpath): - # unencrypted, just use default implementation + if os.path.isdir(fullpath): + # directory listing SimpleHTTPRequestHandler.do_GET(self) - else: - # encrypt file transfers - if os.path.isfile(fullpath) or os.path.islink(fullpath): + elif os.path.isfile(fullpath) or os.path.islink(fullpath): + if args.zip: + # ZIP file + self.send_response(200) + self.send_header('Content-type', "application/zip") + self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.zip') + self.end_headers() + + if args.encrypt: + # password-protected ZIP file (temporarily persisted to disk) + with temporary_filename(suffix='.zip') as tmpFileName: + pyminizip.compress(fullpath, None, tmpFileName, args.key, 1) + with open(tmpFileName, 'rb') as f: + while chunk := f.read(65536): + self.wfile.write(chunk) + + else: + # encrypted ZIP file (streamed) + for chunk in stream_zip(LocalFilesForZip([fullpath])): + self.wfile.write(chunk) + + elif args.encrypt: + # encrypted file self.send_response(200) self.send_header('Content-type', 'application/octet-stream') self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.encrypted') @@ -73,7 +118,11 @@ def do_GET(self): break else: - self.send_error(404, "Not Found") + # unencrypted file + SimpleHTTPRequestHandler.do_GET(self) + + else: + self.send_error(404, "Not Found") ################################################################################################### @@ -101,8 +150,9 @@ def main(): defaultDebug = os.getenv('EXTRACTED_FILE_HTTP_SERVER_DEBUG', 'false') defaultEncrypt = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ENCRYPT', 'false') + defaultZip = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ZIP', 'false') defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440)) - defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'quarantined') + defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'infected') defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path) parser = argparse.ArgumentParser( @@ -146,7 +196,7 @@ def main(): const=True, default=defaultEncrypt, metavar='true|false', - help=f"Encrypt files with aes-256-cbc ({defaultEncrypt})", + help=f"Encrypt files (with -z/--zip, or with aes-256-cbc) ({defaultEncrypt})", ) parser.add_argument( '-k', @@ -157,6 +207,17 @@ def main(): type=str, default=defaultKey, ) + parser.add_argument( + '-z', + '--zip', + dest='zip', + type=str2bool, + nargs='?', + const=True, + default=defaultZip, + metavar='true|false', + help=f"Zip file ({defaultZip})", + ) try: parser.error = parser.exit args = parser.parse_args()