for idaholab#288, file carve download with password-protected zip file

Dockerfiles/file-monitor.Dockerfile

@@ -50,7 +50,8 @@ ARG EXTRACTED_FILE_CAPA_VERBOSE=false
 ARG EXTRACTED_FILE_HTTP_SERVER_DEBUG=false
 ARG EXTRACTED_FILE_HTTP_SERVER_ENABLE=false
 ARG EXTRACTED_FILE_HTTP_SERVER_ENCRYPT=false
-ARG EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined
+ARG EXTRACTED_FILE_HTTP_SERVER_ZIP=false
+ARG EXTRACTED_FILE_HTTP_SERVER_KEY=infected
 ARG EXTRACTED_FILE_HTTP_SERVER_PORT=8440
 
 ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
@@ -90,6 +91,7 @@ ENV CAPA_BIN "${CAPA_DIR}/capa"
 ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG
 ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE
 ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT
+ENV EXTRACTED_FILE_HTTP_SERVER_ZIP $EXTRACTED_FILE_HTTP_SERVER_ZIP
 ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY
 ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT
 
@@ -137,7 +139,16 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour
       python3-requests \
       python3-zmq \
       rsync && \
-    python3 -m pip install --break-system-packages --no-compile --no-cache-dir clamd supervisor yara-python python-magic psutil pycryptodome watchdog && \
+    python3 -m pip install --break-system-packages --no-compile --no-cache-dir \
+      clamd \
+      psutil \
+      pycryptodome \
+      pyminizip \
+      python-magic \
+      stream-zip \
+      supervisor \
+      watchdog \
+      yara-python && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
       echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
       chmod +x "$SUPERCRONIC" && \

config/zeek-secret.env.example

@@ -1,5 +1,7 @@
 # A VirusTotal Public API v.20 used to submit hashes of Zeek-extracted files
 VTOT_API2_KEY=0
-# Specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files served over HTTP
-EXTRACTED_FILE_HTTP_SERVER_KEY=quarantined
+# Specifies the password for encrypted Zeek-extracted files served over HTTP
+#   If EXTRACTED_FILE_HTTP_SERVER_ZIP is true this is the password for the Zip file,
+#   otherwise it is the AES-256-CBC decryption password
+EXTRACTED_FILE_HTTP_SERVER_KEY=infected
 K8S_SECRET=True

config/zeek.env.example

@@ -49,6 +49,8 @@ EXTRACTED_FILE_UPDATE_RULES=false
 EXTRACTED_FILE_PIPELINE_VERBOSITY=
 # Whether or not to serve the directory containing Zeek-extracted over HTTP at ./extracted-files/
 EXTRACTED_FILE_HTTP_SERVER_ENABLE=false
+# Whether or not Zeek-extracted files served over HTTP will be archived in a Zip file
+EXTRACTED_FILE_HTTP_SERVER_ZIP=false
 # Whether or not Zeek-extracted files served over HTTP will be AES-256-CBC-encrypted
 EXTRACTED_FILE_HTTP_SERVER_ENCRYPT=true
 # Environment variables for tweaking Zeek at runtime (see local.zeek)

docs/file-scanning.md

@@ -25,4 +25,4 @@ The `EXTRACTED_FILE_PRESERVATION` [environment variable in `zeek.env`](malcolm-c
 * `all`: preserve flagged files in `./zeek-logs/extract_files/quarantine` and all other extracted files in `./zeek-logs/extract_files/preserved`
 * `none`: preserve no extracted files
 
-The `EXTRACTED_FILE_HTTP_SERVER_…` [environment variables in `zeek.env`](malcolm-config.md#MalcolmConfigEnvVars) configure access to the Zeek-extracted files path through the means of a simple HTTPS directory server. Beware that Zeek-extracted files may contain malware. As such, these files may be optionally encrypted upon download (and decrypted using `openssl`, e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`)
+The `EXTRACTED_FILE_HTTP_SERVER_…` [environment variables in `zeek.env`](malcolm-config.md#MalcolmConfigEnvVars) configure access to the Zeek-extracted files path through the means of a simple HTTPS directory server. Beware that Zeek-extracted files may contain malware. As such, these files may be optionally ZIP archived (with or without a password) or encrypted (to be decrypted using `openssl`, e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`) upon download.

docs/kubernetes.md

@@ -410,7 +410,9 @@ Select file preservation behavior (quarantined): 1
 
 Expose web interface for downloading preserved files? (y / N): y
 
-Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted): quarantined
+ZIP downloaded preserved files? (y / N): y
+
+Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected): infected
 
 Scan extracted files with ClamAV? (Y / n): y
 

docs/malcolm-config.md

@@ -81,8 +81,9 @@ Although the configuration script automates many of the following configuration
     - `EXTRACTED_FILE_ENABLE_CLAMAV` – if set to `true`, [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) will be scanned with [ClamAV](https://www.clamav.net/)
     - `EXTRACTED_FILE_ENABLE_YARA` – if set to `true`, [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) will be scanned with [Yara](https://github.com/VirusTotal/yara)
     - `EXTRACTED_FILE_HTTP_SERVER_ENABLE` – if set to `true`, the directory containing [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) will be served over HTTP at `./extracted-files/` (e.g., **https://localhost/extracted-files/** if connecting locally)
-    - `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` – if to `true`, the Zeek-extracted files will be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`)
-    - `EXTRACTED_FILE_HTTP_SERVER_KEY` – specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files; used in conjunction with `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT`
+    - `EXTRACTED_FILE_HTTP_SERVER_ZIP` – if to `true`, the Zeek-extracted files will be archived in a ZIP file upon download
+    - `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` – if to `true`, the Zeek-extracted files will be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`) upon download
+    - `EXTRACTED_FILE_HTTP_SERVER_KEY` – specifies the password for the ZIP archive if `EXTRACTED_FILE_HTTP_SERVER_ZIP` is `true`; otherwise, this specifies the AES-256-CBC decryption password for encrypted Zeek-extracted files if `EXTRACTED_FILE_HTTP_SERVER_ENCRYPT` is `true`
     - `EXTRACTED_FILE_IGNORE_EXISTING` – if set to `true`, files extant in `./zeek-logs/extract_files/`  directory will be ignored on startup rather than scanned
     - `EXTRACTED_FILE_PRESERVATION` – determines behavior for preservation of [Zeek-extracted files](file-scanning.md#ZeekFileExtraction)
     - `EXTRACTED_FILE_UPDATE_RULES` – if set to `true`, file scanner engines (e.g., ClamAV, Capa, Yara) will periodically update their rule definitions (default `false`)

docs/malcolm-hedgehog-e2e-iso-install.md

@@ -235,8 +235,10 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest
         + `none`: preserve no extracted files
 * **Expose web interface for downloading preserved files?**
     - Answering **Y** enables access to the Zeek-extracted files path through the means of a simple HTTPS directory server at **https://<Malcolm host or IP address>/extracted-files/**. Beware that Zeek-extracted files may contain malware.
-* **Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)**
-    - If a password is specified here, Zeek-extracted files downloaded as described under the previous question will be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`).
+* **ZIP downloaded preserved files?**
+    - Answering **Y** will cause that Zeek-extracted files downloaded as described under the previous question will be archived using the ZIP file format.
+* **Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)** and **Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)**
+    - A non-blank value will be used as either the ZIP archive file password (if the previous question was answered **Y**) or as the encryption key for the file to be AES-256-CBC-encrypted in an `openssl enc`-compatible format (e.g., `openssl enc -aes-256-cbc -d -in example.exe.encrypted -out example.exe`).
 * **Scan extracted files with ClamAV?**
     - Answer **Y** to scan extracted files with [ClamAV](https://www.clamav.net/), an antivirus engine.
 * **Scan extracted files with Yara?**

docs/ubuntu-install-example.md

@@ -169,7 +169,9 @@ Select file preservation behavior (quarantined): 1
 
 Expose web interface for downloading preserved files? (y / N): y
 
-Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted): decryptme
+ZIP downloaded preserved files? (y / N): y
+
+Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected): infected
 
 Scan extracted files with ClamAV? (y / N): y
 

file-monitor/supervisord.conf

@@ -152,6 +152,7 @@ redirect_stderr=true
 [program:fileserve]
 command=/usr/local/bin/zeek_carved_http_server.py
   --port %(ENV_EXTRACTED_FILE_HTTP_SERVER_PORT)s
+  --zip %(ENV_EXTRACTED_FILE_HTTP_SERVER_ZIP)s
   --encrypt %(ENV_EXTRACTED_FILE_HTTP_SERVER_ENCRYPT)s
   --directory /zeek/extract_files
 autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s

scripts/install.py

@@ -1197,6 +1197,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
         clamAvScan = False
         fileScanRuleUpdate = False
         fileCarveHttpServer = False
+        fileCarveHttpServerZip = False
         fileCarveHttpServeEncryptKey = ''
 
         if InstallerYesOrNo('Enable file extraction with Zeek?', default=bool(fileCarveModeDefault)):
@@ -1229,8 +1230,13 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
                     'Expose web interface for downloading preserved files?', default=args.fileCarveHttpServer
                 )
                 if fileCarveHttpServer:
+                    fileCarveHttpServerZip = InstallerYesOrNo(
+                        'ZIP downloaded preserved files?', default=args.fileCarveHttpServerZip
+                    )
                     fileCarveHttpServeEncryptKey = InstallerAskForString(
-                        'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)',
+                        'Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)'
+                        if fileCarveHttpServerZip
+                        else 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)',
                         default=args.fileCarveHttpServeEncryptKey,
                     )
                 if fileCarveMode is not None:
@@ -1774,11 +1780,19 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
                 'EXTRACTED_FILE_HTTP_SERVER_ENABLE',
                 TrueOrFalseNoQuote(fileCarveHttpServer),
             ),
+            # ZIP HTTP server for extracted files
+            EnvValue(
+                os.path.join(args.configDir, 'zeek.env'),
+                'EXTRACTED_FILE_HTTP_SERVER_ZIP',
+                TrueOrFalseNoQuote(fileCarveHttpServerZip),
+            ),
             # encrypt HTTP server for extracted files
             EnvValue(
                 os.path.join(args.configDir, 'zeek.env'),
                 'EXTRACTED_FILE_HTTP_SERVER_ENCRYPT',
-                TrueOrFalseNoQuote(fileCarveHttpServer and (len(fileCarveHttpServeEncryptKey) > 0)),
+                TrueOrFalseNoQuote(
+                    fileCarveHttpServer and (len(fileCarveHttpServeEncryptKey) > 0) and (not fileCarveHttpServerZip)
+                ),
             ),
             # key for encrypted HTTP-served extracted files (' -> '' for escaping in YAML)
             EnvValue(
@@ -3703,14 +3717,24 @@ def main():
         default=False,
         help='Expose web interface for downloading preserved files',
     )
+    fileCarveArgGroup.add_argument(
+        '--extracted-file-server-zip',
+        dest='fileCarveHttpServerZip',
+        type=str2bool,
+        metavar="true|false",
+        nargs='?',
+        const=True,
+        default=False,
+        help='ZIP downloaded preserved files',
+    )
     fileCarveArgGroup.add_argument(
         '--extracted-file-server-password',
         dest='fileCarveHttpServeEncryptKey',
         required=False,
         metavar='<string>',
         type=str,
         default='',
-        help='AES-256-CBC encryption password for downloaded preserved files (blank for unencrypted)',
+        help='ZIP archive or AES-256-CBC encryption password for downloaded preserved files (blank for unencrypted)',
     )
     fileCarveArgGroup.add_argument(
         '--extracted-file-clamav',

shared/bin/zeek_carved_http_server.py

@@ -9,14 +9,26 @@
 import argparse
 import hashlib
 import os
+import pyminizip
 import sys
-from threading import Thread
-from socketserver import ThreadingMixIn
-from http.server import HTTPServer, SimpleHTTPRequestHandler
 from Crypto.Cipher import AES
+from datetime import datetime
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+from socketserver import ThreadingMixIn
+from stat import S_IFREG
+from stream_zip import ZIP_32, stream_zip
+from threading import Thread
 
 
-from malcolm_utils import str2bool, eprint, EVP_KEY_SIZE, PKCS5_SALT_LEN, OPENSSL_ENC_MAGIC, EVP_BytesToKey
+from malcolm_utils import (
+    str2bool,
+    eprint,
+    temporary_filename,
+    EVP_KEY_SIZE,
+    PKCS5_SALT_LEN,
+    OPENSSL_ENC_MAGIC,
+    EVP_BytesToKey,
+)
 
 ###################################################################################################
 args = None
@@ -26,6 +38,19 @@
 orig_path = os.getcwd()
 
 
+###################################################################################################
+#
+def LocalFilesForZip(names):
+    now = datetime.now()
+
+    def contents(name):
+        with open(name, 'rb') as f:
+            while chunk := f.read(65536):
+                yield chunk
+
+    return ((os.path.join('.', os.path.basename(name)), now, S_IFREG | 0o600, ZIP_32, contents(name)) for name in names)
+
+
 ###################################################################################################
 #
 class HTTPHandler(SimpleHTTPRequestHandler):
@@ -43,13 +68,33 @@ def do_GET(self):
 
         fullpath = self.translate_path(self.path)
 
-        if (not args.encrypt) or os.path.isdir(fullpath):
-            # unencrypted, just use default implementation
+        if os.path.isdir(fullpath):
+            # directory listing
             SimpleHTTPRequestHandler.do_GET(self)
 
-        else:
-            # encrypt file transfers
-            if os.path.isfile(fullpath) or os.path.islink(fullpath):
+        elif os.path.isfile(fullpath) or os.path.islink(fullpath):
+            if args.zip:
+                # ZIP file
+                self.send_response(200)
+                self.send_header('Content-type', "application/zip")
+                self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.zip')
+                self.end_headers()
+
+                if args.encrypt:
+                    # password-protected ZIP file (temporarily persisted to disk)
+                    with temporary_filename(suffix='.zip') as tmpFileName:
+                        pyminizip.compress(fullpath, None, tmpFileName, args.key, 1)
+                        with open(tmpFileName, 'rb') as f:
+                            while chunk := f.read(65536):
+                                self.wfile.write(chunk)
+
+                else:
+                    # encrypted ZIP file (streamed)
+                    for chunk in stream_zip(LocalFilesForZip([fullpath])):
+                        self.wfile.write(chunk)
+
+            elif args.encrypt:
+                # encrypted file
                 self.send_response(200)
                 self.send_header('Content-type', 'application/octet-stream')
                 self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.encrypted')
@@ -73,7 +118,11 @@ def do_GET(self):
                             break
 
             else:
-                self.send_error(404, "Not Found")
+                # unencrypted file
+                SimpleHTTPRequestHandler.do_GET(self)
+
+        else:
+            self.send_error(404, "Not Found")
 
 
 ###################################################################################################
@@ -101,8 +150,9 @@ def main():
 
     defaultDebug = os.getenv('EXTRACTED_FILE_HTTP_SERVER_DEBUG', 'false')
     defaultEncrypt = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ENCRYPT', 'false')
+    defaultZip = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ZIP', 'false')
     defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440))
-    defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'quarantined')
+    defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'infected')
     defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path)
 
     parser = argparse.ArgumentParser(
@@ -146,7 +196,7 @@ def main():
         const=True,
         default=defaultEncrypt,
         metavar='true|false',
-        help=f"Encrypt files with aes-256-cbc ({defaultEncrypt})",
+        help=f"Encrypt files (with -z/--zip, or with aes-256-cbc) ({defaultEncrypt})",
     )
     parser.add_argument(
         '-k',
@@ -157,6 +207,17 @@ def main():
         type=str,
         default=defaultKey,
     )
+    parser.add_argument(
+        '-z',
+        '--zip',
+        dest='zip',
+        type=str2bool,
+        nargs='?',
+        const=True,
+        default=defaultZip,
+        metavar='true|false',
+        help=f"Zip file ({defaultZip})",
+    )
     try:
         parser.error = parser.exit
         args = parser.parse_args()