work in progress for idaholab#288

Dockerfiles/file-monitor.Dockerfile

@@ -51,6 +51,7 @@ ARG EXTRACTED_FILE_HTTP_SERVER_DEBUG=false
 ARG EXTRACTED_FILE_HTTP_SERVER_ENABLE=false
 ARG EXTRACTED_FILE_HTTP_SERVER_ZIP=true
 ARG EXTRACTED_FILE_HTTP_SERVER_KEY=infected
+ARG EXTRACTED_FILE_HTTP_SERVER_RECURSIVE=true
 ARG EXTRACTED_FILE_HTTP_SERVER_PORT=8440
 
 ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
@@ -91,6 +92,7 @@ ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG
 ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE
 ENV EXTRACTED_FILE_HTTP_SERVER_ZIP $EXTRACTED_FILE_HTTP_SERVER_ZIP
 ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY
+ENV EXTRACTED_FILE_HTTP_SERVER_RECURSIVE $EXTRACTED_FILE_HTTP_SERVER_RECURSIVE
 ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT
 
 ENV SUPERCRONIC_VERSION "0.2.29"
@@ -215,6 +217,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour
 COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/
 COPY --chmod=755 shared/bin/zeek_carve*.py /usr/local/bin/
+COPY --chmod=755 shared/bin/extracted_files_http_server.py /usr/local/bin/
 COPY --chmod=644 shared/bin/watch_common.py /usr/local/bin/
 COPY --chmod=644 scripts/malcolm_utils.py /usr/local/bin/
 COPY --chmod=644 file-monitor/supervisord.conf /etc/supervisord.conf

config/zeek.env.example

@@ -51,6 +51,8 @@ EXTRACTED_FILE_PIPELINE_VERBOSITY=
 EXTRACTED_FILE_HTTP_SERVER_ENABLE=false
 # Whether or not Zeek-extracted files served over HTTP will be archived in a Zip file
 EXTRACTED_FILE_HTTP_SERVER_ZIP=false
+# HTTP server will look in subdirectories for requested filename (e.g., in "/quarantined" and "/preserved")
+EXTRACTED_FILE_HTTP_SERVER_RECURSIVE=true
 # Environment variables for tweaking Zeek at runtime (see local.zeek)
 #   Set to any non-blank value to disable the corresponding feature
 ZEEK_DISABLE_HASH_ALL_FILES=

file-monitor/supervisord.conf

@@ -150,9 +150,10 @@ stdout_logfile_maxbytes=0
 redirect_stderr=true
 
 [program:fileserve]
-command=/usr/local/bin/zeek_carved_http_server.py
+command=/usr/local/bin/extracted_files_http_server.py
   --port %(ENV_EXTRACTED_FILE_HTTP_SERVER_PORT)s
   --zip %(ENV_EXTRACTED_FILE_HTTP_SERVER_ZIP)s
+  --recursive %(ENV_EXTRACTED_FILE_HTTP_SERVER_RECURSIVE)s
   --directory /zeek/extract_files
 autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s
 autorestart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s

shared/bin/zeek_carved_http_server.py → shared/bin/extracted_files_http_server.py

@@ -67,52 +67,62 @@ def do_GET(self):
         global args
 
         fullpath = self.translate_path(self.path)
+        fileBaseName = os.path.basename(fullpath)
 
         if os.path.isdir(fullpath):
             # directory listing
             SimpleHTTPRequestHandler.do_GET(self)
 
-        elif os.path.isfile(fullpath) or os.path.islink(fullpath):
-            if args.zip:
-                # ZIP file (streamed, AES-encrypted with password or unencrypted)
-                self.send_response(200)
-                self.send_header('Content-type', "application/zip")
-                self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.zip')
-                self.end_headers()
-
-                for chunk in stream_zip(LocalFilesForZip([fullpath]), password=args.key if args.key else None):
-                    self.wfile.write(chunk)
-
-            elif args.key:
-                # openssl-compatible encrypted file
-                self.send_response(200)
-                self.send_header('Content-type', 'application/octet-stream')
-                self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.encrypted')
-                self.end_headers()
-                salt = os.urandom(PKCS5_SALT_LEN)
-                key, iv = EVP_BytesToKey(EVP_KEY_SIZE, AES.block_size, hashlib.sha256, salt, args.key.encode('utf-8'))
-                cipher = AES.new(key, AES.MODE_CBC, iv)
-                encrypted = b""
-                encrypted += OPENSSL_ENC_MAGIC
-                encrypted += salt
-                self.wfile.write(encrypted)
-                with open(fullpath, 'rb') as f:
-                    padding = b''
-                    while True:
-                        chunk = f.read(cipher.block_size)
-                        if len(chunk) < cipher.block_size:
-                            remaining = cipher.block_size - len(chunk)
-                            padding = bytes([remaining] * remaining)
-                        self.wfile.write(cipher.encrypt(chunk + padding))
-                        if padding:
-                            break
+        else:
+            if recursive and (not os.path.isfile(fullpath)) and (not os.path.islink(fullpath)):
+                for root, dirs, files in os.walk(os.path.dirname(fullpath)):
+                    if fileBaseName in files:
+                        fullpath = os.path.join(root, fileBaseName)
+                        break
+
+            if os.path.isfile(fullpath) or os.path.islink(fullpath):
+                if args.zip:
+                    # ZIP file (streamed, AES-encrypted with password or unencrypted)
+                    self.send_response(200)
+                    self.send_header('Content-type', "application/zip")
+                    self.send_header('Content-Disposition', f'attachment; filename={fileBaseName}.zip')
+                    self.end_headers()
+
+                    for chunk in stream_zip(LocalFilesForZip([fullpath]), password=args.key if args.key else None):
+                        self.wfile.write(chunk)
+
+                elif args.key:
+                    # openssl-compatible encrypted file
+                    self.send_response(200)
+                    self.send_header('Content-type', 'application/octet-stream')
+                    self.send_header('Content-Disposition', f'attachment; filename={fileBaseName}.encrypted')
+                    self.end_headers()
+                    salt = os.urandom(PKCS5_SALT_LEN)
+                    key, iv = EVP_BytesToKey(
+                        EVP_KEY_SIZE, AES.block_size, hashlib.sha256, salt, args.key.encode('utf-8')
+                    )
+                    cipher = AES.new(key, AES.MODE_CBC, iv)
+                    encrypted = b""
+                    encrypted += OPENSSL_ENC_MAGIC
+                    encrypted += salt
+                    self.wfile.write(encrypted)
+                    with open(fullpath, 'rb') as f:
+                        padding = b''
+                        while True:
+                            chunk = f.read(cipher.block_size)
+                            if len(chunk) < cipher.block_size:
+                                remaining = cipher.block_size - len(chunk)
+                                padding = bytes([remaining] * remaining)
+                            self.wfile.write(cipher.encrypt(chunk + padding))
+                            if padding:
+                                break
+
+                else:
+                    # original file, unencrypted
+                    SimpleHTTPRequestHandler.do_GET(self)
 
             else:
-                # original file, unencrypted
-                SimpleHTTPRequestHandler.do_GET(self)
-
-        else:
-            self.send_error(404, "Not Found")
+                self.send_error(404, "Not Found")
 
 
 ###################################################################################################
@@ -140,6 +150,7 @@ def main():
 
     defaultDebug = os.getenv('EXTRACTED_FILE_HTTP_SERVER_DEBUG', 'false')
     defaultZip = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ZIP', 'false')
+    defaultRecursive = os.getenv('EXTRACTED_FILE_HTTP_SERVER_RECURSIVE', 'false')
     defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440))
     defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'infected')
     defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path)
@@ -196,6 +207,17 @@ def main():
         metavar='true|false',
         help=f"Zip file ({defaultZip})",
     )
+    parser.add_argument(
+        '-r',
+        '--recursive',
+        dest='recursive',
+        type=str2bool,
+        nargs='?',
+        const=True,
+        default=defaultRecursive,
+        metavar='true|false',
+        help=f"Recursively look for requested file if not found",
+    )
     try:
         parser.error = parser.exit
         args = parser.parse_args()