From abd514aa791b46b4a91ceccdacb831416ed769c4 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 4 Jan 2024 13:12:36 -0700 Subject: [PATCH] work in progress for idaholab/Malcolm#288 --- Dockerfiles/file-monitor.Dockerfile | 3 + config/zeek.env.example | 2 + file-monitor/supervisord.conf | 3 +- ...rver.py => extracted_files_http_server.py} | 100 +++++++++++------- 4 files changed, 68 insertions(+), 40 deletions(-) rename shared/bin/{zeek_carved_http_server.py => extracted_files_http_server.py} (64%) diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 2ba1f0f2b..aae54c3b3 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -51,6 +51,7 @@ ARG EXTRACTED_FILE_HTTP_SERVER_DEBUG=false ARG EXTRACTED_FILE_HTTP_SERVER_ENABLE=false ARG EXTRACTED_FILE_HTTP_SERVER_ZIP=true ARG EXTRACTED_FILE_HTTP_SERVER_KEY=infected +ARG EXTRACTED_FILE_HTTP_SERVER_RECURSIVE=true ARG EXTRACTED_FILE_HTTP_SERVER_PORT=8440 ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH @@ -91,6 +92,7 @@ ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE ENV EXTRACTED_FILE_HTTP_SERVER_ZIP $EXTRACTED_FILE_HTTP_SERVER_ZIP ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY +ENV EXTRACTED_FILE_HTTP_SERVER_RECURSIVE $EXTRACTED_FILE_HTTP_SERVER_RECURSIVE ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT ENV SUPERCRONIC_VERSION "0.2.29" @@ -215,6 +217,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ COPY --chmod=755 shared/bin/zeek_carve*.py /usr/local/bin/ +COPY --chmod=755 shared/bin/extracted_files_http_server.py /usr/local/bin/ COPY --chmod=644 shared/bin/watch_common.py /usr/local/bin/ COPY --chmod=644 scripts/malcolm_utils.py /usr/local/bin/ COPY --chmod=644 file-monitor/supervisord.conf /etc/supervisord.conf diff --git a/config/zeek.env.example b/config/zeek.env.example index 142445c9f..ab208f6a2 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -51,6 +51,8 @@ EXTRACTED_FILE_PIPELINE_VERBOSITY= EXTRACTED_FILE_HTTP_SERVER_ENABLE=false # Whether or not Zeek-extracted files served over HTTP will be archived in a Zip file EXTRACTED_FILE_HTTP_SERVER_ZIP=false +# HTTP server will look in subdirectories for requested filename (e.g., in "/quarantined" and "/preserved") +EXTRACTED_FILE_HTTP_SERVER_RECURSIVE=true # Environment variables for tweaking Zeek at runtime (see local.zeek) # Set to any non-blank value to disable the corresponding feature ZEEK_DISABLE_HASH_ALL_FILES= diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf index 81eb7383f..2a9a1fe39 100644 --- a/file-monitor/supervisord.conf +++ b/file-monitor/supervisord.conf @@ -150,9 +150,10 @@ stdout_logfile_maxbytes=0 redirect_stderr=true [program:fileserve] -command=/usr/local/bin/zeek_carved_http_server.py +command=/usr/local/bin/extracted_files_http_server.py --port %(ENV_EXTRACTED_FILE_HTTP_SERVER_PORT)s --zip %(ENV_EXTRACTED_FILE_HTTP_SERVER_ZIP)s + --recursive %(ENV_EXTRACTED_FILE_HTTP_SERVER_RECURSIVE)s --directory /zeek/extract_files autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s autorestart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s diff --git a/shared/bin/zeek_carved_http_server.py b/shared/bin/extracted_files_http_server.py similarity index 64% rename from shared/bin/zeek_carved_http_server.py rename to shared/bin/extracted_files_http_server.py index 6f416f108..edd76170e 100755 --- a/shared/bin/zeek_carved_http_server.py +++ b/shared/bin/extracted_files_http_server.py @@ -67,52 +67,62 @@ def do_GET(self): global args fullpath = self.translate_path(self.path) + fileBaseName = os.path.basename(fullpath) if os.path.isdir(fullpath): # directory listing SimpleHTTPRequestHandler.do_GET(self) - elif os.path.isfile(fullpath) or os.path.islink(fullpath): - if args.zip: - # ZIP file (streamed, AES-encrypted with password or unencrypted) - self.send_response(200) - self.send_header('Content-type', "application/zip") - self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.zip') - self.end_headers() - - for chunk in stream_zip(LocalFilesForZip([fullpath]), password=args.key if args.key else None): - self.wfile.write(chunk) - - elif args.key: - # openssl-compatible encrypted file - self.send_response(200) - self.send_header('Content-type', 'application/octet-stream') - self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.encrypted') - self.end_headers() - salt = os.urandom(PKCS5_SALT_LEN) - key, iv = EVP_BytesToKey(EVP_KEY_SIZE, AES.block_size, hashlib.sha256, salt, args.key.encode('utf-8')) - cipher = AES.new(key, AES.MODE_CBC, iv) - encrypted = b"" - encrypted += OPENSSL_ENC_MAGIC - encrypted += salt - self.wfile.write(encrypted) - with open(fullpath, 'rb') as f: - padding = b'' - while True: - chunk = f.read(cipher.block_size) - if len(chunk) < cipher.block_size: - remaining = cipher.block_size - len(chunk) - padding = bytes([remaining] * remaining) - self.wfile.write(cipher.encrypt(chunk + padding)) - if padding: - break + else: + if recursive and (not os.path.isfile(fullpath)) and (not os.path.islink(fullpath)): + for root, dirs, files in os.walk(os.path.dirname(fullpath)): + if fileBaseName in files: + fullpath = os.path.join(root, fileBaseName) + break + + if os.path.isfile(fullpath) or os.path.islink(fullpath): + if args.zip: + # ZIP file (streamed, AES-encrypted with password or unencrypted) + self.send_response(200) + self.send_header('Content-type', "application/zip") + self.send_header('Content-Disposition', f'attachment; filename={fileBaseName}.zip') + self.end_headers() + + for chunk in stream_zip(LocalFilesForZip([fullpath]), password=args.key if args.key else None): + self.wfile.write(chunk) + + elif args.key: + # openssl-compatible encrypted file + self.send_response(200) + self.send_header('Content-type', 'application/octet-stream') + self.send_header('Content-Disposition', f'attachment; filename={fileBaseName}.encrypted') + self.end_headers() + salt = os.urandom(PKCS5_SALT_LEN) + key, iv = EVP_BytesToKey( + EVP_KEY_SIZE, AES.block_size, hashlib.sha256, salt, args.key.encode('utf-8') + ) + cipher = AES.new(key, AES.MODE_CBC, iv) + encrypted = b"" + encrypted += OPENSSL_ENC_MAGIC + encrypted += salt + self.wfile.write(encrypted) + with open(fullpath, 'rb') as f: + padding = b'' + while True: + chunk = f.read(cipher.block_size) + if len(chunk) < cipher.block_size: + remaining = cipher.block_size - len(chunk) + padding = bytes([remaining] * remaining) + self.wfile.write(cipher.encrypt(chunk + padding)) + if padding: + break + + else: + # original file, unencrypted + SimpleHTTPRequestHandler.do_GET(self) else: - # original file, unencrypted - SimpleHTTPRequestHandler.do_GET(self) - - else: - self.send_error(404, "Not Found") + self.send_error(404, "Not Found") ################################################################################################### @@ -140,6 +150,7 @@ def main(): defaultDebug = os.getenv('EXTRACTED_FILE_HTTP_SERVER_DEBUG', 'false') defaultZip = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ZIP', 'false') + defaultRecursive = os.getenv('EXTRACTED_FILE_HTTP_SERVER_RECURSIVE', 'false') defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440)) defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'infected') defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path) @@ -196,6 +207,17 @@ def main(): metavar='true|false', help=f"Zip file ({defaultZip})", ) + parser.add_argument( + '-r', + '--recursive', + dest='recursive', + type=str2bool, + nargs='?', + const=True, + default=defaultRecursive, + metavar='true|false', + help=f"Recursively look for requested file if not found", + ) try: parser.error = parser.exit args = parser.parse_args()