Merge pull request #962 from LDO-CERT/dev

2.0.0
LDO-CERT · Jan 8, 2024 · 185fd6b · 185fd6b
2 parents f1612bd + 9c65b4c
commit 185fd6b
Show file tree

Hide file tree

Showing 64 changed files with 1,438 additions and 3,016 deletions.
diff --git a/.envs/.local/.django b/.envs/.local/.django
@@ -35,3 +35,7 @@ VOLATILITY_SYMBOL_DOWNLOAD_PATH=https://downloads.volatilityfoundation.org/volat
 http_proxy=
 https_proxy=
 no_proxy=
+
+# CSRF_TRUSTED_ORIGINS
+# ------------------------------------------------------------------------------
+CSRF_TRUSTED_ORIGINS=
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -3,45 +3,54 @@ on:
   push:
     branches:
       - master
+    tags:
+      - "v*"
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
-  
+
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Build the stack
-        run: | 
+        run: |
           export DOCKER_BUILDKIT=0
           export COMPOSE_DOCKER_CLI_BUILD=0
           docker-compose build
-      - name: start containers 
-        run: docker-compose up -d  
+
+      - name: start containers
+        run: docker-compose up -d
+
       - name: show container and networks
         run: docker network ls && docker ps -a
-      - name: run pytest  
+
+      - name: run pytest
         run: docker-compose run django pytest
-#      - name: curl
-#        run: docker run --rm --network orochi_default curlimages/curl:latest http://django:8000          
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v1
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      - name: Push image
-        run: docker-compose push
-
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
 
       - name: Stop containers
         if: always()

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,18 +4,18 @@ fail_fast: true
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
 
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.12.0
     hooks:
       - id: black
 
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.10.1
+    rev: 5.13.2
     hooks:
       - id: isort
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,28 +1,31 @@
 {
-    "workbench.colorCustomizations": {
-        "activityBar.activeBackground": "#ba3257",
-        "activityBar.activeBorder": "#52b02f",
-        "activityBar.background": "#ba3257",
-        "activityBar.foreground": "#e7e7e7",
-        "activityBar.inactiveForeground": "#e7e7e799",
-        "activityBarBadge.background": "#52b02f",
-        "activityBarBadge.foreground": "#15202b",
-        "statusBar.background": "#922744",
-        "statusBar.foreground": "#e7e7e7",
-        "statusBarItem.hoverBackground": "#ba3257",
-        "titleBar.activeBackground": "#922744",
-        "titleBar.activeForeground": "#e7e7e7",
-        "titleBar.inactiveBackground": "#92274499",
-        "titleBar.inactiveForeground": "#e7e7e799",
-        "sash.hoverBorder": "#ba3257",
-        "statusBarItem.remoteBackground": "#922744",
-        "statusBarItem.remoteForeground": "#e7e7e7",
-        "commandCenter.border": "#e7e7e799"
-    },
-    "peacock.color": "#922744",
-    "python.linting.pylintEnabled": false,
-    "python.linting.mypyEnabled": false,
-    "python.linting.enabled": true,
-    "python.formatting.provider": "black",
-    "python.linting.flake8Enabled": false
+  "workbench.colorCustomizations": {
+    "activityBar.activeBackground": "#ba3257",
+    "activityBar.activeBorder": "#52b02f",
+    "activityBar.background": "#ba3257",
+    "activityBar.foreground": "#e7e7e7",
+    "activityBar.inactiveForeground": "#e7e7e799",
+    "activityBarBadge.background": "#52b02f",
+    "activityBarBadge.foreground": "#15202b",
+    "statusBar.background": "#922744",
+    "statusBar.foreground": "#e7e7e7",
+    "statusBarItem.hoverBackground": "#ba3257",
+    "titleBar.activeBackground": "#922744",
+    "titleBar.activeForeground": "#e7e7e7",
+    "titleBar.inactiveBackground": "#92274499",
+    "titleBar.inactiveForeground": "#e7e7e799",
+    "sash.hoverBorder": "#ba3257",
+    "statusBarItem.remoteBackground": "#922744",
+    "statusBarItem.remoteForeground": "#e7e7e7",
+    "commandCenter.border": "#e7e7e799"
+  },
+  "peacock.color": "#922744",
+  "python.linting.pylintEnabled": false,
+  "python.linting.mypyEnabled": false,
+  "python.linting.enabled": true,
+  "python.formatting.provider": "none",
+  "python.linting.flake8Enabled": false,
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter"
+  }
 }
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+# OROCHI 2.0.0 [2024/01/09]
+- Update libs and UI
+- Re-Run default enabled plugins [[#950](https://github.com/LDO-CERT/orochi/issues/950)]
+- Pending task count [[#255](https://github.com/LDO-CERT/orochi/issues/255)]
+- Update vt python libs
+
 # OROCHI 1.3.1 [2022/01/17]
 - Unzip password protected file [#484](https://github.com/LDO-CERT/orochi/issues/484)
 - Md5 support for dumped files [#489](https://github.com/LDO-CERT/orochi/issues/489)

diff --git a/compose/local/dask/Dockerfile b/compose/local/dask/Dockerfile
@@ -1,4 +1,4 @@
-FROM daskdev/dask:2022.12.1-py3.10
+FROM daskdev/dask:dev-py3.11
 ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get --allow-releaseinfo-change update \

diff --git a/compose/local/dask/daskworkerinit.py b/compose/local/dask/daskworkerinit.py
@@ -1,9 +1,7 @@
-import sys
 import os
-import django
-from volatility3 import framework
+import sys
 
-framework.clear_cache()
+import django
 
 os.environ["DATABASE_URL"] = "postgres://{}:{}@{}:{}/{}".format(
     os.environ["POSTGRES_USER"],

diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim-buster as common-base
+FROM python:3.11-slim-buster as common-base
 
 ENV DJANGO_SETTINGS_MODULE config.settings.local
 ENV PYTHONUNBUFFERED 1

diff --git a/config/settings/base.py b/config/settings/base.py
@@ -1,6 +1,7 @@
 """
 Base settings to build other settings files upon.
 """
+
 from pathlib import Path
 
 import environ
@@ -10,8 +11,7 @@
 APPS_DIR = ROOT_DIR / "orochi"
 env = environ.Env()
 
-READ_DOT_ENV_FILE = env.bool("DJANGO_READ_DOT_ENV_FILE", default=False)
-if READ_DOT_ENV_FILE:
+if READ_DOT_ENV_FILE := env.bool("DJANGO_READ_DOT_ENV_FILE", default=False):
     # OS environment variables take precedence over variables from .env
     env.read_env(str(ROOT_DIR / ".env"))
 
@@ -128,6 +128,7 @@
     "django.middleware.common.BrokenLinkEmailsMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "orochi.website.middleware.UpdatesMiddleware",
+    "allauth.account.middleware.AccountMiddleware",
 ]
 
 # STATIC
@@ -278,7 +279,7 @@
 
 # django-cors-headers - https://github.com/adamchainz/django-cors-headers#setup
 CORS_URLS_REGEX = r"^/api/.*$"
-
+CSRF_TRUSTED_ORIGINS = env("CSRF_TRUSTED_ORIGINS")
 # OROCHI CONFIGURATIONS
 # -------------------------------------------------------------------------------
 

diff --git a/docker-compose-swarm.yml b/docker-compose-swarm.yml
@@ -23,7 +23,12 @@ volumes:
     driver_opts:
       type: "nfs"
       o: "addr=nfs_server_ip,nolock,soft,rw,nfsvers=4"
-      device: ":/var/nfs/yara"
+  cache_path:
+    driver: local
+    driver_opts:
+      type: "nfs"
+      o: "addr=nfs_server_ip,nolock,soft,rw,nfsvers=4"
+      device: ":/var/nfs/cache"
   postgres_data:
     driver: local
     driver_opts:
@@ -44,8 +49,8 @@ volumes:
       device: ":/var/nfs/elasticsearch/data"
 
 services:
-  django:
-    &django
+
+  django: &django
     build:
       context: .
       dockerfile: ./compose/local/django/Dockerfile
@@ -54,6 +59,8 @@ services:
         - https_proxy
     image: ghcr.io/ldo-cert/orochi_django:latest
     container_name: orochi_django
+    hostname: django
+    restart: always
     depends_on:
       - postgres
       - mailhog
@@ -62,6 +69,7 @@ services:
       - symbols_path:/src/volatility3/volatility3/symbols
       - plugin_path:/src/volatility3/volatility3/framework/plugins/custom
       - yara_path:/yara
+      - cache_path:/root/.cache/volatility3
     env_file:
       - ./.envs/.local/.django
       - ./.envs/.local/.postgres
@@ -78,6 +86,8 @@ services:
       dockerfile: ./compose/local/postgres/Dockerfile
     image: ghcr.io/ldo-cert/orochi_postgres:latest
     container_name: orochi_postgres
+    hostname: postgres
+    restart: always
     volumes:
       - postgres_data:/var/lib/postgresql/data
       - postgres_backups:/backups
@@ -89,27 +99,33 @@ services:
   mailhog:
     image: mailhog/mailhog:v1.0.1
     container_name: orochi_mailhog
+    hostname: mailhog
+    restart: always
     ports:
       - "8025:8025"
 
   redis:
     image: redis:6.2.5
     container_name: orochi_redis
+    hostname: redis
+    restart: always
     ports:
       - "6379:6379"
 
   es01:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.4
+    image: elasticsearch:8.9.0
     container_name: orochi_es01
     environment:
+      - discovery.type=single-node
       - node.name=es01
-      - cluster.name=es-docker-cluster
-      - cluster.initial_master_nodes=es01
-      - bootstrap.memory_lock=false
       - "ES_JAVA_OPTS=-Xms1G -Xmx1G"
-      - xpack.ml.enabled=false
       - logger.level=WARN
+      - xpack.ml.enabled=false
       - xpack.security.enabled=false
+      - xpack.watcher.enabled=false
+      - ingest.geoip.downloader.enabled=false
+    hostname: es01
+    restart: always
     ulimits:
       memlock:
         soft: -1
@@ -120,26 +136,49 @@ services:
       - 9200:9200
 
   kib01:
-    image: docker.elastic.co/kibana/kibana:7.17.4
+    image: kibana:8.9.0
     container_name: orochi_kib01
     ports:
       - 5601:5601
+    hostname: kib01
+    restart: always
     environment:
       ELASTICSEARCH_URL: http://es01:9200
       ELASTICSEARCH_HOSTS: http://es01:9200
       TELEMETRY_ENABLED: "false"
       MONITORING_ENABLED: "false"
 
   scheduler:
-    image: daskdev/dask:2022.12.1-py3.10
+    image: ghcr.io/ldo-cert/orochi_worker:latest
+    build:
+      context: .
+      dockerfile: ./compose/local/dask/Dockerfile
+      args:
+        - http_proxy
+        - https_proxy
     container_name: orochi_scheduler
     hostname: dask-scheduler
+    restart: always
     env_file:
       - ./.envs/.local/.dask
+      - ./.envs/.local/.django
+      - ./.envs/.local/.postgres
     ports:
       - "8786:8786"
       - "8787:8787"
-    command: [ "dask-scheduler" ]
+    command:
+      [
+        "dask",
+        "scheduler",
+        "--preload",
+        "/opt/daskworkerinit.py"
+      ]
+    volumes:
+      - media_path:/media
+      - symbols_path:/src/volatility3/volatility3/symbols
+      - plugin_path:/src/volatility3/volatility3/framework/plugins/custom
+      - yara_path:/yara
+      - cache_path:/root/.cache/volatility3
 
   worker:
     image: ghcr.io/ldo-cert/orochi_worker:latest
@@ -149,6 +188,9 @@ services:
       args:
         - http_proxy
         - https_proxy
+    container_name: orochi_worker
+    hostname: dask-worker
+    restart: always
     environment:
       - http_proxy=${http_proxy}
       - https_proxy=${https_proxy}
@@ -171,6 +213,7 @@ services:
       - symbols_path:/src/volatility3/volatility3/symbols
       - plugin_path:/src/volatility3/volatility3/framework/plugins/custom
       - yara_path:/yara
+      - cache_path:/root/.cache/volatility3
     env_file:
       - ./.envs/.local/.django
       - ./.envs/.local/.postgres