Add FDA to quickstart setup (#477)

- set a long living token in keycloak --------- Co-authored-by: ekneg54 <[email protected]>
fkie-cad · Feb 7, 2024 · 28f69ff · 28f69ff
1 parent a948f3a
commit 28f69ff
Show file tree

Hide file tree

Showing 9 changed files with 8,720 additions and 25 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -28,6 +28,7 @@
 * rewrite Logprep Configuration object see documentation for more details
 * rewrite Runner 
 * delete MultiProcessingPipeline class to simplify multiprocesing
+* add FDA to the quickstart setup
 
 ### Bugfix
 

diff --git a/README.md b/README.md
@@ -314,7 +314,6 @@ logprep test config $CONFIG
 Where `$CONFIG` is the path or uri to a configuration file (see the documentation about the
 [configuration](https://logprep.readthedocs.io/en/latest/user_manual/configuration/index.html)).
 
-
 ### Reload the Configuration
 
 To change the configuration of Logprep it is not needed to restart Logprep entirely.
@@ -349,7 +348,7 @@ The environment can either be started with a Logprep container or without one:
      It starts and connects `Kafka`, `logprep`, `Opensearch` and `Opensearch Dashboards`.
   2. Run Logprep against loaded environment from main `Logprep` directory:
      ```bash
-     logprep quickstart/exampledata/config/pipeline.yml
+     logprep run quickstart/exampledata/config/pipeline.yml
      ```
 
 ### Run with Logprep Container
@@ -372,24 +371,50 @@ The environment can either be started with a Logprep container or without one:
     logprep http://localhost:8081/config/pipeline.yml
     ```
 
+### Run with getting config from FDA with oauth2 authentication
+
+Start logprep by using the oauth2 profile with docker-compose:
+
+    ```bash
+    docker-compose --profile oauth2 up -d
+    ```
+
+
+In order to run logprep with the FDA configuration it is necessary to set the following environment
+variables:
+
+- `LOGPREP_CONFIG_AUTH_METHOD=oauth`
+- `LOGPREP_CONFIG_AUTH_TOKEN=<TOKEN>` (can be intercepted from the FDA<->Keycloak http requests)
+- `LOGPREP_INPUT=<INPUT_CONNECTOR_CONFIG>`
+- `LOGPREP_OUTPUT=<OUTPUT_CONNECTOR_CONFIG>`
+- `LOGPREP_OPTIONS=<GENERAL_LOGPREP_CONFIG>`
+
+Once they are set logprep can be started with:
+
+```bash
+logprep run "http://localhost:8000/api/v1/pipelines?stage=prod&logclass=ExampleClass"
+```
+
 ### Interacting with the Quickstart Environment
 
 The start up takes a few seconds to complete, but once everything is up
 and running it is possible to write JSON events into Kafka and read the processed events in
 Opensearch Dashboards. Following services are available after start up:
 
-| Service                | Location | Credentials |
-|:-----------------------|:----|:-----|
-| Kafka:                 | `localhost:9092` | |
-| Kafka Exporter:        | `localhost:9308` | |
-| Logprep metrics:       | `localhost:8000` | |
-| Opensearch:            | `localhost:9200` | |
-| Opensearch Dashboards: | `localhost:5601` | |
-| Grafana Dashboards:    | `localhost:3000` | admin:admin |
-| Prometheus:            | `localhost:9090` | |
-| Nginx:                 | `localhost:8081` | user:password |
-| Keycloak:              | `localhost:8080` | admin:admin |
-| Postgres:              | `localhost:5432` | keycloak:bitnami |
+| Service                | Location          | User     | Password |
+|:-----------------------|:------------------|:---------|:---------|
+| Kafka:                 | `localhost:9092`  | /        | /        |
+| Kafka Exporter:        | `localhost:9308`  | /        | /        |
+| Logprep metrics:       | `localhost:8001`  | /        | /        |
+| Opensearch:            | `localhost:9200`  | /        | /        |
+| Opensearch Dashboards: | `localhost:5601`  | /        | /        |
+| Grafana Dashboards:    | `localhost:3000`  | admin    | admin    |
+| Prometheus:            | `localhost:9090`  | /        | /        |
+| Nginx:                 | `localhost:8081`  | user     | password |
+| Keycloak:              | `localhost:8080`  | admin    | admin    |
+| Keycloak Postgres:     | `localhost:5432`  | keycloak | bitnami  |
+| FDA:                   | `localhost:8002`  | logprep  | logprep  |
+| FDA Postgres:          | `localhost:25432` | fda      | fda      |
 
 The example rules that are used in the docker instance of Logprep can be found
 in `quickstart/exampledata/rules`.

diff --git a/quickstart/docker-compose.yml b/quickstart/docker-compose.yml
@@ -27,7 +27,6 @@ services:
     volumes:
       - data:/usr/share/opensearch/data
     network_mode: host
-
   dashboards:
     image: public.ecr.aws/opensearchproject/opensearch-dashboards:2
     container_name: dashboards
@@ -39,7 +38,6 @@ services:
     expose:
       - 5601
     network_mode: host
-
   kafka:
     image: bitnami/kafka:3.4
     container_name: kafka
@@ -67,7 +65,6 @@ services:
           "CMD-SHELL",
           "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic consumer --describe"
         ]
-
   kafka-exporter:
     image: bitnami/kafka-exporter
     container_name: kafkaexporter
@@ -78,7 +75,6 @@ services:
     depends_on:
       kafka:
         condition: service_healthy
-
   logprep:
     build:
       context: ..
@@ -87,18 +83,16 @@ services:
     profiles:
       - logprep
     expose:
-      - 8000
+      - 8001
     network_mode: host
     depends_on:
       - kafka
       - opensearch
-
     volumes:
       - ../quickstart/:/home/logprep/quickstart/
     entrypoint:
       - logprep
       - /home/logprep/quickstart/exampledata/config/pipeline.yml
-
   grafana:
     image: bitnami/grafana:latest
     container_name: grafana
@@ -162,9 +156,65 @@ services:
       POSTGRESQL_USERNAME: keycloak
     volumes:
       # use this folder to persist your postgresql data by dumping the database to it after changes
-      # outside the container: chmod 777 quickstart/exampledata/config/postgresql
+      # outside the container: chmod 777 quickstart/exampledata/config/keycloak
       # inside the container: pg_dump keycloak -U keycloak -W --file /docker-entrypoint-initdb.d/keycloak_db.sql
-      - ../quickstart/exampledata/config/postgresql:/docker-entrypoint-initdb.d
+      - ../quickstart/exampledata/config/keycloak:/docker-entrypoint-initdb.d
+  fda-api:
+    container_name: fda-api
+    image: registry.gitlab.com/z-e-u-s/fda/fda-backend:0.16.0
+    network_mode: host
+    expose:
+      - 8000
+    environment:
+      POSTGRES_PASSWORD: fda
+      POSTGRES_USER: fda
+      POSTGRES_DB: fda
+      POSTGRES_HOST: localhost
+      POSTGRES_PORT: 25432
+      DJANGO_SECRET_KEY: "django-insecure-*w($$5i@@iq%!ygufa%%@nfdplt(!e#hoahnjy^@6xdutl8mlqz"
+      ALLOWED_HOSTS: '["*"]'
+      CORS_ALLOWED_ORIGINS: '["http://localhost"]'
+      KEYCLOAK_SERVER_URL: "http://localhost:8080"
+      KEYCLOAK_REALM: logprep
+      KEYCLOAK_CLIENT_ID: fda-backend
+      KEYCLOAK_CLIENT_SECRET: tYfkKygb1g2Hf6fmAInoq3XPK1OILbSp
+      KEYCLOAK_CLIENT_ID_FOR_AUTHZ_ROLES: fda
+      SERVICE_BASE_URL: 'http://localhost:8000/'
+    depends_on:
+      fda-db:
+        condition: service_healthy
+  fda-db:
+    container_name: fda-db
+    image: "postgres:14"
+    network_mode: host
+    expose:
+      - 25432
+    volumes:
+      # use this folder to persist your postgresql data by dumping the database to it after changes
+      # outside the container: chmod 777 quickstart/exampledata/config/fda
+      # inside the container: pg_dump -p 25432 fda -U fda -W --file /docker-entrypoint-initdb.d/fda_db.sql
+      - ../quickstart/exampledata/config/fda:/docker-entrypoint-initdb.d
+    environment:
+      POSTGRES_PASSWORD: fda
+      POSTGRES_USER: fda
+      POSTGRES_DB: fda
+    command: -p 25432
+    healthcheck:
+      test: "pg_isready -p 25432 --username=$$POSTGRES_USER && psql -p 25432 --username=$$POSTGRES_USER --list"
+      interval: 8s
+      timeout: 5s
+      retries: 10
+  fda-frontend:
+    image: registry.gitlab.com/z-e-u-s/fda/fda-frontend:0.16.0
+    container_name: fda-frontend
+    network_mode: host
+    restart: always
+    volumes:
+      - ../quickstart/exampledata/config/fda/config.json:/usr/share/nginx/html/assets/config/keycloak/config.json
+      - ../quickstart/exampledata/config/fda/default.conf:/etc/nginx/conf.d/default.conf
+    expose:
+      - 8002
+
 volumes:
   data:
 

diff --git a/quickstart/exampledata/config/fda/config.json b/quickstart/exampledata/config/fda/config.json
@@ -0,0 +1,5 @@
+{
+  "KEYCLOAK_URL": "http://localhost:8080",
+  "KEYCLOAK_REALM": "logprep",
+  "KEYCLOAK_CLIENT_ID": "fda"
+}
diff --git a/quickstart/exampledata/config/fda/default.conf b/quickstart/exampledata/config/fda/default.conf
@@ -0,0 +1,26 @@
+ssl_session_cache    shared:SSL:10m;
+server {
+  listen 8002;
+  sendfile on;
+  default_type application/octet-stream;
+
+  gzip on;
+  gzip_http_version 1.1;
+  gzip_disable      "MSIE [1-6]\.";
+  gzip_min_length   256;
+  gzip_vary         on;
+  gzip_proxied      expired no-cache no-store private auth;
+  gzip_types        text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+  gzip_comp_level   9;
+
+  root /usr/share/nginx/html;
+
+  location / {
+    try_files $uri $uri/ /index.html =404;
+  }
+
+  # must be changed for the target environment. e.g. k8s = pod name of api service
+  location /api {
+    proxy_pass http://localhost:8000;
+  }
+}