diff --git a/.github/secret_scanning.yml b/.github/secret_scanning.yml index 3d036d4d4..3e87bf78b 100644 --- a/.github/secret_scanning.yml +++ b/.github/secret_scanning.yml @@ -1,4 +1,4 @@ paths-ignore: - - "quickstart/exampledata/config/credentials.yml" - - "quickstart/exampledata/config/nginx/mtls.conf.d/*.key" - - "quickstart/exampledata/config/nginx/mtls.conf.d/*.crt" + - "examples/exampledata/config/credentials.yml" + - "examples/exampledata/config/nginx/mtls.conf.d/*.key" + - "examples/exampledata/config/nginx/mtls.conf.d/*.crt" diff --git a/.gitignore b/.gitignore index 30c797895..bb0cf0161 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ experiments **/_static/*.xlsx logprep.log /charts/logprep/charts +examples/k8s/charts diff --git a/CHANGELOG.md b/CHANGELOG.md index 96cf82f38..a1aaf67de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,15 @@ ## next release ### Breaking ### Features + ### Improvements * a result object was added to processors and pipelines * each processor returns an object including the processor name, generated extra_data, warnings and errors * the pipeline returns an object with the list of all processor result objects +* add kubernetes opensiem deployment example +* move quickstart setup to compose example ### Bugfix @@ -316,7 +319,7 @@ * add possibility to convert hex to int in `calculator` processor with new added function `from_hex` * add metrics on rule level -* add grafana example dashboards under `quickstart/exampledata/config/grafana/dashboards` +* add grafana example dashboards under `examples/exampledata/config/grafana/dashboards` * add new configuration field `id` for all rules to identify rules in metrics and logs - if no `id` is given, the `id` will be generated in a stable way - add verification of rule `id` uniqueness on processor level over both rule trees to ensure metrics are counted correctly on rule level diff --git a/README.md b/README.md index 8e33a94b9..bf7d2371c 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ allowing further applications besides log handling. This readme provides basic information about the following topics: - [About Logprep](#about-logprep) - [Getting Started](https://logprep.readthedocs.io/en/latest/getting_started.html) -- [Docker Quickstart](https://logprep.readthedocs.io/en/latest/getting_started.html#logprep-quickstart-environment) +- [Deployment Examples](https://logprep.readthedocs.io/en/latest/examples/index.html) - [Event Generation](https://logprep.readthedocs.io/en/latest/user_manual/execution.html#event-generation) - [Documentation](https://logprep.readthedocs.io/en/latest) - [Contributing](#contributing) diff --git a/charts/logprep/Chart.yaml b/charts/logprep/Chart.yaml index 50f71b756..85dc04efa 100644 --- a/charts/logprep/Chart.yaml +++ b/charts/logprep/Chart.yaml @@ -6,7 +6,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "13.2.0" +version: "13.2.1" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/logprep/templates/deployment.yaml b/charts/logprep/templates/deployment.yaml index 87e3bebf6..c47a4d632 100644 --- a/charts/logprep/templates/deployment.yaml +++ b/charts/logprep/templates/deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "logprep.name" . }} + name: {{ include "logprep.fullname" . }} labels: {{- include "logprep.labels" . | nindent 4 }} spec: @@ -14,6 +14,7 @@ spec: {{- include "logprep.selectorLabels" . | nindent 6 }} template: metadata: + name: {{ include "logprep.fullname" . }} labels: {{- include "logprep.labels" . | nindent 8 }} annotations: diff --git a/charts/logprep/values.yaml b/charts/logprep/values.yaml index 9ec46ba9e..3a12a402a 100644 --- a/charts/logprep/values.yaml +++ b/charts/logprep/values.yaml @@ -118,6 +118,9 @@ logger: # # See https://logprep.readthedocs.io/en/latest/user_manual/configuration/input.html # for available configuration options. +# Note: +# For the `http_input` endpoints you have to add the endpoint `/health: plaintext` to ensure +# readiness probes are working. input: {} # The logprep output connector configuration diff --git a/doc/source/development/programaticly_start_logprep.rst b/doc/source/development/programaticly_start_logprep.rst index 4c1426ab7..51de7c9a4 100644 --- a/doc/source/development/programaticly_start_logprep.rst +++ b/doc/source/development/programaticly_start_logprep.rst @@ -23,10 +23,10 @@ An example with input connector and preprocessors could look like this: "predetector": { "type": "pre_detector", "specific_rules": [ - "quickstart/exampledata/rules/pre_detector/specific" + "examples/exampledata/rules/pre_detector/specific" ], "generic_rules": [ - "quickstart/exampledata/rules/pre_detector/generic" + "examples/exampledata/rules/pre_detector/generic" ], "pre_detector_topic": "output_topic" } @@ -61,10 +61,10 @@ An example without input connector and preprocessors could look like this: "predetector": { "type": "pre_detector", "specific_rules": [ - "quickstart/exampledata/rules/pre_detector/specific" + "examples/exampledata/rules/pre_detector/specific" ], "generic_rules": [ - "quickstart/exampledata/rules/pre_detector/generic" + "examples/exampledata/rules/pre_detector/generic" ], "pre_detector_topic": "output_topic" } diff --git a/doc/source/examples/compose.rst b/doc/source/examples/compose.rst index 9e1747f80..f4c264aef 100644 --- a/doc/source/examples/compose.rst +++ b/doc/source/examples/compose.rst @@ -5,7 +5,7 @@ Docker Compose Example Deployment To demonstrate the functionality of logprep this repo comes with a complete `kafka`, `logprep` and `opensearch` stack. To get it running `docker` with compose support must be first installed. -The docker compose file is located in the directory `quickstart`. +The docker compose file is located in the directory `examples/compose`. A prerequisite is to run `sysctl -w vm.max_map_count=262144`, otherwise Opensearch might not properly start. @@ -14,7 +14,7 @@ The environment can either be started with a Logprep container or without one: Run without Logprep Container (default) --------------------------------------- - 1. Run from within the `quickstart` directory: + 1. Run from within the `examples/compose` directory: .. code-block:: bash @@ -25,14 +25,14 @@ Run without Logprep Container (default) .. code-block:: bash - logprep run quickstart/exampledata/config/pipeline.yml + logprep run examples/exampledata/config/pipeline.yml If logprep is run with the metrics enabled, the necessary environment variable has to be set first: .. code-block:: bash export PROMETHEUS_MULTIPROC_DIR="tmp/logprep" - logprep run quickstart/exampledata/config/pipeline.yml + logprep run examples/exampledata/config/pipeline.yml @@ -40,7 +40,7 @@ Run without Logprep Container (default) Run with Logprep Container -------------------------- - * Run from within the `quickstart` directory: + * Run from within the `examples/compose` directory: .. code-block:: bash @@ -50,7 +50,7 @@ Run with Logprep Container Run with getting config from http server with basic authentication ------------------------------------------------------------------ - * Run from within the `quickstart` directory: + * Run from within the `examples/compose` directory: .. code-block:: bash @@ -60,14 +60,14 @@ Run with getting config from http server with basic authentication .. code-block:: bash - export LOGPREP_CREDENTIALS_FILE="quickstart/exampledata/config/credentials.yml" + export LOGPREP_CREDENTIALS_FILE="examples/exampledata/config/credentials.yml" logprep run http://localhost:8081/config/pipeline.yml Run with getting config from http server with mTLS authentication ----------------------------------------------------------------- - * Run from within the `quickstart` directory: + * Run from within the `examples/compose` directory: .. code-block:: bash @@ -77,12 +77,12 @@ Run with getting config from http server with mTLS authentication .. code-block:: bash - export LOGPREP_CREDENTIALS_FILE="quickstart/exampledata/config/credentials.yml" + export LOGPREP_CREDENTIALS_FILE="examples/exampledata/config/credentials.yml" logprep run https://localhost:8082/config/pipeline.yml -Interacting with the Quickstart Environment -------------------------------------------- +Interacting with the Compose Environment +---------------------------------------- The start up takes a few seconds to complete, but once everything is up and running it is possible to write JSON events into Kafka and read the processed events in @@ -111,9 +111,9 @@ UCL Postgres: `localhost:5432` ucl ucl ====================== ================= ======================== ======================= The example rules that are used in the docker instance of Logprep can be found -in `quickstart/exampledata/rules`. +in `examples/exampledata/rules`. Example events that trigger for the example rules can be found in -`quickstart/exampledata/input_logdata/logclass/test_input.jsonl`. +`examples/exampledata/input_logdata/logclass/test_input.jsonl`. These events can be added to Kafka with the following command: .. code-block:: bash @@ -133,12 +133,17 @@ Utilizing FDA and UCL If you want to try out the FDA and UCL you first have to do some preparations. -0. Run the quickstart setup with the :code:`oauth2` profile: - :code:`docker compose --profile oauth2 up -d`. +0. Run the example compose setup with the :code:`oauth2` profile: + +.. code-block:: bash + + docker compose --profile oauth2 up -d. + + 1. Sign into the keycloak admin panel and create a logprep user in the :code:`logprep` realm. Make sure that the user is part of the :code:`logprep-admin` group and has a password. If you choose a password other than :code:`logprep` you have to update the credentials file - :code:`quickstart/exampledata/config/credentials.yml`, such that the password of + :code:`examples/exampledata/config/credentials.yml`, such that the password of :code:`http://localhost:3001` and :code:`http://localhost:3002` reflects your choice. 2. You have to login to the FDA with the previously created user and create a release, as well as your first logclass. It is also necessary to add an example event to this logclass in order @@ -148,19 +153,19 @@ If you want to try out the FDA and UCL you first have to do some preparations. your created logprep user and then configure required Use-Cases. At the current moment these configuration are not yet processed by logprep though, as the ucl only provides a mock endpoint which doesn't contain your Use-Case configurations. -4. Set the env :code:`LOGPREP_CREDENTIALS_FILE` to :code:`quickstart/exampledata/config/credentials.yml` +4. Set the env and run logprep + + .. code-block:: bash + + export LOGPREP_CREDENTIALS_FILE="examples/exampledata/config/credentials.yml" + logprep run examples/exampledata/config/pipeline.yml "http://localhost:3002/api/v1/pipelines?stage=prod&logclass=ExampleClass" "http://localhost:3001/api/v1/general-predetection" -Once you have set everything up you can run logprep with the following command. Just consider that the first :code:`pipeline.yml` argument is used to define a proper :code:`input` and :code:`output` as those are not part of the FDA/UCL output. Also, in the second argument you should ensure that the :code:`stage` and :code:`loglcass` are set properly. -.. code-block:: bash - - logprep run quickstart/exampledata/config/pipeline.yml "http://localhost:3002/api/v1/pipelines?stage=prod&logclass=ExampleClass" "http://localhost:3001/api/v1/general-predetection" - .. note:: - If you did use the quickstart setup before and run into problems it is advised to first pull + If you did use the example compose setup before and run into problems it is advised to first pull all images again to update them to the latest version: - :code:`docker compose -f ./quickstart/docker-compose.yml pull`. + :code:`docker compose -f ./examples/compose/docker-compose.yml pull`. diff --git a/doc/source/examples/minikube.rst b/doc/source/examples/minikube.rst index eded3b3a1..ec9bb7e95 100644 --- a/doc/source/examples/minikube.rst +++ b/doc/source/examples/minikube.rst @@ -45,15 +45,26 @@ with the following commands: sudo mv linux-amd64/helm /usr/local/bin/helm sudo chmod +x /usr/local/bin/helm +.. code-block:: bash + :caption: add helm repositories + + helm repo add bitnami https://charts.bitnami.com/bitnami + .. code-block:: bash :caption: Configure and start minikube minikube config set driver docker + minikube config set cpus 16 + minikube config set memory 16GB + minikube addons enable ingress minikube start Deploy the example ------------------ +The following steps install the actual opensiem example on the minikube cluster. +It will install + At first you have to install the prometheus PodMonitor CRD: .. code-block:: bash @@ -62,9 +73,56 @@ At first you have to install the prometheus PodMonitor CRD: kubectl apply -f https://raw.githubusercontent.com/prometheus-community/helm-charts/main/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml -Next you can install logprep using: +Then you have to update and build the helm subcharts repository: .. code-block:: bash - :caption: Install logprep + :caption: Add the bitnami helm repository + + helm dependencies update ./examples/k8s + helm dependencies build ./examples/k8s + +Next you are ready to install the opensiem example using: - helm install logprep charts/logprep +.. code-block:: bash + :caption: Install opensiem + + helm install opensiem examples/k8s + +Make the cluster locally resolvable: + +.. code-block:: bash + :caption: add hosts entry to resolve the cluster + + echo "$( minikube ip ) connector.opensiem dashboards.opensiem grafana.opensiem" | sudo tee -a /etc/hosts + +Test the defined ingresses: + +.. code-block:: bash + :caption: Test the opensiem example ingress + + curl -v http://connector.opensiem/health + curl -v http://dashboards.opensiem + +Test the opensiem connector: + +.. code-block:: bash + :caption: Test the opensiem example connector + + ❯ logprep generate http --input-dir ./examples/exampledata/input_logdata/ --target-url http://connector.opensiem --events 100 --batch-size 10 + + 2024-07-17 11:15:35 301643 Generator INFO : Log level set to 'NOTSET' + 2024-07-17 11:15:35 301643 Generator INFO : Started Data Processing + 2024-07-17 11:15:35 301643 Input INFO : Reading input dataset and creating temporary event collections in: '/tmp/logprep_a51e1vh6' + 2024-07-17 11:15:35 301643 Input INFO : Preparing data took: 0.0042 seconds + 2024-07-17 11:15:35 301643 Input INFO : Cleaned up temp dir: '/tmp/logprep_a51e1vh6' + 2024-07-17 11:15:35 301643 Generator INFO : Completed with following statistics: { + "Number of failed events": 0, + "Number of successfull events": 100, + "Requests Connection Errors": 0, + "Requests Timeouts": 0, + "Requests http status 200": 10, + "Requests total": 10 + } + 2024-07-17 11:15:35 301643 Generator INFO : Execution time: 0.067013 seconds + +open your browser and go to `http://dashboards.opensiem`_ to see the generated data in the opensearch dashboards. diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 47e1859f9..34189b618 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -61,4 +61,26 @@ Helm This option can be used to deploy logprep on a kubernetes cluster. -tbd +At first you have to install the prometheus PodMonitor CRD: + +.. code-block:: bash + :caption: Install the prometheus PodMonitor CRD + + kubectl apply -f https://raw.githubusercontent.com/prometheus-community/helm-charts/main/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml + + +To install latest stable release: + +.. code-block:: bash + + helm repo add logprep https://fkie-cad.github.io/Logprep + helm install logprep logprep/logprep + + +To install from cloned github repository: + +.. code-block:: bash + + git clone https://github.com/fkie-cad/Logprep.git + cd Logprep + helm install logprep charts/logprep diff --git a/quickstart/docker-compose.yml b/examples/compose/docker-compose.yml similarity index 86% rename from quickstart/docker-compose.yml rename to examples/compose/docker-compose.yml index 63f893e0d..4cb8e7b4d 100644 --- a/quickstart/docker-compose.yml +++ b/examples/compose/docker-compose.yml @@ -100,21 +100,21 @@ services: - kafka - opensearch volumes: - - ../quickstart/:/home/logprep/quickstart/ + - ../../examples/:/home/logprep/examples/ tmpfs: - /tmp/logprep/prometheus_multiproc entrypoint: - logprep - run - - /home/logprep/quickstart/exampledata/config/pipeline.yml + - /home/logprep/examples/exampledata/config/pipeline.yml grafana: image: bitnami/grafana:latest container_name: grafana ports: - "3000:3000" volumes: - - ../quickstart/exampledata/config/grafana/datasources:/opt/bitnami/grafana/conf/provisioning/datasources - - ../quickstart/exampledata/config/grafana/dashboards:/opt/bitnami/grafana/conf/provisioning/dashboards + - ../../examples/exampledata/config/grafana/datasources:/opt/bitnami/grafana/conf/provisioning/datasources + - ../../examples/exampledata/config/grafana/dashboards:/opt/bitnami/grafana/conf/provisioning/dashboards networks: - prometheus prometheus: @@ -123,7 +123,7 @@ services: ports: - "9090:9090" volumes: - - ../quickstart/exampledata/config/prometheus/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml + - ../../examples/exampledata/config/prometheus/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml networks: - kafka - prometheus @@ -138,9 +138,9 @@ services: ports: - "8081:8081" volumes: - - ../quickstart/exampledata:/usr/share/nginx/html:ro - - ../quickstart/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ../quickstart/exampledata/config/nginx/conf.d:/etc/nginx/conf.d:ro + - ../../examples/exampledata:/usr/share/nginx/html:ro + - ../../examples/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ../../examples/exampledata/config/nginx/conf.d:/etc/nginx/conf.d:ro mtls-config: image: nginx:latest container_name: mtls-config @@ -149,9 +149,9 @@ services: ports: - "8082:8082" volumes: - - ../quickstart/exampledata:/usr/share/nginx/html:ro - - ../quickstart/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ../quickstart/exampledata/config/nginx/mtls.conf.d:/etc/nginx/conf.d:ro + - ../../examples/exampledata:/usr/share/nginx/html:ro + - ../../examples/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ../../examples/exampledata/config/nginx/mtls.conf.d:/etc/nginx/conf.d:ro keycloak: image: bitnami/keycloak:latest container_name: keycloak @@ -160,7 +160,7 @@ services: profiles: - oauth2 volumes: - - ../quickstart/exampledata/config/keycloak:/opt/keycloak/data/import + - ../../examples/exampledata/config/keycloak:/opt/keycloak/data/import environment: KEYCLOAK_ADMIN_USER: admin KEYCLOAK_ADMIN_PASSWORD: admin @@ -263,8 +263,8 @@ services: ports: - "3002:80" volumes: - - ../quickstart/exampledata/config/fda/config.json:/usr/share/nginx/html/assets/config/keycloak/config.json - - ../quickstart/exampledata/config/fda/default.conf:/etc/nginx/conf.d/default.conf + - ../../examples/exampledata/config/fda/config.json:/usr/share/nginx/html/assets/config/keycloak/config.json + - ../../examples/exampledata/config/fda/default.conf:/etc/nginx/conf.d/default.conf networks: - fda ucl-api: @@ -338,9 +338,9 @@ services: ports: - "3001:80" volumes: - - ../quickstart/exampledata/config/ucl/nginx.example.conf:/etc/nginx/conf.d/default.conf - - ../quickstart/exampledata/config/ucl/general.example.json:/usr/share/nginx/html/assets/config/general/config.json - - ../quickstart/exampledata/config/ucl/keycloak.example.json:/usr/share/nginx/html/assets/config/keycloak/config.json + - ../../examples/exampledata/config/ucl/nginx.example.conf:/etc/nginx/conf.d/default.conf + - ../../examples/exampledata/config/ucl/general.example.json:/usr/share/nginx/html/assets/config/general/config.json + - ../../examples/exampledata/config/ucl/keycloak.example.json:/usr/share/nginx/html/assets/config/keycloak/config.json networks: - ucl diff --git a/quickstart/exampledata/config/credentials.yml b/examples/exampledata/config/credentials.yml similarity index 65% rename from quickstart/exampledata/config/credentials.yml rename to examples/exampledata/config/credentials.yml index 6a8c59496..8ca296a8a 100644 --- a/quickstart/exampledata/config/credentials.yml +++ b/examples/exampledata/config/credentials.yml @@ -15,14 +15,14 @@ getter: username: user password: password "https://localhost:8082": - client_key: quickstart/exampledata/config/nginx/mtls.conf.d/client.key - cert: quickstart/exampledata/config/nginx/mtls.conf.d/client.crt - ca_cert: quickstart/exampledata/config/nginx/mtls.conf.d/ca.crt + client_key: examples/exampledata/config/nginx/mtls.conf.d/client.key + cert: examples/exampledata/config/nginx/mtls.conf.d/client.crt + ca_cert: examples/exampledata/config/nginx/mtls.conf.d/ca.crt input: endpoints: /auth-json: username: user - password_file: quickstart/exampledata/config/user_password.txt + password_file: examples/exampledata/config/user_password.txt /lab/123/ABC/auditlog: username: user - password_file: quickstart/exampledata/config/user_password.txt + password_file: examples/exampledata/config/user_password.txt diff --git a/quickstart/exampledata/config/dummy-output.yml b/examples/exampledata/config/dummy-output.yml similarity index 100% rename from quickstart/exampledata/config/dummy-output.yml rename to examples/exampledata/config/dummy-output.yml diff --git a/quickstart/exampledata/config/fda/config.json b/examples/exampledata/config/fda/config.json similarity index 100% rename from quickstart/exampledata/config/fda/config.json rename to examples/exampledata/config/fda/config.json diff --git a/quickstart/exampledata/config/fda/default.conf b/examples/exampledata/config/fda/default.conf similarity index 100% rename from quickstart/exampledata/config/fda/default.conf rename to examples/exampledata/config/fda/default.conf diff --git a/quickstart/exampledata/config/grafana/dashboards/kafka.json b/examples/exampledata/config/grafana/dashboards/kafka.json similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/kafka.json rename to examples/exampledata/config/grafana/dashboards/kafka.json diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json b/examples/exampledata/config/grafana/dashboards/logprep-dashboard.json similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json rename to examples/exampledata/config/grafana/dashboards/logprep-dashboard.json diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-processors.json b/examples/exampledata/config/grafana/dashboards/logprep-processors.json similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/logprep-processors.json rename to examples/exampledata/config/grafana/dashboards/logprep-processors.json diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-provider.yml b/examples/exampledata/config/grafana/dashboards/logprep-provider.yml similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/logprep-provider.yml rename to examples/exampledata/config/grafana/dashboards/logprep-provider.yml diff --git a/quickstart/exampledata/config/grafana/dashboards/sample.yaml b/examples/exampledata/config/grafana/dashboards/sample.yaml similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/sample.yaml rename to examples/exampledata/config/grafana/dashboards/sample.yaml diff --git a/quickstart/exampledata/config/grafana/datasources/prometheus.yml b/examples/exampledata/config/grafana/datasources/prometheus.yml similarity index 100% rename from quickstart/exampledata/config/grafana/datasources/prometheus.yml rename to examples/exampledata/config/grafana/datasources/prometheus.yml diff --git a/quickstart/exampledata/config/grafana/datasources/sample.yaml b/examples/exampledata/config/grafana/datasources/sample.yaml similarity index 100% rename from quickstart/exampledata/config/grafana/datasources/sample.yaml rename to examples/exampledata/config/grafana/datasources/sample.yaml diff --git a/quickstart/exampledata/config/http_pipeline.yml b/examples/exampledata/config/http_pipeline.yml similarity index 100% rename from quickstart/exampledata/config/http_pipeline.yml rename to examples/exampledata/config/http_pipeline.yml diff --git a/quickstart/exampledata/config/keycloak/logprep_realm_export.json b/examples/exampledata/config/keycloak/logprep_realm_export.json similarity index 100% rename from quickstart/exampledata/config/keycloak/logprep_realm_export.json rename to examples/exampledata/config/keycloak/logprep_realm_export.json diff --git a/quickstart/exampledata/config/nginx/conf.d/.htpasswd b/examples/exampledata/config/nginx/conf.d/.htpasswd similarity index 100% rename from quickstart/exampledata/config/nginx/conf.d/.htpasswd rename to examples/exampledata/config/nginx/conf.d/.htpasswd diff --git a/quickstart/exampledata/config/nginx/conf.d/default.conf b/examples/exampledata/config/nginx/conf.d/default.conf similarity index 100% rename from quickstart/exampledata/config/nginx/conf.d/default.conf rename to examples/exampledata/config/nginx/conf.d/default.conf diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/ca.crt b/examples/exampledata/config/nginx/mtls.conf.d/ca.crt similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/ca.crt rename to examples/exampledata/config/nginx/mtls.conf.d/ca.crt diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/ca.key b/examples/exampledata/config/nginx/mtls.conf.d/ca.key similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/ca.key rename to examples/exampledata/config/nginx/mtls.conf.d/ca.key diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/client.crt b/examples/exampledata/config/nginx/mtls.conf.d/client.crt similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/client.crt rename to examples/exampledata/config/nginx/mtls.conf.d/client.crt diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/client.csr b/examples/exampledata/config/nginx/mtls.conf.d/client.csr similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/client.csr rename to examples/exampledata/config/nginx/mtls.conf.d/client.csr diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/client.key b/examples/exampledata/config/nginx/mtls.conf.d/client.key similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/client.key rename to examples/exampledata/config/nginx/mtls.conf.d/client.key diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/default.conf b/examples/exampledata/config/nginx/mtls.conf.d/default.conf similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/default.conf rename to examples/exampledata/config/nginx/mtls.conf.d/default.conf diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/server.crt b/examples/exampledata/config/nginx/mtls.conf.d/server.crt similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/server.crt rename to examples/exampledata/config/nginx/mtls.conf.d/server.crt diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/server.csr b/examples/exampledata/config/nginx/mtls.conf.d/server.csr similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/server.csr rename to examples/exampledata/config/nginx/mtls.conf.d/server.csr diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/server.key b/examples/exampledata/config/nginx/mtls.conf.d/server.key similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/server.key rename to examples/exampledata/config/nginx/mtls.conf.d/server.key diff --git a/quickstart/exampledata/config/nginx/nginx.conf b/examples/exampledata/config/nginx/nginx.conf similarity index 100% rename from quickstart/exampledata/config/nginx/nginx.conf rename to examples/exampledata/config/nginx/nginx.conf diff --git a/quickstart/exampledata/config/pipeline.yml b/examples/exampledata/config/pipeline.yml similarity index 67% rename from quickstart/exampledata/config/pipeline.yml rename to examples/exampledata/config/pipeline.yml index 71fd7e70b..b45ff72f2 100644 --- a/quickstart/exampledata/config/pipeline.yml +++ b/examples/exampledata/config/pipeline.yml @@ -15,26 +15,26 @@ metrics: pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic - dissector: type: dissector specific_rules: - - quickstart/exampledata/rules/dissector/specific/ + - examples/exampledata/rules/dissector/specific/ generic_rules: - - quickstart/exampledata/rules/dissector/generic/ + - examples/exampledata/rules/dissector/generic/ - dropper: type: dropper specific_rules: - - quickstart/exampledata/rules/dropper/specific + - examples/exampledata/rules/dropper/specific generic_rules: - - quickstart/exampledata/rules/dropper/generic + - examples/exampledata/rules/dropper/generic - filter: "test_dropper" dropper: drop: @@ -44,37 +44,37 @@ pipeline: - pre_detector: type: pre_detector specific_rules: - - quickstart/exampledata/rules/pre_detector/specific + - examples/exampledata/rules/pre_detector/specific generic_rules: - - quickstart/exampledata/rules/pre_detector/generic + - examples/exampledata/rules/pre_detector/generic outputs: - opensearch: sre - tree_config: quickstart/exampledata/rules/pre_detector/tree_config.json - alert_ip_list_path: quickstart/exampledata/rules/pre_detector/alert_ips.yml + tree_config: examples/exampledata/rules/pre_detector/tree_config.json + alert_ip_list_path: examples/exampledata/rules/pre_detector/alert_ips.yml - amides: type: amides specific_rules: - - quickstart/exampledata/rules/amides/specific + - examples/exampledata/rules/amides/specific generic_rules: - - quickstart/exampledata/rules/amides/generic - models_path: quickstart/exampledata/models/model.zip + - examples/exampledata/rules/amides/generic + models_path: examples/exampledata/models/model.zip num_rule_attributions: 10 max_cache_entries: 1000000 decision_threshold: 0.32 - pseudonymizer: type: pseudonymizer - pubkey_analyst: quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem - pubkey_depseudo: quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem - regex_mapping: quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml + pubkey_analyst: examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem + pubkey_depseudo: examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem + regex_mapping: examples/exampledata/rules/pseudonymizer/regex_mapping.yml hash_salt: a_secret_tasty_ingredient outputs: - opensearch: pseudonyms specific_rules: - - quickstart/exampledata/rules/pseudonymizer/specific/ + - examples/exampledata/rules/pseudonymizer/specific/ generic_rules: - - quickstart/exampledata/rules/pseudonymizer/generic/ + - examples/exampledata/rules/pseudonymizer/generic/ max_cached_pseudonyms: 1000000 - calculator: diff --git a/quickstart/exampledata/config/postgresql/keycloak_db.sql b/examples/exampledata/config/postgresql/keycloak_db.sql similarity index 100% rename from quickstart/exampledata/config/postgresql/keycloak_db.sql rename to examples/exampledata/config/postgresql/keycloak_db.sql diff --git a/quickstart/exampledata/config/prometheus/prometheus.yml b/examples/exampledata/config/prometheus/prometheus.yml similarity index 100% rename from quickstart/exampledata/config/prometheus/prometheus.yml rename to examples/exampledata/config/prometheus/prometheus.yml diff --git a/quickstart/exampledata/config/ucl/general.example.json b/examples/exampledata/config/ucl/general.example.json similarity index 100% rename from quickstart/exampledata/config/ucl/general.example.json rename to examples/exampledata/config/ucl/general.example.json diff --git a/quickstart/exampledata/config/ucl/keycloak.example.json b/examples/exampledata/config/ucl/keycloak.example.json similarity index 100% rename from quickstart/exampledata/config/ucl/keycloak.example.json rename to examples/exampledata/config/ucl/keycloak.example.json diff --git a/quickstart/exampledata/config/ucl/nginx.example.conf b/examples/exampledata/config/ucl/nginx.example.conf similarity index 100% rename from quickstart/exampledata/config/ucl/nginx.example.conf rename to examples/exampledata/config/ucl/nginx.example.conf diff --git a/quickstart/exampledata/config/user_password.txt b/examples/exampledata/config/user_password.txt similarity index 100% rename from quickstart/exampledata/config/user_password.txt rename to examples/exampledata/config/user_password.txt diff --git a/quickstart/exampledata/input_logdata/logclass/config.yaml b/examples/exampledata/input_logdata/logclass/config.yaml similarity index 100% rename from quickstart/exampledata/input_logdata/logclass/config.yaml rename to examples/exampledata/input_logdata/logclass/config.yaml diff --git a/quickstart/exampledata/input_logdata/logclass/test_input.jsonl b/examples/exampledata/input_logdata/logclass/test_input.jsonl similarity index 100% rename from quickstart/exampledata/input_logdata/logclass/test_input.jsonl rename to examples/exampledata/input_logdata/logclass/test_input.jsonl diff --git a/quickstart/exampledata/models/model.zip b/examples/exampledata/models/model.zip similarity index 100% rename from quickstart/exampledata/models/model.zip rename to examples/exampledata/models/model.zip diff --git a/quickstart/exampledata/rules/amides/generic/amides_generic.yml b/examples/exampledata/rules/amides/generic/amides_generic.yml similarity index 100% rename from quickstart/exampledata/rules/amides/generic/amides_generic.yml rename to examples/exampledata/rules/amides/generic/amides_generic.yml diff --git a/quickstart/exampledata/rules/amides/specific/amides_specific.yml b/examples/exampledata/rules/amides/specific/amides_specific.yml similarity index 100% rename from quickstart/exampledata/rules/amides/specific/amides_specific.yml rename to examples/exampledata/rules/amides/specific/amides_specific.yml diff --git a/quickstart/exampledata/rules/dissector/generic/example_rule.yml b/examples/exampledata/rules/dissector/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dissector/generic/example_rule.yml rename to examples/exampledata/rules/dissector/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/dissector/specific/example_rule.yml b/examples/exampledata/rules/dissector/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dissector/specific/example_rule.yml rename to examples/exampledata/rules/dissector/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/dropper/generic/example_rule.yml b/examples/exampledata/rules/dropper/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dropper/generic/example_rule.yml rename to examples/exampledata/rules/dropper/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/dropper/specific/example_rule.yml b/examples/exampledata/rules/dropper/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dropper/specific/example_rule.yml rename to examples/exampledata/rules/dropper/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/labeler/generic/example_rule.yml b/examples/exampledata/rules/labeler/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/labeler/generic/example_rule.yml rename to examples/exampledata/rules/labeler/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/labeler/schema.json b/examples/exampledata/rules/labeler/schema.json similarity index 100% rename from quickstart/exampledata/rules/labeler/schema.json rename to examples/exampledata/rules/labeler/schema.json diff --git a/quickstart/exampledata/rules/labeler/specific/example_rule.yml b/examples/exampledata/rules/labeler/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/labeler/specific/example_rule.yml rename to examples/exampledata/rules/labeler/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/pre_detector/alert_ips.yml b/examples/exampledata/rules/pre_detector/alert_ips.yml similarity index 100% rename from quickstart/exampledata/rules/pre_detector/alert_ips.yml rename to examples/exampledata/rules/pre_detector/alert_ips.yml diff --git a/quickstart/exampledata/rules/pre_detector/generic/example_rule.yml b/examples/exampledata/rules/pre_detector/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pre_detector/generic/example_rule.yml rename to examples/exampledata/rules/pre_detector/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/pre_detector/specific/example_rule.yml b/examples/exampledata/rules/pre_detector/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pre_detector/specific/example_rule.yml rename to examples/exampledata/rules/pre_detector/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/pre_detector/tree_config.json b/examples/exampledata/rules/pre_detector/tree_config.json similarity index 100% rename from quickstart/exampledata/rules/pre_detector/tree_config.json rename to examples/exampledata/rules/pre_detector/tree_config.json diff --git a/quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem b/examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem rename to examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem diff --git a/quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem b/examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem rename to examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem diff --git a/quickstart/exampledata/rules/pseudonymizer/generic/example_rule.yml b/examples/exampledata/rules/pseudonymizer/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/generic/example_rule.yml rename to examples/exampledata/rules/pseudonymizer/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml b/examples/exampledata/rules/pseudonymizer/regex_mapping.yml similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml rename to examples/exampledata/rules/pseudonymizer/regex_mapping.yml diff --git a/quickstart/exampledata/rules/pseudonymizer/specific/example_rule.yml b/examples/exampledata/rules/pseudonymizer/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/specific/example_rule.yml rename to examples/exampledata/rules/pseudonymizer/specific/example_rule.yml diff --git a/examples/k8s/.helmignore b/examples/k8s/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/examples/k8s/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/examples/k8s/Chart.yaml b/examples/k8s/Chart.yaml new file mode 100644 index 000000000..810227c04 --- /dev/null +++ b/examples/k8s/Chart.yaml @@ -0,0 +1,30 @@ +apiVersion: v2 +name: opensiem +description: a deployment example for logprep siem deployment +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: "1.0.0" + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.0.0" + +dependencies: + - name: logprep + repository: file://../../charts/logprep + version: 13.1.1 + alias: connector + - name: logprep + repository: file://../../charts/logprep + version: 13.1.1 + - name: kafka + repository: https://charts.bitnami.com/bitnami + version: 29.3.7 + - name: opensearch + repository: https://charts.bitnami.com/bitnami + version: 1.2.6 diff --git a/examples/k8s/templates/_helpers.tpl b/examples/k8s/templates/_helpers.tpl new file mode 100644 index 000000000..43a483388 --- /dev/null +++ b/examples/k8s/templates/_helpers.tpl @@ -0,0 +1,77 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "opensiem.name" -}} +{{- printf "%s" .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "opensiem.fullname" -}} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "opensiem.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "opensiem.release" -}} +{{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +{{- end }} + +{{/* +Common labels +*/}} +{{- define "opensiem.labels" -}} +helm.sh/chart: {{ include "opensiem.chart" . }} +{{ include "opensiem.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/application: {{ include "opensiem.name" . }} +{{- range $key, $value := .Values.extraLabels }} +{{ $key}}: {{ $value | quote }} +{{- end }} +{{- end }} + +{{/* +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "opensiem.selectorLabels" -}} +app.kubernetes.io/name: {{ include "opensiem.fullname" . }} +app.kubernetes.io/instance: {{ include "opensiem.release" . }} +{{- end }} + +{{/* +{{- end }} + +{{/* +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "opensiem.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "opensiem.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + diff --git a/examples/k8s/templates/ingress.yaml b/examples/k8s/templates/ingress.yaml new file mode 100644 index 000000000..551365712 --- /dev/null +++ b/examples/k8s/templates/ingress.yaml @@ -0,0 +1,53 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: connector-ingress +spec: + ingressClassName: nginx + rules: + - host: connector.opensiem + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opensiem-connector-http-input + port: + number: 9000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: dashboard-ingress +spec: + ingressClassName: nginx + rules: + - host: dashboards.opensiem + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opensiem-opensearch-dashboards + port: + number: 5601 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-ingress +spec: + ingressClassName: nginx + rules: + - host: grafana.opensiem + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opensiem-grafana + port: + number: 3000 diff --git a/examples/k8s/values.yaml b/examples/k8s/values.yaml new file mode 100644 index 000000000..2753c4415 --- /dev/null +++ b/examples/k8s/values.yaml @@ -0,0 +1,110 @@ +connector: + image: + tag: py3.11-latest + replicas: 1 + resources: + limits: + memory: "2Gi" + cpu: "1" + requests: + memory: "2Gi" + cpu: "250m" + secrets: {} + logger: + level: DEBUG + input: + type: http_input + message_backlog_size: 15000 + collect_meta: True + metafield_name: "@metadata" + uvicorn_config: + host: 0.0.0.0 + port: 9000 + workers: 2 + access_log: true + server_header: false + date_header: false + endpoints: + /auth-json: json + /json: json + /lab/123/(ABC|DEF)/pl.*: plaintext + /lab/123/ABC/auditlog: jsonl + /health: plaintext + output: + type: confluentkafka_output + topic: consumer + error_topic: errors + flush_timeout: 300 + send_timeout: 0 + kafka_config: + bootstrap.servers: opensiem-kafka:9092 + compression.type: gzip + statistics.interval.ms: "60000" + ingress: + enabled: true + +## for additional configurations see: `https://github.com/bitnami/charts/blob/main/bitnami/kafka/values.yaml` +kafka: + listeners: + client: + protocol: PLAINTEXT + controller: + replicaCount: 3 + metrics: + jmx: + enabled: true + provisioning: + enabled: true + replicationFactor: 3 + numPartitions: 10 + topics: + - name: consumer + - name: producer + +logprep: + image: + tag: py3.11-latest + input: + type: confluentkafka_input + topic: consumer + kafka_config: + bootstrap.servers: opensiem-kafka:9092 + group.id: cgroup3 + enable.auto.commit: "true" + auto.commit.interval.ms: "10000" + enable.auto.offset.store: "false" + queued.min.messages: "100000" + queued.max.messages.kbytes: "65536" + statistics.interval.ms: "60000" + preprocessing: + version_info_target_field: Logprep_version_info + log_arrival_time_target_field: event.ingested + hmac: + target: + key: "thisisasecureandrandomkey" + output_field: Full_event + output: + type: opensearch_output + hosts: + - opensiem-opensearch:9200 + default_index: processed + error_index: errors + message_backlog_size: 10000 + timeout: 10000 + flush_timeout: 60 + max_retries: 3 + parallel_bulk: false + user: admin + secret: admin +## for additional configurations see: `https://github.com/bitnami/charts/blob/main/bitnami/opensearch/values.yaml` +opensearch: + dashboards: + enabled: true + ingest: + replicaCount: 1 + master: + replicaCount: 1 + data: + replicaCount: 1 + coordinating: + replicaCount: 1 diff --git a/logprep/connector/http/input.py b/logprep/connector/http/input.py index 5f412d568..55690affd 100644 --- a/logprep/connector/http/input.py +++ b/logprep/connector/http/input.py @@ -45,7 +45,7 @@ endpoints: /firstendpoint: username: user - password_file: quickstart/exampledata/config/user_password.txt + password_file: examples/exampledata/config/user_password.txt /second*: username: user password: secret_password diff --git a/logprep/metrics/metrics.py b/logprep/metrics/metrics.py index 99ae80813..caf12bcf7 100644 --- a/logprep/metrics/metrics.py +++ b/logprep/metrics/metrics.py @@ -3,7 +3,7 @@ :code:`logprep_number_of_processed_events_total` or :code:`logprep_processing_time_per_event_sum`. Examples of grafana dashboards can be found in `the logprep github repo -`_ +`_ Configuration ============= diff --git a/logprep/processor/amides/processor.py b/logprep/processor/amides/processor.py index d6652d105..5e0a6de87 100644 --- a/logprep/processor/amides/processor.py +++ b/logprep/processor/amides/processor.py @@ -28,7 +28,7 @@ Since there is a plethora of different SIEM event types, the current implementation focuses on events that provide process command lines. Command lines are most commonly targeted by SIEM rules while they are also highly vulnerable to evasions. The rules and models for AMIDES provided in the -quickstart example are for Sysmon Process Creation events. In general, the Amides rule format allows +deployment examples are for Sysmon Process Creation events. In general, the Amides rule format allows to create rules for other event types that provide process command lines, e.g. Process Creation events generated by Windows Security Auditing. diff --git a/logprep/util/configuration.py b/logprep/util/configuration.py index f6f8f09c8..3dc5a6a90 100644 --- a/logprep/util/configuration.py +++ b/logprep/util/configuration.py @@ -53,26 +53,26 @@ pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic - dissectorname: type: dissector specific_rules: - - quickstart/exampledata/rules/dissector/specific/ + - examples/exampledata/rules/dissector/specific/ generic_rules: - - quickstart/exampledata/rules/dissector/generic/ + - examples/exampledata/rules/dissector/generic/ - dropper: type: dropper specific_rules: - - quickstart/exampledata/rules/dropper/specific + - examples/exampledata/rules/dropper/specific generic_rules: - - quickstart/exampledata/rules/dropper/generic + - examples/exampledata/rules/dropper/generic - filter: "test_dropper" dropper: drop: @@ -82,37 +82,37 @@ - pre_detector: type: pre_detector specific_rules: - - quickstart/exampledata/rules/pre_detector/specific + - examples/exampledata/rules/pre_detector/specific generic_rules: - - quickstart/exampledata/rules/pre_detector/generic + - examples/exampledata/rules/pre_detector/generic outputs: - opensearch: sre - tree_config: quickstart/exampledata/rules/pre_detector/tree_config.json - alert_ip_list_path: quickstart/exampledata/rules/pre_detector/alert_ips.yml + tree_config: examples/exampledata/rules/pre_detector/tree_config.json + alert_ip_list_path: examples/exampledata/rules/pre_detector/alert_ips.yml - amides: type: amides specific_rules: - - quickstart/exampledata/rules/amides/specific + - examples/exampledata/rules/amides/specific generic_rules: - - quickstart/exampledata/rules/amides/generic - models_path: quickstart/exampledata/models/model.zip + - examples/exampledata/rules/amides/generic + models_path: examples/exampledata/models/model.zip num_rule_attributions: 10 max_cache_entries: 1000000 decision_threshold: 0.32 - pseudonymizer: type: pseudonymizer - pubkey_analyst: quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem - pubkey_depseudo: quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem - regex_mapping: quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml + pubkey_analyst: examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem + pubkey_depseudo: examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem + regex_mapping: examples/exampledata/rules/pseudonymizer/regex_mapping.yml hash_salt: a_secret_tasty_ingredient outputs: - opensearch: pseudonyms specific_rules: - - quickstart/exampledata/rules/pseudonymizer/specific/ + - examples/exampledata/rules/pseudonymizer/specific/ generic_rules: - - quickstart/exampledata/rules/pseudonymizer/generic/ + - examples/exampledata/rules/pseudonymizer/generic/ max_cached_pseudonyms: 1000000 - calculator: @@ -173,12 +173,12 @@ pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic" + - examples/exampledata/rules/labeler/generic" export LOGPREP_OUTPUT=" output: kafka: diff --git a/tests/acceptance/test_full_configuration.py b/tests/acceptance/test_full_configuration.py index a09dd3fb6..5175c31ab 100644 --- a/tests/acceptance/test_full_configuration.py +++ b/tests/acceptance/test_full_configuration.py @@ -87,12 +87,12 @@ def test_start_of_logprep_from_http_with_templated_url_and_config(): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic """, "LOGPREP_OUTPUT": """ output: diff --git a/tests/testdata/unit/requester/generic_rules/requester.json b/tests/testdata/unit/requester/generic_rules/requester.json index 2564b1e24..d7154a894 100644 --- a/tests/testdata/unit/requester/generic_rules/requester.json +++ b/tests/testdata/unit/requester/generic_rules/requester.json @@ -2,7 +2,7 @@ { "filter": "message1", "requester": { - "url": "http://localhost:32000/quickstart/exampledata/config/pipeline.yml", + "url": "http://localhost:32000/examples/exampledata/config/pipeline.yml", "method": "GET" } } diff --git a/tests/testdata/unit/requester/specific_rules/requester.json b/tests/testdata/unit/requester/specific_rules/requester.json index 1ae41c251..701c4e95e 100644 --- a/tests/testdata/unit/requester/specific_rules/requester.json +++ b/tests/testdata/unit/requester/specific_rules/requester.json @@ -2,7 +2,7 @@ { "filter": "message", "requester": { - "url": "http://localhost:32000/quickstart/exampledata/config/pipeline.yml", + "url": "http://localhost:32000/examples/exampledata/config/pipeline.yml", "method": "GET" } } diff --git a/tests/unit/connector/test_real_kafka.py b/tests/unit/connector/test_real_kafka.py index 25eb6a909..a35d00f8b 100644 --- a/tests/unit/connector/test_real_kafka.py +++ b/tests/unit/connector/test_real_kafka.py @@ -24,7 +24,7 @@ def setup_module(): if not in_ci: subprocess.run( - ["docker", "compose", "-f", "quickstart/docker-compose.yml", "up", "-d", "kafka"] + ["docker", "compose", "-f", "example/compose/docker-compose.yml", "up", "-d", "kafka"] ) diff --git a/tests/unit/test_quickstart.py b/tests/unit/test_quickstart.py index 18be79ee7..fbd686676 100644 --- a/tests/unit/test_quickstart.py +++ b/tests/unit/test_quickstart.py @@ -5,19 +5,19 @@ from logprep import run_logprep -class TestQuickstart: - QUICKSTART_CONFIG_PATH = "quickstart/exampledata/config/pipeline.yml" +class TestExampleCompose: + EXAMPLE_CONFIG_PATH = "examples/exampledata/config/pipeline.yml" @mock.patch("os.environ", new={"PROMETHEUS_MULTIPROC_DIR": "/tmp"}) - def test_quickstart_setup_is_valid(self): - """ensures the quickstart rules are valid""" + def test_example_compose_setup_is_valid(self): + """ensures the example rules are valid""" with mock.patch( "sys.argv", [ "logprep", "test", "config", - self.QUICKSTART_CONFIG_PATH, + self.EXAMPLE_CONFIG_PATH, ], ): with pytest.raises(SystemExit) as e_info: diff --git a/tests/unit/test_run_logprep.py b/tests/unit/test_run_logprep.py index 7f61b55d0..cf4e4870a 100644 --- a/tests/unit/test_run_logprep.py +++ b/tests/unit/test_run_logprep.py @@ -55,7 +55,7 @@ def setup_method(self): "logprep.util.configuration.Configuration.as_yaml", ), ( - "test dry-run tests/testdata/config/config.yml quickstart/exampledata/input_logdata/test_input.jsonl", + "test dry-run tests/testdata/config/config.yml examples/exampledata/input_logdata/test_input.jsonl", "logprep.util.rule_dry_runner.DryRunner.run", ), ( @@ -255,7 +255,7 @@ def test_logprep_exits_on_request_exception(self): @mock.patch("logprep.util.rule_dry_runner.DryRunner.run") def test_test_dry_run_starts_dry_runner(self, mock_dry_runner): config_path = ("tests/testdata/config/config.yml",) - events_path = "quickstart/exampledata/input_logdata/test_input.jsonl" + events_path = "examples/exampledata/input_logdata/test_input.jsonl" result = self.cli_runner.invoke(cli, ["test", "dry-run", *config_path, events_path]) assert result.exit_code == 0 mock_dry_runner.assert_called() diff --git a/tests/unit/util/test_configuration.py b/tests/unit/util/test_configuration.py index 3c067d58a..4a027b339 100644 --- a/tests/unit/util/test_configuration.py +++ b/tests/unit/util/test_configuration.py @@ -196,7 +196,7 @@ def test_pipeline_property_is_merged_from_configs(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] @@ -377,9 +377,9 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], } } ] @@ -393,10 +393,10 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], - "generic_rules": ["quickstart/exampledata/rules/labeler/generic"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], + "generic_rules": ["examples/exampledata/rules/labeler/generic"], "SOME_UNKNOWN_OPTION": "FOO", } } @@ -411,10 +411,10 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], - "generic_rules": ["quickstart/exampledata/rules/labeler/generic"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], + "generic_rules": ["examples/exampledata/rules/labeler/generic"], "SOME UNKNOWN OPTION": "FOO", } } @@ -444,10 +444,10 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], - "generic_rules": ["quickstart/exampledata/rules/labeler/generic"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], + "generic_rules": ["examples/exampledata/rules/labeler/generic"], "SOME UNKNOWN OPTION": "FOO", } }, @@ -523,12 +523,12 @@ def test_verify_verifies_config(self, tmp_path, test_case, test_config, error_co pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic """, "LOGPREP_OUTPUT": """ output: @@ -780,7 +780,7 @@ def test_reload_raises_on_invalid_processor_config(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] @@ -805,7 +805,7 @@ def test_reload_raises_on_invalid_processor_config(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] @@ -854,7 +854,7 @@ def test_reload_raises_on_same_version(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] diff --git a/tests/unit/util/test_getter.py b/tests/unit/util/test_getter.py index c64d51d39..ddbeb2c27 100644 --- a/tests/unit/util/test_getter.py +++ b/tests/unit/util/test_getter.py @@ -545,9 +545,7 @@ def test_get_raw_raises_if_credential_file_env_set_and_unauthorizes(self): http_getter: HttpGetter = GetterFactory.from_string(f"https://{domain}/bar") with mock.patch.dict( "os.environ", - { - ENV_NAME_LOGPREP_CREDENTIALS_FILE: "quickstart/exampledata/config/credentials.yml" - }, + {ENV_NAME_LOGPREP_CREDENTIALS_FILE: "examples/exampledata/config/credentials.yml"}, ): http_getter.get_json() assert error.value.response.status_code == 401