diff --git a/charts/logprep/templates/gateway.yaml b/charts/logprep/templates/gateway.yaml deleted file mode 100644 index 3cf1090e0..000000000 --- a/charts/logprep/templates/gateway.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{- if .Values.ingress.enabled -}} ---- -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: {{ include "logprep.fullname" . }} - labels: - {{- include "logprep.labels" . | nindent 4 }} -spec: - selector: - istio: ingressgateway - servers: - - hosts: - - "*" - port: - name: http - number: 80 - protocol: HTTP - tls: - httpsRedirect: true - - hosts: - - "*" - port: - name: https - number: 443 - protocol: HTTPS - tls: - credentialName: {{ .Values.ingress.certificate.name }} - mode: SIMPLE -{{- end}} \ No newline at end of file diff --git a/charts/logprep/templates/virtual-service.yaml b/charts/logprep/templates/virtual-service.yaml deleted file mode 100644 index 1557ea250..000000000 --- a/charts/logprep/templates/virtual-service.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- if .Values.input -}} -{{- if eq .Values.input.type "http_input" -}} ---- -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: {{ include "logprep.fullname" . }} - labels: - {{- include "logprep.labels" . | nindent 4 }} -spec: - gateways: - - {{ include "logprep.fullname" . }}-gateway - hosts: - - {{ .Values.ingress.domain }} - http: - {{ $logprep_fullname := include "logprep.fullname" .}} - {{ $uvicorn_port := .Values.input.uvicorn_config.port }} - {{ $response_headers := .Values.ingress.response_headers }} - {{ range $key, $value := .Values.input.endpoints }} - - match: - - uri: - regex: {{ $key | quote }} - route: - - destination: - host: {{ $logprep_fullname }}-http-input - port: - number: {{ $uvicorn_port }} - headers: - response: - set: - {{- toYaml $response_headers | nindent 16 }} - {{ end }} -{{- end}} -{{- end}} -{{- end}} diff --git a/charts/logprep/values.yaml b/charts/logprep/values.yaml index 763fff430..a4204b0d6 100644 --- a/charts/logprep/values.yaml +++ b/charts/logprep/values.yaml @@ -167,22 +167,3 @@ configurations: # admin # admin2 artifacts: [] - -# The ingress configuration -# If enabled, an istio based ingress will be deployed. This option is only useful -# if the logprep configuration has a http_input configured. -ingress: - enabled: false - domain: "localhost" - certificate: - name: "istio-gateway-certificate" - response_headers: - Cache-Control: no-cache - Content-Security-Policy: default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'self'; - Cross-Origin-Resource-Policy: same-site - Referrer-Policy: strict-origin-when-cross-origin - Strict-Transport-Security: max-age=31536000; includeSubdomains - Permissions-Policy: geolocation=(), camera=(), microphone=() - X-XSS-Protection: 1; mode=block - X-Content-Type-Options: nosniff - X-Frame-Options: DENY diff --git a/doc/source/examples/minikube.rst b/doc/source/examples/minikube.rst index a56572930..b0fc8559e 100644 --- a/doc/source/examples/minikube.rst +++ b/doc/source/examples/minikube.rst @@ -51,6 +51,7 @@ with the following commands: minikube config set driver docker minikube config set cpus 16 minikube config set memory 16GB + minikube addons enable ingress minikube start Deploy the example @@ -64,51 +65,56 @@ At first you have to install the prometheus PodMonitor CRD: kubectl apply -f https://raw.githubusercontent.com/prometheus-community/helm-charts/main/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml -Then install istio (for details see: `https://istio.io/latest/docs/setup/install/helm/`_. ): +Then you have to update and build the helm subcharts repository: .. code-block:: bash - :caption: Install istio + :caption: Add the bitnami helm repository - kubectl create namespace istio-system - helm repo add istio https://istio-release.storage.googleapis.com/charts - helm repo update - helm install istio-base istio/base -n istio-system --set defaultRevision=opensiem --wait - helm install istiod istio/istiod -n istio-system --wait + helm dependencies update ./examples/k8s + helm dependencies build ./examples/k8s +Next you are ready to install the opensiem example using: .. code-block:: bash - :caption: Install istio ingress gateway - - kubectl create namespace istio-ingress - helm install istio-ingress istio/gateway -n istio-ingress + :caption: Install opensiem -.. code-block:: bash - :caption: Verifiy the istio installation + helm install opensiem examples/k8s - ❯ helm ls -n istio-system - NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION - istio-base istio-system 1 2024-07-15 14:54:54.029747408 +0200 CEST deployed base-1.22.2 1.22.2 - istiod istio-system 1 2024-07-15 14:57:41.496783572 +0200 CEST deployed istiod-1.22.2 1.22.2 +Make the cluster locally resolvable: - ❯ kubectl get deployments -n istio-system --output wide - NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR - istiod 1/1 1 1 24m discovery docker.io/istio/pilot:1.22.2 istio=pilot +.. code-block:: bash + :caption: add hosts entry to resolve the cluster - ❯ kubectl get pods -n istio-ingress - NAME READY STATUS RESTARTS AGE - istio-ingress-7f5f6f58b8-sv6gk 1/1 Running 0 16m + echo "$( minikube ip ) connector.opensiem dashboards.opensiem grafana.opensiem" | sudo tee -a /etc/hosts -Then you have to update and build the helm subcharts repository: +Test the defined ingresses: .. code-block:: bash - :caption: Add the bitnami helm repository + :caption: Test the opensiem example ingress - helm dependencies update ./examples/k8s - helm dependencies build ./examples/k8s + curl -v http://connector.opensiem/health + curl -v http://dashboards.opensiem -Next you are ready to install the opensiem example using: +Test the opensiem connector: .. code-block:: bash - :caption: Install opensiem + :caption: Test the opensiem example connector - helm install opensiem examples/k8s + ❯ logprep generate http --input-dir ./examples/exampledata/input_logdata/ --target-url http://connector.opensiem --events 100 --batch-size 10 + + 2024-07-17 11:15:35 301643 Generator INFO : Log level set to 'NOTSET' + 2024-07-17 11:15:35 301643 Generator INFO : Started Data Processing + 2024-07-17 11:15:35 301643 Input INFO : Reading input dataset and creating temporary event collections in: '/tmp/logprep_a51e1vh6' + 2024-07-17 11:15:35 301643 Input INFO : Preparing data took: 0.0042 seconds + 2024-07-17 11:15:35 301643 Input INFO : Cleaned up temp dir: '/tmp/logprep_a51e1vh6' + 2024-07-17 11:15:35 301643 Generator INFO : Completed with following statistics: { + "Number of failed events": 0, + "Number of successfull events": 100, + "Requests Connection Errors": 0, + "Requests Timeouts": 0, + "Requests http status 200": 10, + "Requests total": 10 + } + 2024-07-17 11:15:35 301643 Generator INFO : Execution time: 0.067013 seconds + +open your browser and go to `http://dashboards.opensiem`_ to see the generated data in the opensearch dashboards. diff --git a/examples/k8s/templates/ingress.yaml b/examples/k8s/templates/ingress.yaml new file mode 100644 index 000000000..551365712 --- /dev/null +++ b/examples/k8s/templates/ingress.yaml @@ -0,0 +1,53 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: connector-ingress +spec: + ingressClassName: nginx + rules: + - host: connector.opensiem + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opensiem-connector-http-input + port: + number: 9000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: dashboard-ingress +spec: + ingressClassName: nginx + rules: + - host: dashboards.opensiem + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opensiem-opensearch-dashboards + port: + number: 5601 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-ingress +spec: + ingressClassName: nginx + rules: + - host: grafana.opensiem + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opensiem-grafana + port: + number: 3000 diff --git a/tests/unit/charts/test_ingress_gateway.py b/tests/unit/charts/test_ingress_gateway.py deleted file mode 100644 index 3765e842d..000000000 --- a/tests/unit/charts/test_ingress_gateway.py +++ /dev/null @@ -1,28 +0,0 @@ -# pylint: disable=missing-docstring -# pylint: disable=attribute-defined-outside-init -# pylint: disable=protected-access - - -from tests.unit.charts.test_base import TestBaseChartTest - - -class TestIngressGateway(TestBaseChartTest): - - def test_ingress_gateway_is_rendered(self): - logprep_values = {"ingress": {"enabled": True}} - self.manifests = self.render_chart("logprep", logprep_values) - ingress_gateway = self.manifests.by_query( - "kind: Gateway AND apiVersion: networking.istio.io/v1alpha3" - ) - assert ingress_gateway - assert len(ingress_gateway) == 1 - ingress_gateway = ingress_gateway[0] - assert ingress_gateway["metadata"]["name"] == "logprep-logprep" - - def test_ingress_gateway_is_not_rendered(self): - logprep_values = {"ingress": {"enabled": False}} - self.manifests = self.render_chart("logprep", logprep_values) - ingress_gateway = self.manifests.by_query( - "kind: Gateway AND apiVersion: networking.istio.io/v1alpha3" - ) - assert not ingress_gateway diff --git a/tests/unit/charts/test_virtual_service.py b/tests/unit/charts/test_virtual_service.py deleted file mode 100644 index a3952cb58..000000000 --- a/tests/unit/charts/test_virtual_service.py +++ /dev/null @@ -1,78 +0,0 @@ -# pylint: disable=missing-docstring -# pylint: disable=attribute-defined-outside-init -# pylint: disable=protected-access - - -from logprep.util.configuration import yaml -from tests.unit.charts.test_base import TestBaseChartTest - -http_input_config = { - "type": "http_input", - "message_backlog_size": 150, - "collect_meta": True, - "metafield_name": "@metadata", - "uvicorn_config": { - "host": "0.0.0.0", - "port": 9000, - "workers": 2, - "access_log": True, - "server_header": False, - "date_header": False, - }, - "endpoints": { - "/auth-json": "json", - "/json": "json", - "/lab/123/(ABC|DEF)/pl.*": "plaintext", - "/lab/123/ABC/auditlog": "jsonl", - }, -} - - -class TestIstioVirtualService(TestBaseChartTest): - - def test_virtual_service_is_rendered(self): - logprep_values = {"ingress": {"enabled": True}, "input": http_input_config} - self.manifests = self.render_chart("logprep", logprep_values) - virtual_service = self.manifests.by_query( - "kind: VirtualService AND apiVersion: networking.istio.io/v1alpha3" - ) - assert virtual_service - assert len(virtual_service) == 1 - virtual_service = virtual_service[0] - assert virtual_service["metadata"]["name"] == "logprep-logprep" - - def test_virtual_service_has_endpoint_routes(self): - logprep_values = {"ingress": {"enabled": True}, "input": http_input_config} - self.manifests = self.render_chart("logprep", logprep_values) - virtual_service = self.manifests.by_query( - "kind: VirtualService AND apiVersion: networking.istio.io/v1alpha3" - )[0] - defined_routes = [ - route["match"][0]["uri"]["regex"] for route in virtual_service["spec.http"] - ] - for endpoint in http_input_config["endpoints"]: - assert endpoint in defined_routes - - def test_virtual_service_routes_have_response_header(self): - logprep_values = {"ingress": {"enabled": True}, "input": http_input_config} - self.manifests = self.render_chart("logprep", logprep_values) - virtual_service = self.manifests.by_query( - "kind: VirtualService AND apiVersion: networking.istio.io/v1alpha3" - )[0] - response_headers_for_routes = [ - route["route"][0]["headers"]["response"]["set"] - for route in virtual_service["spec.http"] - ] - expected_headers = { - "Cache-Control": "no-cache", - "Content-Security-Policy": "default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'self';", - "Cross-Origin-Resource-Policy": "same-site", - "Referrer-Policy": "strict-origin-when-cross-origin", - "Strict-Transport-Security": "max-age=31536000; includeSubdomains", - "Permissions-Policy": "geolocation=(), camera=(), microphone=()", - "X-XSS-Protection": "1; mode=block", - "X-Content-Type-Options": "nosniff", - "X-Frame-Options": "DENY", - } - for headers in response_headers_for_routes: - assert headers == expected_headers