Event export using vector & sidecar

.env

@@ -12,7 +12,7 @@ POSTGRES_PORT=5432
 API_BASE_URL_PATH=/auditlog
 RETRACED_API_BASE=http://localhost:3000/auditlog
 POSTGRES_DATABASE=retraced
-LOG_LEVEL=warn
+LOG_LEVEL=info
 ELASTICSEARCH_NODES=http://127.0.0.1:9200
 NSQD_HOST=127.0.0.1
 HOSTNAME=retraced-api-67856674bf-kwq7f
@@ -28,6 +28,10 @@ NEXTAUTH_SECRET=secret
 RETRACED_HOST_URL=http://retraced-api:3000/auditlog
 RETRACED_EXTERNAL_URL=http://localhost:3000/auditlog
 
+# Export Logs
+# EXPORT_WEBHOOK_URL=http://vector:9000
+# EXPORT_WEBHOOK_USERNAME=admin
+# EXPORT_WEBHOOK_PASSWORD=admin
 # OpenTelemetry 
 # https://opentelemetry.io/docs/concepts/sdk-configuration/otlp-exporter-configuration/
 # If you have any issues with using the otel exporter and want to enable debug logs
@@ -41,4 +45,4 @@ OTEL_EXPORTER_OTLP_METRICS_HEADERS=
 GEOIPUPDATE_LICENSE_KEY=
 GEOIPUPDATE_ACCOUNT_ID=
 GEOIPUPDATE_USE_MMDB=
-GEOIPUPDATE_DB_DIR=/etc/mmdb
+GEOIPUPDATE_DB_DIR=/etc/mmdb

.github/workflows/main.yml

@@ -153,14 +153,18 @@ jobs:
         run: |
           echo "SHA7=$(echo ${GITHUB_SHA} | cut -c1-7)" >> $GITHUB_OUTPUT
           imagePath="${{ secrets.DOCKER_HUB_USERNAME }}/retraced"
+          sidecarPath="${{ secrets.DOCKER_HUB_USERNAME }}/vector"
 
           if [[ "$GITHUB_REF" != *\/release ]]
           then
             imagePath="${{ secrets.DOCKER_HUB_USERNAME }}/retraced-beta"
+            sidecarPath="${{ secrets.DOCKER_HUB_USERNAME }}/vector-beta"
           fi
           echo "${imagePath}"
+          echo "${sidecarPath}"
 
-          echo "IMAGE_PATH=${imagePath}" >> $GITHUB_OUTPUT
+          echo "RETRACED_IMAGE_PATH=${imagePath}" >> $GITHUB_OUTPUT
+          echo "VECTOR_IMAGE_PATH=${sidecarPath}" >> $GITHUB_OUTPUT
 
       - name: Set up Docker Buildx
         if: github.ref == 'refs/heads/release'
@@ -178,20 +182,35 @@ jobs:
           username: boxyhq
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
-      - name: Build and push
+      - name: Build and push Retraced
         if: github.ref == 'refs/heads/release'
-        id: docker_build
+        id: docker_build_retraced
         uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./deploy/Dockerfile-slim
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ steps.slug.outputs.IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }},${{ steps.slug.outputs.IMAGE_PATH }}:${{ steps.slug.outputs.SHA7 }},${{ steps.slug.outputs.IMAGE_PATH }}:${{ needs.ci.outputs.NPM_VERSION }}
+          tags: ${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }},${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ steps.slug.outputs.SHA7 }},${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ needs.ci.outputs.NPM_VERSION }}
 
-      - name: Image digest
+      - name: Build and push Vector
         if: github.ref == 'refs/heads/release'
-        run: echo ${{ steps.docker_build.outputs.digest }}
+        id: docker_build_vector
+        uses: docker/build-push-action@v5
+        with:
+          context: ./
+          file: ./deploy/vector-sidecar/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.slug.outputs.VECTOR_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }},${{ steps.slug.outputs.VECTOR_IMAGE_PATH }}:${{ steps.slug.outputs.SHA7 }},${{ steps.slug.outputs.VECTOR_IMAGE_PATH }}:${{ needs.ci.outputs.NPM_VERSION }}
+
+      - name: Image digest Retraced
+        if: github.ref == 'refs/heads/release'
+        run: echo ${{ steps.docker_build_retraced.outputs.digest }}
+
+      - name: Image digest Vector
+        if: github.ref == 'refs/heads/release'
+        run: echo ${{ steps.docker_build_vector.outputs.digest }}
 
       - name: Login to GitHub Container Registry
         if: github.ref == 'refs/heads/release'
@@ -213,9 +232,15 @@ jobs:
         env:
           COSIGN_KEY: ${{secrets.COSIGN_KEY}}
 
-      - name: Sign the image
+      - name: Sign the image [Retraced]
         if: github.ref == 'refs/heads/release'
-        run: cosign sign --key /tmp/cosign.key -y ${{ steps.slug.outputs.IMAGE_PATH }}@${{ steps.docker_build.outputs.digest }}
+        run: cosign sign --key /tmp/cosign.key -y ${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}@${{ steps.docker_build_retraced.outputs.digest }}
+        env:
+          COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}}
+
+      - name: Sign the image [Vector]
+        if: github.ref == 'refs/heads/release'
+        run: cosign sign --key /tmp/cosign.key -y ${{ steps.slug.outputs.VECTOR_IMAGE_PATH }}@${{ steps.docker_build_vector.outputs.digest }}
         env:
           COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}}
 
@@ -224,82 +249,127 @@ jobs:
         with:
           format: spdx
           artifact-name: retraced_sbom.spdx
+
       - name: Publish report [SPDX]
         uses: anchore/sbom-action/publish-sbom@v0
         with:
           sbom-artifact-match: ".*\\.spdx$"
+
       - name: Create SBOM Report [CycloneDx]
         uses: anchore/sbom-action@v0
         with:
           format: cyclonedx
           artifact-name: retraced_sbom.cyclonedx
+
       - name: Publish report [CycloneDx]
         uses: anchore/sbom-action/publish-sbom@v0
         with:
           sbom-artifact-match: ".*\\.cyclonedx$"
+
       - name: Download artifact for SPDX Report
         if: github.ref == 'refs/heads/release'
         uses: actions/download-artifact@v3
         with:
           name: retraced_sbom.spdx
+
       - name: Download artifact for CycloneDx Report
         if: github.ref == 'refs/heads/release'
         uses: actions/download-artifact@v3
         with:
           name: retraced_sbom.cyclonedx
+
       - name: Remove older SBOMs
         if: github.ref == 'refs/heads/release'
         run: rm -rf ./sbom*.* || true
+
       - name: Move SPDX Report
         if: github.ref == 'refs/heads/release'
         run: mv retraced_sbom.spdx "./sbom.spdx"
+
       - name: Move CycloneDx Report
         if: github.ref == 'refs/heads/release'
         run: mv retraced_sbom.cyclonedx "./sbom.cyclonedx"
 
-      - name: Create SBOM Report [Docker][SPDX]
+      - name: Create SBOM Report [Docker][SPDX][Retraced]
+        if: github.ref == 'refs/heads/release'
+        uses: anchore/sbom-action@v0
+        with:
+          image: ${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
+          format: spdx
+          artifact-name: retraced_docker_sbom.spdx
+
+      - name: Create SBOM Report [Docker][SPDX][Vector]
         if: github.ref == 'refs/heads/release'
         uses: anchore/sbom-action@v0
         with:
-          image: ${{ steps.slug.outputs.IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
+          image: ${{ steps.slug.outputs.VECTOR_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
           format: spdx
-          artifact-name: docker_sbom.spdx
+          artifact-name: vector_docker_sbom.spdx
+
       - name: Publish report [Docker][SPDX]
         if: github.ref == 'refs/heads/release'
         uses: anchore/sbom-action/publish-sbom@v0
         with:
           sbom-artifact-match: ".*\\.spdx$"
-      - name: Create SBOM Report [Docker][CycloneDx]
+
+      - name: Create SBOM Report [Docker][CycloneDx][Retraced]
+        if: github.ref == 'refs/heads/release'
+        uses: anchore/sbom-action@v0
+        with:
+          image: ${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
+          format: cyclonedx
+          artifact-name: retraced_docker_sbom.cyclonedx
+
+      - name: Create SBOM Report [Docker][CycloneDx][Vector]
         if: github.ref == 'refs/heads/release'
         uses: anchore/sbom-action@v0
         with:
-          image: ${{ steps.slug.outputs.IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
+          image: ${{ steps.slug.outputs.VECTOR_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
           format: cyclonedx
-          artifact-name: docker_sbom.cyclonedx
+          artifact-name: vector_docker_sbom.cyclonedx
+
       - name: Publish report [Docker][CycloneDx]
         if: github.ref == 'refs/heads/release'
         uses: anchore/sbom-action/publish-sbom@v0
         with:
           sbom-artifact-match: ".*\\.cyclonedx$"
-      - name: Download artifact for SPDX Report [Docker]
+
+      - name: Download artifact for SPDX Report [Docker][Retraced]
         if: github.ref == 'refs/heads/release'
         uses: actions/download-artifact@v3
         with:
-          name: docker_sbom.spdx
-      - name: Download artifact for CycloneDx Report [Docker]
+          name: retraced_docker_sbom.spdx
+
+      - name: Download artifact for SPDX Report [Docker][Vector]
+        if: github.ref == 'refs/heads/release'
+        uses: actions/download-artifact@v3
+        with:
+          name: vector_docker_sbom.spdx
+
+      - name: Download artifact for CycloneDx Report [Docker][Retraced]
         if: github.ref == 'refs/heads/release'
         uses: actions/download-artifact@v3
         with:
-          name: docker_sbom.cyclonedx
+          name: retraced_docker_sbom.cyclonedx
+
+      - name: Download artifact for CycloneDx Report [Docker][Vector]
+        if: github.ref == 'refs/heads/release'
+        uses: actions/download-artifact@v3
+        with:
+          name: vector_docker_sbom.cyclonedx
+
       - name: Create/Clear folder [Docker]
         if: github.ref == 'refs/heads/release'
         run: mkdir -p ./_docker/ && rm -rf ./_docker/*.* || true
 
       - name: Move Report & cleanup [Docker]
         if: github.ref == 'refs/heads/release'
         run: |
-          mv docker_sbom.spdx "./_docker/sbom.spdx" || true
-          mv docker_sbom.cyclonedx ./_docker/sbom.cyclonedx || true
+          mv retraced_docker_sbom.spdx "./_docker/retraced_sbom.spdx" || true
+          mv retraced_docker_sbom.cyclonedx ./_docker/retraced_sbom.cyclonedx || true
+          mv vector_docker_sbom.spdx "./_docker/vector_sbom.spdx" || true
+          mv vector_docker_sbom.cyclonedx ./_docker/vector_sbom.cyclonedx || true
+
       - name: ORAS Setup
         if: github.ref == 'refs/heads/release'
         run: |
@@ -320,7 +390,7 @@ jobs:
           fi
           cosign sign -y --key /tmp/cosign.key ghcr.io/${{ github.repository }}/sbom${{ needs.ci.outputs.IMAGE_SUFFIX }}@${ORAS_DIGEST}
           cd _docker || true
-          result=$(../oras_install/oras push ghcr.io/${{ github.repository }}/sbom${{ needs.ci.outputs.IMAGE_SUFFIX }}:docker-${{ needs.ci.outputs.NPM_VERSION }} ./sbom.*)
+          result=$(../oras_install/oras push ghcr.io/${{ github.repository }}/sbom${{ needs.ci.outputs.IMAGE_SUFFIX }}:docker-${{ needs.ci.outputs.NPM_VERSION }} ./*sbom.*)
           ORAS_DIGEST=$(echo $result | grep -oE 'sha256:[a-f0-9]{64}')
           if [ -z "$ORAS_DIGEST" ]; then
             echo "Error: ORAS_DIGEST is empty"

.gitignore

@@ -106,5 +106,6 @@ test-results.xml
 .env.development.local
 .env.test.local
 .env.production.local
+vector/*
 mmdb/**/**
-GeoIP.conf
+GeoIP.conf

deploy/crontab

@@ -4,3 +4,4 @@
 * * * * *		curl -d "{}" http://${NSQD_HOST}:${NSQD_HTTP_PORT}/pub?topic=every_minute
 0 12 1-8 * *		test -z "$RETRACED_DISABLE_GEOSYNC" && test $(date +\%u) -eq 3 && curl -d "{}" http://${NSQD_HOST}:${NSQD_HTTP_PORT}/pub?topic=first_wed_of_month
 * * * * * * *		curl -d "{}" http://${NSQD_HOST}:${NSQD_HTTP_PORT}/pub?topic=every_second
+*/5 * * * *		curl -d "{}" http://${NSQD_HOST}:${NSQD_HTTP_PORT}/pub?topic=pull_events_for_export

deploy/vector-sidecar/Dockerfile

@@ -0,0 +1,23 @@
+FROM timberio/vector:0.X-alpine as builder
+
+FROM node:18.18.0-alpine3.18
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
+COPY --from=builder /etc/vector/vector.yaml /etc/vector/vector.yaml
+COPY --from=builder /var/lib/vector /var/lib/vector
+COPY vector.json /etc/vector/config/vector.json
+
+WORKDIR /src
+COPY pm2.config.js /src/pm2.config.js
+ADD package.json /src
+ADD package-lock.json /src
+RUN npm install
+# Copy the Node.js application code
+ADD . /src
+RUN npm run build
+RUN npm i -g pm2
+ENV PORT 3002
+# Expose the port used by the Node.js application
+EXPOSE 3002
+EXPOSE 8686
+# CMD vector -w --config-dir /etc/vector/config & node /src/build/src/ee/_vector-sidecar/index.js
+CMD ["pm2-runtime", "start", "/src/pm2.config.js"]

docker-compose.yaml

@@ -29,6 +29,7 @@ x-common-variables: &common-variables
   GEOIPUPDATE_USE_MMDB: ${GEOIPUPDATE_USE_MMDB}
   GEOIPUPDATE_ACCOUNT_ID: ${GEOIPUPDATE_ACCOUNT_ID}
   PG_SEARCH: ${PG_SEARCH}
+  VECTOR_SIDECAR_PORT: 3002
 
 services:
   retraced-api:
@@ -199,7 +200,7 @@ services:
     restart: "on-failure"
 
   admin-portal:
-    image: boxyhq/jackson:1.14.2
+    image: boxyhq/jackson:1.16.1
     ports:
       - "5225:5225"
     networks:
@@ -218,8 +219,33 @@ services:
       - RETRACED_HOST_URL=http://retraced-api:3000/auditlog
       - RETRACED_EXTERNAL_URL=http://localhost:3000/auditlog
       - RETRACED_ADMIN_ROOT_TOKEN=dev
+      - BOXYHQ_HOSTED=${BOXYHQ_HOSTED}
+      - BOXYHQ_LICENSE_KEY=${BOXYHQ_LICENSE_KEY}
     depends_on:
       - "retraced-api"
     restart: "always"
+
+  vector:
+    build:
+      context: .
+      dockerfile: ./deploy/vector-sidecar/Dockerfile
+    environment: *common-variables
+    ports:
+      - "8686:8686"
+      - "9393:9229"
+      - "3002:3002"
+      - "9999:9999"
+    volumes:
+      - ./vector/data:/var/lib/vector/
+    depends_on:
+      elasticsearch:
+        condition: service_started
+      postgres:
+        condition: service_started
+      nsqd:
+        condition: service_started
+    networks:
+      - retraced
 volumes:
   mmdb:
+  postgres-data:

kustomize/base/kustomization.yaml

@@ -8,5 +8,6 @@ resources:
   - ./nsqd-service.yaml
   - ./processor-deployment.yaml
   - ./geoipupdate.yaml
+  - ./vector-deployment.yaml
 
 namespace: default

kustomize/base/vector-deployment.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: retraced-vector-sidecar
+  namespace: '{{repl ConfigOption "namespace"}}'
+spec:
+  selector:
+    matchLabels:
+      tier: vector
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: auditlog
+        tier: vector
+    spec:
+      containers:
+        - name: vector
+          image: retracedhq/vector-local
+          imagePullPolicy: IfNotPresent
+          startupProbe:
+            httpGet:
+              port: 3002
+              path: /api/v1/health
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 5
+          readinessProbe:
+            httpGet:
+              port: 3002
+              path: /api/v1/vector/health
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 5
+            successThreshold: 2
+          ports:
+            - containerPort: 3002
+            - containerPort: 8686
+          envFrom:
+            - secretRef:
+                name: auditlog
+          resources:
+            requests:
+              cpu: 100m
+            limits:
+              cpu: 1000m