Merge branch 'main' into kafka-integration

parseablehq · Dec 15, 2024 · 05b9965 · 05b9965
2 parents 4432ee2 + 80a3ff4
commit 05b9965
Show file tree

Hide file tree

Showing 15 changed files with 291 additions and 203 deletions.
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
@@ -0,0 +1,57 @@
+on:
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "helm/**"
+      - "assets/**"
+      - "**.md"
+  push:
+    branches:
+      - main
+
+name: Lint, Test and Coverage Report
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: ${{ runner.os }}-cargo
+
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-hack, cargo-llvm-cov, nextest
+
+      - name: Check with clippy
+        run: cargo hack clippy --verbose --each-feature --no-dev-deps -- -D warnings
+
+      - name: Check docs
+        run: cargo hack doc --verbose --no-deps --each-feature --no-dev-deps
+
+      - name: Doctests
+        run: cargo hack --each-feature test --doc
+
+      - name: Tests
+        run: cargo hack --each-feature llvm-cov --no-report nextest
+
+      - name: Genrate coverage report
+        run: cargo llvm-cov report --lcov --output-path coverage.lcov
+
+      - name: Upload Coverage Report
+        uses: coverallsapp/[email protected]
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          file: ./coverage.lcov
+          parallel: true
+          flag-name: run-${{ matrix.os }}-cargo
+
+      - name: Finish Coverage Report
+        uses: coverallsapp/[email protected]
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          parallel-finished: true
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
diff --git a/helm-releases/parseable-1.6.4.tgz b/helm-releases/parseable-1.6.4.tgz
diff --git a/helm/Chart.yaml b/helm/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: parseable
 description: Helm chart for Parseable Server
 type: application
-version: 1.6.3
+version: 1.6.4
 appVersion: "v1.6.3"
 maintainers:
   - name: Parseable Team
@@ -18,3 +18,4 @@ dependencies:
   version: 0.48.0
   repository: https://fluent.github.io/helm-charts
   condition: fluent-bit.enabled
+
diff --git a/helm/templates/ingestor-statefulset.yaml b/helm/templates/ingestor-statefulset.yaml
@@ -54,7 +54,9 @@ spec:
                 apiVersion: v1
                 fieldPath: metadata.name
           {{- end }}
-          {{- range $secret := .Values.parseable.s3ModeSecret }}
+
+          {{- if and .Values.parseable.s3ModeSecret .Values.parseable.s3ModeSecret.enabled }}
+          {{- range $secret := .Values.parseable.s3ModeSecret.secrets }}
           {{- range $key := $secret.keys }}
           {{- $envPrefix := $secret.prefix | default "" | upper }}
           {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
@@ -65,7 +67,10 @@ spec:
                 key: {{ $key }}
           {{- end }}
           {{- end }}
-          {{- range $secret := .Values.parseable.blobModeSecret }}
+          {{- end }}
+
+          {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }}
+          {{- range $secret := .Values.parseable.blobModeSecret.secrets }}
           {{- range $key := $secret.keys }}
           {{- $envPrefix := $secret.prefix | default "" | upper }}
           {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
@@ -76,6 +81,7 @@ spec:
                 key: {{ $key }}
           {{- end }}
           {{- end }}
+          {{- end }}
           - name: P_MODE
             value: "ingest"
         ports:

diff --git a/helm/templates/querier-statefulset.yaml b/helm/templates/querier-statefulset.yaml
@@ -52,11 +52,15 @@ spec:
               fieldRef:
                 apiVersion: v1
                 fieldPath: metadata.name
+          - name: P_MODE
+            value: "query"
           {{- range $key, $value :=  .Values.parseable.env }}
           - name: {{ $key }}
             value: {{ tpl $value $ | quote }}
           {{- end }}
-          {{- range $secret := .Values.parseable.s3ModeSecret }}
+
+          {{- if and .Values.parseable.s3ModeSecret .Values.parseable.s3ModeSecret.enabled }}
+          {{- range $secret := .Values.parseable.s3ModeSecret.secrets }}
           {{- range $key := $secret.keys }}
           {{- $envPrefix := $secret.prefix | default "" | upper }}
           {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
@@ -67,13 +71,10 @@ spec:
                 key: {{ $key }}
           {{- end }}
           {{- end }}
-          - name: P_MODE
-            value: "query"
-          {{- if .Values.parseable.persistence.querier.enabled }}
-          - name: P_HOT_TIER_DIR
-            value: "/parseable/hot-tier"
           {{- end }}
-          {{- range $secret := .Values.parseable.blobModeSecret }}
+
+          {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }}
+          {{- range $secret := .Values.parseable.blobModeSecret.secrets }}
           {{- range $key := $secret.keys }}
           {{- $envPrefix := $secret.prefix | default "" | upper }}
           {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
@@ -84,6 +85,7 @@ spec:
                 key: {{ $key }}
           {{- end }}
           {{- end }}
+          {{- end }}
         ports:
           - containerPort: 8000
         {{- with .Values.readinessProbe }}

diff --git a/helm/templates/standalone-deployment.yaml b/helm/templates/standalone-deployment.yaml
@@ -47,8 +47,8 @@ spec:
             - name: {{ $key }}
               value: {{ tpl $value $ | quote }}
             {{- end }}
-            {{- if .Values.parseable.localModeSecret }}
-            {{- range $secret := .Values.parseable.localModeSecret }}
+            {{- if and .Values.parseable.localModeSecret .Values.parseable.localModeSecret.enabled }}
+            {{- range $secret := .Values.parseable.localModeSecret.secrets }}
             {{- range $key := $secret.keys }}
             {{- $envPrefix := $secret.prefix | default "" | upper }}
             {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
@@ -59,8 +59,10 @@ spec:
                   key: {{ $key }}
             {{- end }}
             {{- end }}
-            {{- else }}
-            {{- range $secret := .Values.parseable.s3ModeSecret }}
+            {{- end }}
+
+            {{- if and .Values.parseable.s3ModeSecret .Values.parseable.s3ModeSecret.enabled }}
+            {{- range $secret := .Values.parseable.s3ModeSecret.secrets }}
             {{- range $key := $secret.keys }}
             {{- $envPrefix := $secret.prefix | default "" | upper }}
             {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
@@ -72,7 +74,9 @@ spec:
             {{- end }}
             {{- end }}
             {{- end }}
-            {{- range $secret := .Values.parseable.blobModeSecret }}
+
+            {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }}
+            {{- range $secret := .Values.parseable.blobModeSecret.secrets }}
             {{- range $key := $secret.keys }}
             {{- $envPrefix := $secret.prefix | default "" | upper }}
             {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
@@ -83,6 +87,7 @@ spec:
                   key: {{ $key }}
             {{- end }}
             {{- end }}
+            {{- end }}
           ports:
             - containerPort: 8000
           {{- with .Values.readinessProbe }}

diff --git a/helm/values.yaml b/helm/values.yaml
@@ -77,46 +77,45 @@ parseable:
   ## blobModeSecret if store is set to blob-store
   ## s3ModeSecret if store is set to s3-store
   localModeSecret:
-    - type: env
-      name: parseable-env-secret
-      prefix: P_
-      keys:
-      - addr
-      - username
-      - password
-      - staging.dir
-      - fs.dir
-  # blobModeSecret:
-  #   - type: env
-  #     name: parseable-env-secret
-  #     prefix: p_
-  #     keys:
-  #     - addr
-  #     - username
-  #     - password
-  #     - azr.access_key
-  #     - azr.account
-  #     - azr.container
-  #     - azr.url
-  # s3ModeSecret:
-  #   - type: env
-  #     name: parseable-env-secret
-  #     prefix: P_
-  #     keys:
-  #     ## Comment / uncomment the following lines as required
-  #     # - tls.cert.path
-  #     # - tls.key.path
-  #     # - storage.upload.interval
-  #     - addr
-  #     - username
-  #     - password
-  #     - staging.dir
-  #     - fs.dir
-  #     - s3.url
-  #     - s3.access.key
-  #     - s3.secret.key
-  #     - s3.bucket
-  #     - s3.region
+    enabled: false
+    secrets:
+      - name: parseable-env-secret
+        prefix: P_
+        keys:
+          - addr
+          - username
+          - password
+          - staging.dir
+          - fs.dir
+  blobModeSecret:
+    enabled: false
+    secrets:
+      - name: parseable-env-secret
+        prefix: P_
+        keys:
+          - addr
+          - username
+          - password
+          - azr.access_key
+          - azr.account
+          - azr.container
+          - azr.url
+  s3ModeSecret:
+    enabled: false
+    secrets:
+      - name: parseable-env-secret
+        prefix: P_
+        keys:
+          - addr
+          - username
+          - password
+          - staging.dir
+          - fs.dir
+          - s3.url
+          - s3.access.key
+          - s3.secret.key
+          - s3.bucket
+          - s3.region
   serviceAccount:
     create: true
     name: "parseable"
@@ -298,6 +297,7 @@ vector:
 fluent-bit:
   enabled: true
   kind: DaemonSet
+  serverHost: parseable.parseable.svc.cluster.local
   replicaCount: 1
   image:
     repository: parseable/fluent-bit
@@ -383,7 +383,7 @@ fluent-bit:
       [OUTPUT]
           Name parseable
           Match kube.*
-          Server_Host parseable.parseable.svc.cluster.local
+          Server_Host {{ .Values.serverHost }}
           Username admin
           Password admin
           Server_Port 80
@@ -393,7 +393,7 @@ fluent-bit:
       [OUTPUT]
           Name parseable
           Match k8s_events
-          Server_Host parseable.parseable.svc.cluster.local
+          Server_Host {{ .Values.serverHost }}
           Server_Port 80
           Username admin
           Password admin