diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml new file mode 100644 index 000000000..eb5a46da4 --- /dev/null +++ b/.github/workflows/coverage.yaml @@ -0,0 +1,57 @@ +on: + pull_request: + paths-ignore: + - "docs/**" + - "helm/**" + - "assets/**" + - "**.md" + push: + branches: + - main + +name: Lint, Test and Coverage Report +jobs: + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - uses: Swatinem/rust-cache@v2 + with: + shared-key: ${{ runner.os }}-cargo + + - uses: taiki-e/install-action@v2 + with: + tool: cargo-hack, cargo-llvm-cov, nextest + + - name: Check with clippy + run: cargo hack clippy --verbose --each-feature --no-dev-deps -- -D warnings + + - name: Check docs + run: cargo hack doc --verbose --no-deps --each-feature --no-dev-deps + + - name: Doctests + run: cargo hack --each-feature test --doc + + - name: Tests + run: cargo hack --each-feature llvm-cov --no-report nextest + + - name: Genrate coverage report + run: cargo llvm-cov report --lcov --output-path coverage.lcov + + - name: Upload Coverage Report + uses: coverallsapp/github-action@v2.2.3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + file: ./coverage.lcov + parallel: true + flag-name: run-${{ matrix.os }}-cargo + + - name: Finish Coverage Report + uses: coverallsapp/github-action@v2.2.3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + parallel-finished: true diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml deleted file mode 100644 index 4f2578cf5..000000000 --- a/.github/workflows/lint.yaml +++ /dev/null @@ -1,61 +0,0 @@ -name: Linter - -on: - pull_request: - paths-ignore: - - 'docs/**' - - 'helm/**' - - 'assets/**' - - '**.md' - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - - unit-tests: - name: Unit tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - uses: actions-rs/cargo@v1 - with: - command: test - - fmt: - name: Rust fmt check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - run: rustup component add rustfmt - - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check - - clippy: - name: Cargo Clippy check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - run: rustup component add clippy - - uses: actions-rs/cargo@v1 - with: - command: clippy - args: -- -D warnings diff --git a/helm-releases/parseable-1.6.4.tgz b/helm-releases/parseable-1.6.4.tgz new file mode 100644 index 000000000..18cc77e6e Binary files /dev/null and b/helm-releases/parseable-1.6.4.tgz differ diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 9860f3d8b..9a10ec40b 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: parseable description: Helm chart for Parseable Server type: application -version: 1.6.3 +version: 1.6.4 appVersion: "v1.6.3" maintainers: - name: Parseable Team @@ -18,3 +18,4 @@ dependencies: version: 0.48.0 repository: https://fluent.github.io/helm-charts condition: fluent-bit.enabled + \ No newline at end of file diff --git a/helm/templates/ingestor-statefulset.yaml b/helm/templates/ingestor-statefulset.yaml index 91ca793c0..50324cb7e 100644 --- a/helm/templates/ingestor-statefulset.yaml +++ b/helm/templates/ingestor-statefulset.yaml @@ -54,7 +54,9 @@ spec: apiVersion: v1 fieldPath: metadata.name {{- end }} - {{- range $secret := .Values.parseable.s3ModeSecret }} + + {{- if and .Values.parseable.s3ModeSecret .Values.parseable.s3ModeSecret.enabled }} + {{- range $secret := .Values.parseable.s3ModeSecret.secrets }} {{- range $key := $secret.keys }} {{- $envPrefix := $secret.prefix | default "" | upper }} {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }} @@ -65,7 +67,10 @@ spec: key: {{ $key }} {{- end }} {{- end }} - {{- range $secret := .Values.parseable.blobModeSecret }} + {{- end }} + + {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }} + {{- range $secret := .Values.parseable.blobModeSecret.secrets }} {{- range $key := $secret.keys }} {{- $envPrefix := $secret.prefix | default "" | upper }} {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }} @@ -76,6 +81,7 @@ spec: key: {{ $key }} {{- end }} {{- end }} + {{- end }} - name: P_MODE value: "ingest" ports: diff --git a/helm/templates/querier-statefulset.yaml b/helm/templates/querier-statefulset.yaml index d48ce1035..98845faaa 100644 --- a/helm/templates/querier-statefulset.yaml +++ b/helm/templates/querier-statefulset.yaml @@ -52,11 +52,15 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.name + - name: P_MODE + value: "query" {{- range $key, $value := .Values.parseable.env }} - name: {{ $key }} value: {{ tpl $value $ | quote }} {{- end }} - {{- range $secret := .Values.parseable.s3ModeSecret }} + + {{- if and .Values.parseable.s3ModeSecret .Values.parseable.s3ModeSecret.enabled }} + {{- range $secret := .Values.parseable.s3ModeSecret.secrets }} {{- range $key := $secret.keys }} {{- $envPrefix := $secret.prefix | default "" | upper }} {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }} @@ -67,13 +71,10 @@ spec: key: {{ $key }} {{- end }} {{- end }} - - name: P_MODE - value: "query" - {{- if .Values.parseable.persistence.querier.enabled }} - - name: P_HOT_TIER_DIR - value: "/parseable/hot-tier" {{- end }} - {{- range $secret := .Values.parseable.blobModeSecret }} + + {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }} + {{- range $secret := .Values.parseable.blobModeSecret.secrets }} {{- range $key := $secret.keys }} {{- $envPrefix := $secret.prefix | default "" | upper }} {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }} @@ -84,6 +85,7 @@ spec: key: {{ $key }} {{- end }} {{- end }} + {{- end }} ports: - containerPort: 8000 {{- with .Values.readinessProbe }} diff --git a/helm/templates/standalone-deployment.yaml b/helm/templates/standalone-deployment.yaml index dc0c8dbfc..abb615884 100644 --- a/helm/templates/standalone-deployment.yaml +++ b/helm/templates/standalone-deployment.yaml @@ -47,8 +47,8 @@ spec: - name: {{ $key }} value: {{ tpl $value $ | quote }} {{- end }} - {{- if .Values.parseable.localModeSecret }} - {{- range $secret := .Values.parseable.localModeSecret }} + {{- if and .Values.parseable.localModeSecret .Values.parseable.localModeSecret.enabled }} + {{- range $secret := .Values.parseable.localModeSecret.secrets }} {{- range $key := $secret.keys }} {{- $envPrefix := $secret.prefix | default "" | upper }} {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }} @@ -59,8 +59,10 @@ spec: key: {{ $key }} {{- end }} {{- end }} - {{- else }} - {{- range $secret := .Values.parseable.s3ModeSecret }} + {{- end }} + + {{- if and .Values.parseable.s3ModeSecret .Values.parseable.s3ModeSecret.enabled }} + {{- range $secret := .Values.parseable.s3ModeSecret.secrets }} {{- range $key := $secret.keys }} {{- $envPrefix := $secret.prefix | default "" | upper }} {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }} @@ -72,7 +74,9 @@ spec: {{- end }} {{- end }} {{- end }} - {{- range $secret := .Values.parseable.blobModeSecret }} + + {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }} + {{- range $secret := .Values.parseable.blobModeSecret.secrets }} {{- range $key := $secret.keys }} {{- $envPrefix := $secret.prefix | default "" | upper }} {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }} @@ -83,6 +87,7 @@ spec: key: {{ $key }} {{- end }} {{- end }} + {{- end }} ports: - containerPort: 8000 {{- with .Values.readinessProbe }} diff --git a/helm/values.yaml b/helm/values.yaml index 949ae2365..49f6759ad 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -77,46 +77,45 @@ parseable: ## blobModeSecret if store is set to blob-store ## s3ModeSecret if store is set to s3-store localModeSecret: - - type: env - name: parseable-env-secret - prefix: P_ - keys: - - addr - - username - - password - - staging.dir - - fs.dir - # blobModeSecret: - # - type: env - # name: parseable-env-secret - # prefix: p_ - # keys: - # - addr - # - username - # - password - # - azr.access_key - # - azr.account - # - azr.container - # - azr.url - # s3ModeSecret: - # - type: env - # name: parseable-env-secret - # prefix: P_ - # keys: - # ## Comment / uncomment the following lines as required - # # - tls.cert.path - # # - tls.key.path - # # - storage.upload.interval - # - addr - # - username - # - password - # - staging.dir - # - fs.dir - # - s3.url - # - s3.access.key - # - s3.secret.key - # - s3.bucket - # - s3.region + enabled: false + secrets: + - name: parseable-env-secret + prefix: P_ + keys: + - addr + - username + - password + - staging.dir + - fs.dir + blobModeSecret: + enabled: false + secrets: + - name: parseable-env-secret + prefix: P_ + keys: + - addr + - username + - password + - azr.access_key + - azr.account + - azr.container + - azr.url + s3ModeSecret: + enabled: false + secrets: + - name: parseable-env-secret + prefix: P_ + keys: + - addr + - username + - password + - staging.dir + - fs.dir + - s3.url + - s3.access.key + - s3.secret.key + - s3.bucket + - s3.region serviceAccount: create: true name: "parseable" @@ -298,6 +297,7 @@ vector: fluent-bit: enabled: true kind: DaemonSet + serverHost: parseable.parseable.svc.cluster.local replicaCount: 1 image: repository: parseable/fluent-bit @@ -383,7 +383,7 @@ fluent-bit: [OUTPUT] Name parseable Match kube.* - Server_Host parseable.parseable.svc.cluster.local + Server_Host {{ .Values.serverHost }} Username admin Password admin Server_Port 80 @@ -393,7 +393,7 @@ fluent-bit: [OUTPUT] Name parseable Match k8s_events - Server_Host parseable.parseable.svc.cluster.local + Server_Host {{ .Values.serverHost }} Server_Port 80 Username admin Password admin diff --git a/index.yaml b/index.yaml index cfe471e3e..91b3cc796 100644 --- a/index.yaml +++ b/index.yaml @@ -34,7 +34,30 @@ entries: parseable: - apiVersion: v2 appVersion: v1.6.3 - created: "2024-11-29T00:08:45.06534311+05:30" + created: "2024-12-10T17:20:03.444421031-05:00" + dependencies: + - condition: vector.enabled + name: vector + repository: https://helm.vector.dev + version: 0.20.1 + - condition: fluent-bit.enabled + name: fluent-bit + repository: https://fluent.github.io/helm-charts + version: 0.48.0 + description: Helm chart for Parseable Server + digest: 3baedde745eb646edb5caab571c30af496f9379fb07785914e6343480c168b06 + maintainers: + - email: hi@parseable.com + name: Parseable Team + url: https://parseable.com + name: parseable + type: application + urls: + - https://charts.parseable.com/helm-releases/parseable-1.6.4.tgz + version: 1.6.4 + - apiVersion: v2 + appVersion: v1.6.3 + created: "2024-12-10T17:20:03.43821683-05:00" dependencies: - condition: vector.enabled name: vector @@ -57,7 +80,7 @@ entries: version: 1.6.3 - apiVersion: v2 appVersion: v1.6.2 - created: "2024-11-29T00:08:45.059250262+05:30" + created: "2024-12-10T17:20:03.431215144-05:00" dependencies: - condition: vector.enabled name: vector @@ -80,7 +103,7 @@ entries: version: 1.6.2 - apiVersion: v2 appVersion: v1.6.1 - created: "2024-11-29T00:08:45.05077333+05:30" + created: "2024-12-10T17:20:03.423805201-05:00" dependencies: - condition: vector.enabled name: vector @@ -103,7 +126,7 @@ entries: version: 1.6.1 - apiVersion: v2 appVersion: v1.6.0 - created: "2024-11-29T00:08:45.043008895+05:30" + created: "2024-12-10T17:20:03.413613355-05:00" dependencies: - condition: vector.enabled name: vector @@ -126,7 +149,7 @@ entries: version: 1.6.0 - apiVersion: v2 appVersion: v1.5.5 - created: "2024-11-29T00:08:45.034231702+05:30" + created: "2024-12-10T17:20:03.405348908-05:00" dependencies: - condition: vector.enabled name: vector @@ -149,7 +172,7 @@ entries: version: 1.5.5 - apiVersion: v2 appVersion: v1.5.4 - created: "2024-11-29T00:08:45.026777254+05:30" + created: "2024-12-10T17:20:03.398094402-05:00" dependencies: - condition: vector.enabled name: vector @@ -172,7 +195,7 @@ entries: version: 1.5.4 - apiVersion: v2 appVersion: v1.5.3 - created: "2024-11-29T00:08:45.018990474+05:30" + created: "2024-12-10T17:20:03.388977285-05:00" dependencies: - condition: vector.enabled name: vector @@ -195,7 +218,7 @@ entries: version: 1.5.3 - apiVersion: v2 appVersion: v1.5.2 - created: "2024-11-29T00:08:45.011693608+05:30" + created: "2024-12-10T17:20:03.381112937-05:00" dependencies: - condition: vector.enabled name: vector @@ -218,7 +241,7 @@ entries: version: 1.5.2 - apiVersion: v2 appVersion: v1.5.1 - created: "2024-11-29T00:08:45.004156481+05:30" + created: "2024-12-10T17:20:03.37177663-05:00" dependencies: - condition: vector.enabled name: vector @@ -241,7 +264,7 @@ entries: version: 1.5.1 - apiVersion: v2 appVersion: v1.5.0 - created: "2024-11-29T00:08:44.996027644+05:30" + created: "2024-12-10T17:20:03.36164978-05:00" dependencies: - condition: vector.enabled name: vector @@ -264,7 +287,7 @@ entries: version: 1.5.0 - apiVersion: v2 appVersion: v1.4.0 - created: "2024-11-29T00:08:44.989389457+05:30" + created: "2024-12-10T17:20:03.35310657-05:00" dependencies: - condition: vector.enabled name: vector @@ -287,7 +310,7 @@ entries: version: 1.4.1 - apiVersion: v2 appVersion: v1.4.0 - created: "2024-11-29T00:08:44.981911848+05:30" + created: "2024-12-10T17:20:03.343252346-05:00" dependencies: - condition: vector.enabled name: vector @@ -310,7 +333,7 @@ entries: version: 1.4.0 - apiVersion: v2 appVersion: v1.3.0 - created: "2024-11-29T00:08:44.973430432+05:30" + created: "2024-12-10T17:20:03.333842856-05:00" dependencies: - condition: vector.enabled name: vector @@ -333,7 +356,7 @@ entries: version: 1.3.1 - apiVersion: v2 appVersion: v1.3.0 - created: "2024-11-29T00:08:44.965107642+05:30" + created: "2024-12-10T17:20:03.327182676-05:00" dependencies: - condition: vector.enabled name: vector @@ -356,7 +379,7 @@ entries: version: 1.3.0 - apiVersion: v2 appVersion: v1.2.0 - created: "2024-11-29T00:08:44.957765008+05:30" + created: "2024-12-10T17:20:03.319091016-05:00" dependencies: - condition: vector.enabled name: vector @@ -379,7 +402,7 @@ entries: version: 1.2.0 - apiVersion: v2 appVersion: v1.1.0 - created: "2024-11-29T00:08:44.950742995+05:30" + created: "2024-12-10T17:20:03.31333821-05:00" dependencies: - condition: vector.enabled name: vector @@ -402,7 +425,7 @@ entries: version: 1.1.0 - apiVersion: v2 appVersion: v1.0.0 - created: "2024-11-29T00:08:44.945545633+05:30" + created: "2024-12-10T17:20:03.307173302-05:00" dependencies: - condition: vector.enabled name: vector @@ -425,7 +448,7 @@ entries: version: 1.0.0 - apiVersion: v2 appVersion: v0.9.0 - created: "2024-11-29T00:08:44.939512651+05:30" + created: "2024-12-10T17:20:03.300297326-05:00" dependencies: - condition: vector.enabled name: vector @@ -448,7 +471,7 @@ entries: version: 0.9.0 - apiVersion: v2 appVersion: v0.8.1 - created: "2024-11-29T00:08:44.933448679+05:30" + created: "2024-12-10T17:20:03.293880211-05:00" dependencies: - condition: vector.enabled name: vector @@ -471,7 +494,7 @@ entries: version: 0.8.1 - apiVersion: v2 appVersion: v0.8.0 - created: "2024-11-29T00:08:44.926692308+05:30" + created: "2024-12-10T17:20:03.286818525-05:00" dependencies: - condition: vector.enabled name: vector @@ -494,7 +517,7 @@ entries: version: 0.8.0 - apiVersion: v2 appVersion: v0.7.3 - created: "2024-11-29T00:08:44.918803911+05:30" + created: "2024-12-10T17:20:03.279285163-05:00" dependencies: - condition: vector.enabled name: vector @@ -517,7 +540,7 @@ entries: version: 0.7.3 - apiVersion: v2 appVersion: v0.7.2 - created: "2024-11-29T00:08:44.911976979+05:30" + created: "2024-12-10T17:20:03.273058112-05:00" dependencies: - condition: vector.enabled name: vector @@ -540,7 +563,7 @@ entries: version: 0.7.2 - apiVersion: v2 appVersion: v0.7.1 - created: "2024-11-29T00:08:44.904921107+05:30" + created: "2024-12-10T17:20:03.266578799-05:00" dependencies: - condition: vector.enabled name: vector @@ -563,7 +586,7 @@ entries: version: 0.7.1 - apiVersion: v2 appVersion: v0.7.0 - created: "2024-11-29T00:08:44.898821973+05:30" + created: "2024-12-10T17:20:03.260042386-05:00" dependencies: - condition: vector.enabled name: vector @@ -586,7 +609,7 @@ entries: version: 0.7.0 - apiVersion: v2 appVersion: v0.6.2 - created: "2024-11-29T00:08:44.890991713+05:30" + created: "2024-12-10T17:20:03.255083174-05:00" dependencies: - condition: vector.enabled name: vector @@ -609,7 +632,7 @@ entries: version: 0.6.2 - apiVersion: v2 appVersion: v0.6.1 - created: "2024-11-29T00:08:44.883903131+05:30" + created: "2024-12-10T17:20:03.249083724-05:00" dependencies: - condition: vector.enabled name: vector @@ -632,7 +655,7 @@ entries: version: 0.6.1 - apiVersion: v2 appVersion: v0.6.0 - created: "2024-11-29T00:08:44.875874141+05:30" + created: "2024-12-10T17:20:03.240792003-05:00" dependencies: - condition: vector.enabled name: vector @@ -655,7 +678,7 @@ entries: version: 0.6.0 - apiVersion: v2 appVersion: v0.5.1 - created: "2024-11-29T00:08:44.868676404+05:30" + created: "2024-12-10T17:20:03.23396611-05:00" dependencies: - condition: vector.enabled name: vector @@ -678,7 +701,7 @@ entries: version: 0.5.1 - apiVersion: v2 appVersion: v0.5.0 - created: "2024-11-29T00:08:44.861730197+05:30" + created: "2024-12-10T17:20:03.226693392-05:00" dependencies: - condition: vector.enabled name: vector @@ -701,7 +724,7 @@ entries: version: 0.5.0 - apiVersion: v2 appVersion: v0.4.4 - created: "2024-11-29T00:08:44.854653371+05:30" + created: "2024-12-10T17:20:03.218278363-05:00" dependencies: - condition: vector.enabled name: vector @@ -724,7 +747,7 @@ entries: version: 0.4.5 - apiVersion: v2 appVersion: v0.4.3 - created: "2024-11-29T00:08:44.847726652+05:30" + created: "2024-12-10T17:20:03.2118222-05:00" dependencies: - condition: vector.enabled name: vector @@ -747,7 +770,7 @@ entries: version: 0.4.4 - apiVersion: v2 appVersion: v0.4.2 - created: "2024-11-29T00:08:44.84085073+05:30" + created: "2024-12-10T17:20:03.205428913-05:00" dependencies: - condition: vector.enabled name: vector @@ -770,7 +793,7 @@ entries: version: 0.4.3 - apiVersion: v2 appVersion: v0.4.1 - created: "2024-11-29T00:08:44.832709942+05:30" + created: "2024-12-10T17:20:03.198395504-05:00" dependencies: - condition: vector.enabled name: vector @@ -793,7 +816,7 @@ entries: version: 0.4.2 - apiVersion: v2 appVersion: v0.4.0 - created: "2024-11-29T00:08:44.825857924+05:30" + created: "2024-12-10T17:20:03.193193538-05:00" dependencies: - condition: vector.enabled name: vector @@ -816,7 +839,7 @@ entries: version: 0.4.1 - apiVersion: v2 appVersion: v0.4.0 - created: "2024-11-29T00:08:44.818783068+05:30" + created: "2024-12-10T17:20:03.186780127-05:00" dependencies: - condition: vector.enabled name: vector @@ -839,7 +862,7 @@ entries: version: 0.4.0 - apiVersion: v2 appVersion: v0.3.1 - created: "2024-11-29T00:08:44.811794892+05:30" + created: "2024-12-10T17:20:03.1792561-05:00" dependencies: - condition: vector.enabled name: vector @@ -862,7 +885,7 @@ entries: version: 0.3.1 - apiVersion: v2 appVersion: v0.3.0 - created: "2024-11-29T00:08:44.806984292+05:30" + created: "2024-12-10T17:20:03.172715307-05:00" description: Helm chart for Parseable Server digest: ff30739229b727dc637f62fd4481c886a6080ce4556bae10cafe7642ddcfd937 name: parseable @@ -872,7 +895,7 @@ entries: version: 0.3.0 - apiVersion: v2 appVersion: v0.2.2 - created: "2024-11-29T00:08:44.806549892+05:30" + created: "2024-12-10T17:20:03.171609995-05:00" description: Helm chart for Parseable Server digest: 477d0dc2f0c07d4f4c32e105d4bdd70c71113add5c2a75ac5f1cb42aa0276db7 name: parseable @@ -882,7 +905,7 @@ entries: version: 0.2.2 - apiVersion: v2 appVersion: v0.2.1 - created: "2024-11-29T00:08:44.80613879+05:30" + created: "2024-12-10T17:20:03.170694187-05:00" description: Helm chart for Parseable Server digest: 84826fcd1b4c579f301569f43b0309c07e8082bad76f5cdd25f86e86ca2e8192 name: parseable @@ -892,7 +915,7 @@ entries: version: 0.2.1 - apiVersion: v2 appVersion: v0.2.0 - created: "2024-11-29T00:08:44.805774371+05:30" + created: "2024-12-10T17:20:03.169740636-05:00" description: Helm chart for Parseable Server digest: 7a759f7f9809f3935cba685e904c021a0b645f217f4e45b9be185900c467edff name: parseable @@ -902,7 +925,7 @@ entries: version: 0.2.0 - apiVersion: v2 appVersion: v0.1.1 - created: "2024-11-29T00:08:44.805373257+05:30" + created: "2024-12-10T17:20:03.168898001-05:00" description: Helm chart for Parseable Server digest: 37993cf392f662ec7b1fbfc9a2ba00ec906d98723e38f3c91ff1daca97c3d0b3 name: parseable @@ -912,7 +935,7 @@ entries: version: 0.1.1 - apiVersion: v2 appVersion: v0.1.0 - created: "2024-11-29T00:08:44.804981967+05:30" + created: "2024-12-10T17:20:03.168067728-05:00" description: Helm chart for Parseable Server digest: 1d580d072af8d6b1ebcbfee31c2e16c907d08db754780f913b5f0032b403789b name: parseable @@ -922,7 +945,7 @@ entries: version: 0.1.0 - apiVersion: v2 appVersion: v0.0.8 - created: "2024-11-29T00:08:44.80455305+05:30" + created: "2024-12-10T17:20:03.16722528-05:00" description: Helm chart for Parseable Server digest: c805254ffa634f96ecec448bcfff9973339aa9487dd8199b21b17b79a4de9345 name: parseable @@ -932,7 +955,7 @@ entries: version: 0.0.8 - apiVersion: v2 appVersion: v0.0.7 - created: "2024-11-29T00:08:44.804020344+05:30" + created: "2024-12-10T17:20:03.166440181-05:00" description: Helm chart for Parseable Server digest: c591f617ed1fe820bb2c72a4c976a78126f1d1095d552daa07c4700f46c4708a name: parseable @@ -942,7 +965,7 @@ entries: version: 0.0.7 - apiVersion: v2 appVersion: v0.0.6 - created: "2024-11-29T00:08:44.803336035+05:30" + created: "2024-12-10T17:20:03.165555386-05:00" description: Helm chart for Parseable Server digest: f9ae56a6fcd6a59e7bee0436200ddbedeb74ade6073deb435b8fcbaf08dda795 name: parseable @@ -952,7 +975,7 @@ entries: version: 0.0.6 - apiVersion: v2 appVersion: v0.0.5 - created: "2024-11-29T00:08:44.802638218+05:30" + created: "2024-12-10T17:20:03.164686292-05:00" description: Helm chart for Parseable Server digest: 4d6b08a064fba36e16feeb820b77e1e8e60fb6de48dbf7ec8410d03d10c26ad0 name: parseable @@ -962,7 +985,7 @@ entries: version: 0.0.5 - apiVersion: v2 appVersion: v0.0.2 - created: "2024-11-29T00:08:44.800852375+05:30" + created: "2024-12-10T17:20:03.163857934-05:00" description: Helm chart for Parseable Server digest: 38a0a3e4c498afbbcc76ebfcb9cb598fa2ca843a53cc93b3cb4f135b85c10844 name: parseable @@ -972,7 +995,7 @@ entries: version: 0.0.2 - apiVersion: v2 appVersion: v0.0.1 - created: "2024-11-29T00:08:44.800073576+05:30" + created: "2024-12-10T17:20:03.162050391-05:00" description: Helm chart for Parseable Server digest: 1f1142db092b9620ee38bb2294ccbb1c17f807b33bf56da43816af7fe89f301e name: parseable @@ -1001,4 +1024,4 @@ entries: urls: - https://charts.parseable.io/helm-releases/parseable-operator-0.0.1.tgz version: 0.0.1 -generated: "2024-11-29T00:08:44.799027282+05:30" +generated: "2024-12-10T17:20:03.161183816-05:00" diff --git a/src/catalog/column.rs b/src/catalog/column.rs index d5db2950d..ef4b5858b 100644 --- a/src/catalog/column.rs +++ b/src/catalog/column.rs @@ -66,6 +66,14 @@ impl TypedStatistics { max: max(this.max, other.max), }) } + + // Ints are casted to Float if self is Float and other in Int + (TypedStatistics::Float(this), TypedStatistics::Int(other)) => { + TypedStatistics::Float(Float64Type { + min: this.min.min(other.min as f64), + max: this.max.max(other.max as f64), + }) + } (TypedStatistics::Float(this), TypedStatistics::Float(other)) => { TypedStatistics::Float(Float64Type { min: this.min.min(other.min), diff --git a/src/event/format/json.rs b/src/event/format/json.rs index 487cb58a6..0a137fdb2 100644 --- a/src/event/format/json.rs +++ b/src/event/format/json.rs @@ -185,7 +185,9 @@ fn valid_type(data_type: &DataType, value: &Value) -> bool { DataType::Boolean => value.is_boolean(), DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => value.is_i64(), DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => value.is_u64(), - DataType::Float16 | DataType::Float32 | DataType::Float64 => value.is_f64(), + DataType::Float16 | DataType::Float32 => value.is_f64(), + // NOTE: All numbers can be ingested as Float64 + DataType::Float64 => value.is_number(), DataType::Utf8 => value.is_string(), DataType::List(field) => { let data_type = field.data_type(); diff --git a/src/event/format/mod.rs b/src/event/format/mod.rs index e637eb4e6..e0bb00daf 100644 --- a/src/event/format/mod.rs +++ b/src/event/format/mod.rs @@ -204,6 +204,24 @@ pub fn override_timestamp_fields( Arc::new(Schema::new(updated_fields)) } +/// All number fields from inferred schema are forced into Float64 +pub fn override_num_fields_from_schema(schema: Vec>) -> Vec> { + schema + .iter() + .map(|field| { + if field.data_type().is_numeric() { + Arc::new(Field::new( + field.name(), + DataType::Float64, + field.is_nullable(), + )) + } else { + field.clone() + } + }) + .collect::>>() +} + pub fn update_field_type_in_schema( inferred_schema: Arc, existing_schema: Option<&HashMap>>, @@ -212,6 +230,10 @@ pub fn update_field_type_in_schema( ) -> Arc { let mut updated_schema = inferred_schema.clone(); + // All number fields from inferred schema are forced into Float64 + updated_schema = Arc::new(Schema::new(override_num_fields_from_schema( + updated_schema.fields().to_vec(), + ))); if let Some(existing_schema) = existing_schema { let existing_fields = get_existing_fields(inferred_schema.clone(), Some(existing_schema)); let existing_timestamp_fields = get_existing_timestamp_fields(existing_schema); diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index 305bfecb0..303f591c3 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -277,7 +277,7 @@ mod tests { use std::{collections::HashMap, sync::Arc}; use actix_web::test::TestRequest; - use arrow_array::{ArrayRef, Float64Array, Int64Array, StringArray}; + use arrow_array::{ArrayRef, Float64Array, StringArray}; use arrow_schema::{DataType, Field}; use serde_json::json; @@ -287,16 +287,11 @@ mod tests { }; trait TestExt { - fn as_int64_arr(&self) -> &Int64Array; fn as_float64_arr(&self) -> &Float64Array; fn as_utf8_arr(&self) -> &StringArray; } impl TestExt for ArrayRef { - fn as_int64_arr(&self) -> &Int64Array { - self.as_any().downcast_ref().unwrap() - } - fn as_float64_arr(&self) -> &Float64Array { self.as_any().downcast_ref().unwrap() } @@ -328,8 +323,8 @@ mod tests { assert_eq!(rb.num_rows(), 1); assert_eq!(rb.num_columns(), 6); assert_eq!( - rb.column_by_name("a").unwrap().as_int64_arr(), - &Int64Array::from_iter([1]) + rb.column_by_name("a").unwrap().as_float64_arr(), + &Float64Array::from_iter([1.0]) ); assert_eq!( rb.column_by_name("b").unwrap().as_utf8_arr(), @@ -368,8 +363,8 @@ mod tests { assert_eq!(rb.num_rows(), 1); assert_eq!(rb.num_columns(), 5); assert_eq!( - rb.column_by_name("a").unwrap().as_int64_arr(), - &Int64Array::from_iter([1]) + rb.column_by_name("a").unwrap().as_float64_arr(), + &Float64Array::from_iter([1.0]) ); assert_eq!( rb.column_by_name("b").unwrap().as_utf8_arr(), @@ -386,7 +381,7 @@ mod tests { let schema = fields_to_map( [ - Field::new("a", DataType::Int64, true), + Field::new("a", DataType::Float64, true), Field::new("b", DataType::Utf8, true), Field::new("c", DataType::Float64, true), ] @@ -400,8 +395,8 @@ mod tests { assert_eq!(rb.num_rows(), 1); assert_eq!(rb.num_columns(), 5); assert_eq!( - rb.column_by_name("a").unwrap().as_int64_arr(), - &Int64Array::from_iter([1]) + rb.column_by_name("a").unwrap().as_float64_arr(), + &Float64Array::from_iter([1.0]) ); assert_eq!( rb.column_by_name("b").unwrap().as_utf8_arr(), @@ -418,7 +413,7 @@ mod tests { let schema = fields_to_map( [ - Field::new("a", DataType::Int64, true), + Field::new("a", DataType::Float64, true), Field::new("b", DataType::Utf8, true), Field::new("c", DataType::Float64, true), ] @@ -488,21 +483,21 @@ mod tests { let schema = rb.schema(); let fields = &schema.fields; - assert_eq!(&*fields[1], &Field::new("a", DataType::Int64, true)); + assert_eq!(&*fields[1], &Field::new("a", DataType::Float64, true)); assert_eq!(&*fields[2], &Field::new("b", DataType::Utf8, true)); - assert_eq!(&*fields[3], &Field::new("c", DataType::Int64, true)); + assert_eq!(&*fields[3], &Field::new("c", DataType::Float64, true)); assert_eq!( - rb.column_by_name("a").unwrap().as_int64_arr(), - &Int64Array::from(vec![None, Some(1), Some(1)]) + rb.column_by_name("a").unwrap().as_float64_arr(), + &Float64Array::from(vec![None, Some(1.0), Some(1.0)]) ); assert_eq!( rb.column_by_name("b").unwrap().as_utf8_arr(), &StringArray::from(vec![Some("hello"), Some("hello"), Some("hello"),]) ); assert_eq!( - rb.column_by_name("c").unwrap().as_int64_arr(), - &Int64Array::from(vec![None, Some(1), None]) + rb.column_by_name("c").unwrap().as_float64_arr(), + &Float64Array::from(vec![None, Some(1.0), None]) ); } @@ -533,8 +528,8 @@ mod tests { assert_eq!(rb.num_rows(), 3); assert_eq!(rb.num_columns(), 6); assert_eq!( - rb.column_by_name("a").unwrap().as_int64_arr(), - &Int64Array::from(vec![None, Some(1), Some(1)]) + rb.column_by_name("a").unwrap().as_float64_arr(), + &Float64Array::from(vec![None, Some(1.0), Some(1.0)]) ); assert_eq!( rb.column_by_name("b").unwrap().as_utf8_arr(), @@ -568,7 +563,7 @@ mod tests { let schema = fields_to_map( [ - Field::new("a", DataType::Int64, true), + Field::new("a", DataType::Float64, true), Field::new("b", DataType::Utf8, true), Field::new("c", DataType::Float64, true), ] @@ -581,8 +576,8 @@ mod tests { assert_eq!(rb.num_rows(), 3); assert_eq!(rb.num_columns(), 6); assert_eq!( - rb.column_by_name("a").unwrap().as_int64_arr(), - &Int64Array::from(vec![None, Some(1), Some(1)]) + rb.column_by_name("a").unwrap().as_float64_arr(), + &Float64Array::from(vec![None, Some(1.0), Some(1.0)]) ); assert_eq!( rb.column_by_name("b").unwrap().as_utf8_arr(), @@ -608,7 +603,7 @@ mod tests { "c": 1 }, { - "a": 1, + "a": "1", "b": "hello", "c": null }, @@ -618,7 +613,7 @@ mod tests { let schema = fields_to_map( [ - Field::new("a", DataType::Int64, true), + Field::new("a", DataType::Float64, true), Field::new("b", DataType::Utf8, true), Field::new("c", DataType::Float64, true), ] @@ -658,8 +653,8 @@ mod tests { assert_eq!(rb.num_rows(), 4); assert_eq!(rb.num_columns(), 7); assert_eq!( - rb.column_by_name("a").unwrap().as_int64_arr(), - &Int64Array::from(vec![Some(1), Some(1), Some(1), Some(1)]) + rb.column_by_name("a").unwrap().as_float64_arr(), + &Float64Array::from(vec![Some(1.0), Some(1.0), Some(1.0), Some(1.0)]) ); assert_eq!( rb.column_by_name("b").unwrap().as_utf8_arr(), @@ -672,13 +667,13 @@ mod tests { ); assert_eq!( - rb.column_by_name("c_a").unwrap().as_int64_arr(), - &Int64Array::from(vec![None, None, Some(1), Some(1)]) + rb.column_by_name("c_a").unwrap().as_float64_arr(), + &Float64Array::from(vec![None, None, Some(1.0), Some(1.0)]) ); assert_eq!( - rb.column_by_name("c_b").unwrap().as_int64_arr(), - &Int64Array::from(vec![None, None, None, Some(2)]) + rb.column_by_name("c_b").unwrap().as_float64_arr(), + &Float64Array::from(vec![None, None, None, Some(2.0)]) ); } } diff --git a/src/metadata.rs b/src/metadata.rs index f768a4e88..5447ea796 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -164,6 +164,20 @@ impl StreamInfo { Ok(Arc::new(schema)) } + /// update the schema in the metadata + pub fn set_schema( + &self, + stream_name: &str, + schema: HashMap>, + ) -> Result<(), MetadataError> { + let mut map = self.write().expect(LOCK_EXPECT); + map.get_mut(stream_name) + .ok_or(MetadataError::StreamMetaNotFound(stream_name.to_string())) + .map(|metadata| { + metadata.schema = schema; + }) + } + pub fn set_alert(&self, stream_name: &str, alerts: Alerts) -> Result<(), MetadataError> { let mut map = self.write().expect(LOCK_EXPECT); map.get_mut(stream_name) diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index e70c326bd..e9ee32f18 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -25,6 +25,7 @@ use super::{ SCHEMA_FILE_NAME, STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY, }; +use crate::event::format::override_num_fields_from_schema; use crate::handlers::http::modal::ingest_server::INGESTOR_META; use crate::handlers::http::users::{DASHBOARDS_DIR, FILTER_DIR, USERS_ROOT_DIR}; use crate::metrics::{EVENTS_STORAGE_SIZE_DATE, LIFETIME_EVENTS_STORAGE_SIZE}; @@ -40,7 +41,7 @@ use crate::{ }; use actix_web_prometheus::PrometheusMetrics; -use arrow_schema::Schema; +use arrow_schema::{Field, Schema}; use async_trait::async_trait; use bytes::Bytes; use chrono::Local; @@ -667,8 +668,21 @@ pub async fn commit_schema_to_storage( schema: Schema, ) -> Result<(), ObjectStorageError> { let storage = CONFIG.storage().get_object_store(); - let stream_schema = storage.get_schema(stream_name).await?; + let mut stream_schema = storage.get_schema(stream_name).await?; + // override the data type of all numeric fields to Float64 + //if data type is not Float64 already + stream_schema = Schema::new(override_num_fields_from_schema( + stream_schema.fields().iter().cloned().collect(), + )); let new_schema = Schema::try_merge(vec![schema, stream_schema]).unwrap(); + + //update the merged schema in the metadata and storage + let schema_map: HashMap> = new_schema + .fields() + .iter() + .map(|field| (field.name().clone(), Arc::clone(field))) + .collect(); + let _ = STREAM_INFO.set_schema(stream_name, schema_map); storage.put_schema(stream_name, &new_schema).await }