Merge pull request #690 from Altinity/v4

remove all duplicated logic, move all macroses and query parsing functionality to golang, and compile golang parser to WASM

.config/webpack/webpack.config.ts

@@ -11,6 +11,7 @@ import ForkTsCheckerWebpackPlugin from 'fork-ts-checker-webpack-plugin';
 import LiveReloadPlugin from 'webpack-livereload-plugin';
 import path from 'path';
 import ReplaceInFileWebpackPlugin from 'replace-in-file-webpack-plugin';
+import NodePolyfillPlugin from 'node-polyfill-webpack-plugin';
 import { Configuration } from 'webpack';
 
 import { getPackageJson, getPluginJson, hasReadme, getEntries, isWSL } from './utils';
@@ -139,6 +140,7 @@ const config = async (env): Promise<Configuration> => {
     },
 
     plugins: [
+      new NodePolyfillPlugin(),
       new CopyWebpackPlugin({
         patterns: [
           // If src/README.md exists use it; otherwise the root README
@@ -192,6 +194,13 @@ const config = async (env): Promise<Configuration> => {
     ],
 
     resolve: {
+      alias: {
+        // Map 'node:' scheme to standard module names
+        'node:crypto': 'crypto-browserify',
+        'node:fs': 'browserify-fs', // Note: fs is not fully supported in the browser
+        'node:util': 'util/',
+        // Add other aliases as needed
+      },
       extensions: ['.js', '.jsx', '.ts', '.tsx'],
       // handle resolving "rootDir" paths
       modules: [path.resolve(process.cwd(), 'src'), 'node_modules'],

.cursorignore

@@ -3,3 +3,4 @@
 .go-cache/
 .idea/
 dist/
+.aider*

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.aider*
+.go-cache
+.idea
+go_coverage
+instrumented
+node_modules
+venv
+coverage

.github/workflows/ci.yml

@@ -18,21 +18,9 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'npm'
 
-      - name: Install dependencies
-        run: npm ci
-
-      - name: Check types
-        run: npm run typecheck
-      - name: Lint
-        run: npm run lint
-      - name: Unit tests
-        run: npm run test:ci
-      - name: Build frontend
-        run: npm run build
-
       - name: Check for backend
         id: check-for-backend
         run: |
@@ -47,6 +35,22 @@ jobs:
         with:
           go-version: '1.23'
 
+      - uses: acifani/setup-tinygo@v2
+        with:
+          tinygo-version: '0.35.0'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Check types
+        run: npm run typecheck
+      - name: Lint
+        run: npm run lint
+      - name: Test frontend
+        run: npm run test:ci
+      - name: Build frontend
+        run: npm run build
+
       - name: Test backend
         if: steps.check-for-backend.outputs.has-backend == 'true'
         uses: magefile/mage-action@v3

.github/workflows/is-compatible.yml

@@ -9,8 +9,11 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'npm'
+      - uses: acifani/setup-tinygo@v2
+        with:
+          tinygo-version: '0.35.0'
       - name: Install dependencies
         run: npm ci
       - name: Build plugin

.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
           cache: 'npm'
 
       - name: Setup Go environment

.github/workflows/testflows.yml

@@ -29,6 +29,10 @@ jobs:
         with:
           ref: ${{ inputs.ref }}
 
+      - uses: acifani/setup-tinygo@v2
+        with:
+          tinygo-version: '0.35.0'
+
       - name: Setup
         run: tests/testflows/infra/setup.sh
 

.gitignore

@@ -11,7 +11,7 @@ yarn-error.log*
 .pnpm-debug.log*
 
 node_modules/
-
+src/static/*
 # Runtime data
 pids
 *.pid

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 3.4.0 (not released yet)
+## Enhancements:
+* add $lttb and $lttbMs macros to allow show down-sampled time series which will contains more outliers than avg or other kind of aggregation, fix https://github.com/Altinity/clickhouse-grafana/issues/500
+
 # 3.3.1 (2024-12-27)
 ## Enhancements:
 * Add using window functions instead of `runningDifference` and `neighbor` for macros, to avoid `allow_deprecated_error_prone_window_functions`, fix https://github.com/Altinity/clickhouse-grafana/issues/572

CONTRIBUTING.md

@@ -20,6 +20,16 @@ docker compose up --no-deps -d grafana clickhouse
 ```
 after that open http://localhost:3000/ to open grafana instance with one clickhouse datasource
 
+#### Ability to debug WASM code
+
+Install https://goo.gle/wasm-debugging-extension
+
+Check in JS console in DevTools something like
+```
+[C/C++ DevTools Support (DWARF)] Loading debug symbols for wasm://wasm/main-006e733a...
+[C/C++ DevTools Support (DWARF)] Loaded debug symbols for wasm://wasm/main-006e733a, found 213 source file(s)
+```
+
 #### Frontend Builder
 
 The frontend builder is the docker container used to transpile the typescript source code into the javascript found in the `dist` dir. This will affect the grafana query and configuration functionality.

README.md

@@ -269,6 +269,49 @@ GROUP BY t
 ORDER BY t
 ```
 
+---
+### $lttb(buckets_number, [field1, ... fieldN], x_field, y_field) - allow show down-sampled time series which will contains more outliers than avg or other kind of aggregation
+
+If bucket_number is `auto`, then it will calculated as `toUInt64( ($to-$from) / $interval )`
+Example usage:
+
+```sql
+$lttb(auto, category, event_time, count(*) c)
+FROM requests GROUP BY category
+```
+
+Query will be transformed into:
+
+```sql
+SELECT category, lttb_result.1 AS event_time, lttb_result.2 AS c FROM (
+    SELECT category, untuple(arrayJoin(lttb(toUInt64( ($to - $from) / $interval ))(event_time, cont(*) AS c))) AS lttb_result
+    FROM requests WHERE $timeFilter GROUP BY category
+) ORDER BY event_time
+```
+
+---
+
+---
+### $lttbMs(buckets_number, [field1,... fieldN], x_field, y_field) - same as $lttb but for time series with ms
+
+If bucket_number is `auto`, then it will calculated as `toUInt64( ($__to-$__from) / $__interval_ms )`
+
+Example usage:
+
+```sql
+$lttbMs(100, event_time, count(*) c)
+FROM requests
+```
+
+Query will be transformed into:
+
+```sql
+SELECT lttb_result.1 AS event_time, lttb_result.2 AS c FROM (
+    SELECT untuple(arrayJoin(lttb(100)(event_time, count(*) AS c))) AS lttb_result
+    FROM requests WHERE $timeFilterMs    
+) ORDER BY event_time
+```
+
 ---
 
 ### $rateColumns(key, value) - is a combination of $columns and $rate

docker-compose.yaml

@@ -137,9 +137,17 @@ services:
 
 
   frontend_builder:
-    image: node:lts
+    build:
+      dockerfile: ./docker/builder/Dockerfile
+    environment:
+      GOCACHE: "/go-cache"
+      RUN_TESTS: "${RUN_TESTS:-.+}"
+      CGO_ENABLED: "0"
     working_dir: /grafana-clickhouse
     volumes:
+      - ./.go-cache:/go/pkg/mod
+      - ./.go-cache:/root/go/pkg/mod
+      - ./.go-cache:/go-cache
       - ./:/grafana-clickhouse
       # for speedup npm install under WSL
       # - "node_modules:/grafana-clickhouse/node_modules"
@@ -151,7 +159,8 @@ services:
       '
 
   backend_builder:
-    image: golang:1.23
+    build:
+      dockerfile: ./docker/builder/Dockerfile
     working_dir: /go/src/grafana-clickhouse
     environment:
       GOCACHE: "/go-cache"
@@ -160,18 +169,20 @@ services:
     volumes:
       - ./:/go/src/grafana-clickhouse
       - ./.go-cache:/go/pkg/mod
+      - ./.go-cache:/root/go/pkg/mod
       - ./.go-cache:/go-cache
     command: |
       bash -xec "
         git config --global --add safe.directory /go/src/grafana-clickhouse &&
         (command -v mage || go install -v github.com/magefile/mage@latest) &&
-        go test -timeout 1m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v ./pkg/ &&
+        go test -timeout 1m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v ./pkg/... &&
         mage -v && 
         chmod +x ./dist/altinity-clickhouse-plugin*
       "
 
   frontend_coverage_builder:
-    image: node:lts
+    build:
+      dockerfile: ./docker/builder/Dockerfile
     working_dir: /grafana-clickhouse
     volumes:
       - ./:/grafana-clickhouse
@@ -180,15 +191,17 @@ services:
     command: |
       bash -xec '
         if [[ "3" == `ls -la ./node_modules/ | wc -l` ]]; then npm install; fi && \
+        npm run build:wasm &&
         mkdir -p instrumented &&
         npm run create_instrumented && 
         cp -R -n src/* instrumented/ &&
-        npm run build:test
+        npm run build:test:frontend &&
         npm run test:coverage
       '
 
   frontend_coverage_generate:
-    image: node:lts
+    build:
+      dockerfile: ./docker/builder/Dockerfile
     working_dir: /grafana-clickhouse
     volumes:
       - ./:/grafana-clickhouse
@@ -205,7 +218,8 @@ services:
       '
 
   backend_coverage_builder:
-    image: golang:1.23
+    build:
+      dockerfile: ./docker/builder/Dockerfile
     working_dir: /go/src/grafana-clickhouse
     environment:
       GOCACHE: "/go-cache"
@@ -214,19 +228,22 @@ services:
     volumes:
       - ./:/go/src/grafana-clickhouse
       - ./.go-cache:/go/pkg/mod
+      - ./.go-cache:/root/go/pkg/mod
       - ./.go-cache:/go-cache
     command: |
       bash -xec "
         git config --global --add safe.directory /go/src/grafana-clickhouse &&
         (command -v mage || go install -v github.com/magefile/mage@latest) &&
         mkdir -p /go/src/grafana-clickhouse/go_coverage/raw &&
-        go test -timeout 1m -cover -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v ./pkg/ -test.gocoverdir="/go/src/grafana-clickhouse/go_coverage" &&
+        go env &&
+        go test -timeout 1m -cover -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v ./pkg/... -test.gocoverdir="/go/src/grafana-clickhouse/go_coverage" &&
         GOOS=linux GOARCH=amd64 go build -cover -buildvcs=false -o ./dist/altinity-clickhouse-plugin_linux_amd64 ./pkg/ &&
         chmod +x dist/altinity-clickhouse-plugin*
       "
 
   backend_coverage_generate:
-    image: golang:1.23
+    build:
+      dockerfile: ./docker/builder/Dockerfile
     working_dir: /go/src/grafana-clickhouse
     volumes:
       - ./:/go/src/grafana-clickhouse

docker/builder/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:latest
+LABEL maintainer="Altinity <[email protected]>"
+
+ARG NODEJS_VERSION=22
+ARG GOLANG_VERSION=1.23
+ARG TINYGO_VERSION=0.35.0
+
+RUN apt update && apt install --no-install-recommends -y curl git gnupg2 software-properties-common && \
+    ( curl -fsSL https://deb.nodesource.com/setup_${NODEJS_VERSION}.x | bash - ) && \
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 52B59B1571A79DBC054901C0F6BC817356A3D45E && \
+    add-apt-repository -y ppa:longsleep/golang-backports && \
+    apt update && apt install -y nodejs golang-${GOLANG_VERSION} && \
+    ln -nsfv "/usr/lib/go-${GOLANG_VERSION}/bin/go" /usr/bin/go && \
+    curl -sL -o "/tmp/tinygo_${TINYGO_VERSION}_amd64.deb" "https://github.com/tinygo-org/tinygo/releases/download/v${TINYGO_VERSION}/tinygo_${TINYGO_VERSION}_amd64.deb" && \
+    dpkg -i "/tmp/tinygo_${TINYGO_VERSION}_amd64.deb"  && \
+    rm -rfv "/tmp/tinygo_${TINYGO_VERSION}_amd64.deb"  && \
+    go install -v github.com/magefile/mage@latest && go env > $HOME/go_env && . $HOME/go_env && ln -nsfv "$GOPATH/bin/mage" /usr/bin/mage && \
+    apt clean && rm -rf /var/lib/apt/lists/*
+

docker/clickhouse/init_schema.sql

@@ -332,3 +332,19 @@ SELECT
    toUInt64(now() - INTERVAL number SECOND)*1000000000 + toUInt64(randUniform(0, 1000000000)) AS tUInt64_9,
    number AS value
 FROM numbers(86400);
+
+DROP TABLE IF EXISTS default.test_lttb SYNC;
+CREATE TABLE default.test_lttb(
+  event_time DateTime,
+  event_time_ms DateTime64(3),
+  category LowCardinality(String),
+  requests UInt64
+) ENGINE=MergeTree() ORDER BY event_time;
+
+INSERT INTO default.test_lttb
+SELECT
+  now() - INTERVAL number SECOND,
+  now() - INTERVAL number SECOND + INTERVAL randUniform(0, 100) MILLISECOND,
+  'category' || toString(number%25),
+  rand() % 1000000
+FROM numbers(86400);