Add SharedMergeTree Cloud tests, remove Node 16 from the CI

ClickHouse · Mar 2, 2024 · 0da258a · 0da258a
1 parent 3234ecf
commit 0da258a
Show file tree

Hide file tree

Showing 16 changed files with 295 additions and 261 deletions.
diff --git a/.docker/clickhouse/single_node_tls/Dockerfile b/.docker/clickhouse/single_node_tls/Dockerfile
@@ -1,4 +1,4 @@
-FROM clickhouse/clickhouse-server:23.11-alpine
+FROM clickhouse/clickhouse-server:24.2-alpine
 COPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs
 RUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \
     && chmod 600 /etc/clickhouse-server/certs/* \

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -26,12 +26,12 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        node: [16, 18, 20]
+        node: [18, 20, 21]
     steps:
       - uses: actions/checkout@main
 
       - name: Setup NodeJS ${{ matrix.node }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
@@ -62,17 +62,17 @@ jobs:
       - uses: actions/checkout@main
 
       - name: Start ClickHouse (version - ${{ matrix.clickhouse }}) in Docker
-        uses: isbang/compose-action@v1.1.0
+        uses: isbang/compose-action@v1.5.1
         env:
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
         with:
           compose-file: 'docker-compose.yml'
           down-flags: '--volumes'
 
       - name: Setup NodeJS
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 16
+          node-version: 20
 
       - name: Install dependencies
         run: |
@@ -88,22 +88,22 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        node: [16, 18, 20]
+        node: [18, 20, 21]
         clickhouse: [head, latest]
 
     steps:
       - uses: actions/checkout@main
 
       - name: Start ClickHouse (version - ${{ matrix.clickhouse }}) in Docker
-        uses: isbang/compose-action@v1.1.0
+        uses: isbang/compose-action@v1.5.1
         env:
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
         with:
           compose-file: 'docker-compose.yml'
           down-flags: '--volumes'
 
       - name: Setup NodeJS ${{ matrix.node }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
@@ -129,22 +129,22 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        node: [16, 18, 20]
+        node: [18, 20, 21]
         clickhouse: [head, latest]
 
     steps:
       - uses: actions/checkout@main
 
       - name: Start ClickHouse cluster (version - ${{ matrix.clickhouse }}) in Docker
-        uses: isbang/compose-action@v1.1.0
+        uses: isbang/compose-action@v1.5.1
         env:
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
         with:
           compose-file: 'docker-compose.cluster.yml'
           down-flags: '--volumes'
 
       - name: Setup NodeJS ${{ matrix.node }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
@@ -167,17 +167,17 @@ jobs:
       - uses: actions/checkout@main
 
       - name: Start ClickHouse cluster (version - ${{ matrix.clickhouse }}) in Docker
-        uses: isbang/compose-action@v1.1.0
+        uses: isbang/compose-action@v1.5.1
         env:
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
         with:
           compose-file: 'docker-compose.cluster.yml'
           down-flags: '--volumes'
 
       - name: Setup NodeJS
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 16
+          node-version: 20
 
       - name: Install dependencies
         run: |
@@ -193,13 +193,13 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        node: [16, 18, 20]
+        node: [18, 20, 21]
 
     steps:
       - uses: actions/checkout@main
 
       - name: Setup NodeJS ${{ matrix.node }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
 
@@ -214,6 +214,33 @@ jobs:
         run: |
           npm run test:node:integration:cloud
 
+  node-integration-tests-cloud-smt:
+    needs: node-unit-tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        node: [18, 20, 21]
+
+    steps:
+      - uses: actions/checkout@main
+
+      - name: Setup NodeJS ${{ matrix.node }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install dependencies
+        run: |
+          npm install
+
+      - name: Run integration tests
+        env:
+          CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }}
+          CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }}
+        run: |
+          npm run test:node:integration:cloud_smt
+
   web-integration-tests-cloud:
     needs: node-unit-tests
     runs-on: ubuntu-latest
@@ -222,9 +249,9 @@ jobs:
       - uses: actions/checkout@main
 
       - name: Setup NodeJS
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 16
+          node-version: 20
 
       - name: Install dependencies
         run: |
@@ -236,3 +263,26 @@ jobs:
           CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
         run: |
           npm run test:web:integration:cloud
+
+  web-integration-tests-cloud-smt:
+    needs: node-unit-tests
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+      - uses: actions/checkout@main
+
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install dependencies
+        run: |
+          npm install
+
+      - name: Run integration tests
+        env:
+          CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }}
+          CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }}
+        run: |
+          npm run test:web:integration:cloud_smt
diff --git a/docker-compose.cluster.yml b/docker-compose.cluster.yml
@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   clickhouse1:
-    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
+    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.2-alpine}'
     ulimits:
       nofile:
         soft: 262144
@@ -19,7 +19,7 @@ services:
       - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml'
 
   clickhouse2:
-    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
+    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.2-alpine}'
     ulimits:
       nofile:
         soft: 262144

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   clickhouse:
-    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
+    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.2-alpine}'
     container_name: 'clickhouse-js-clickhouse-server'
     ports:
       - '8123:8123'

diff --git a/karma.config.cjs b/karma.config.cjs
@@ -38,7 +38,7 @@ module.exports = function (config) {
         'sourcemap',
       ],
     },
-    reporters: ['progress'],
+    reporters: ['mocha'],
     port: 9876,
     colors: true,
     logLevel: config.LOG_INFO,

diff --git a/package.json b/package.json
@@ -34,9 +34,11 @@
     "test:node:integration": ".scripts/jasmine.sh jasmine.node.integration.json",
     "test:node:integration:local_cluster": "CLICKHOUSE_TEST_ENVIRONMENT=local_cluster npm run test:node:integration",
     "test:node:integration:cloud": "CLICKHOUSE_TEST_ENVIRONMENT=cloud npm run test:node:integration",
+    "test:node:integration:cloud_smt": "CLICKHOUSE_TEST_ENVIRONMENT=cloud_smt npm run test:node:integration",
     "test:web": "karma start karma.config.cjs",
     "test:web:integration:local_cluster": "CLICKHOUSE_TEST_ENVIRONMENT=local_cluster npm run test:web",
     "test:web:integration:cloud": "CLICKHOUSE_TEST_ENVIRONMENT=cloud npm run test:web",
+    "test:web:integration:cloud_smt": "CLICKHOUSE_TEST_ENVIRONMENT=cloud_smt npm run test:web",
     "prepare": "husky install"
   },
   "devDependencies": {
@@ -60,6 +62,7 @@
     "karma-chrome-launcher": "^3.2.0",
     "karma-firefox-launcher": "^2.1.2",
     "karma-jasmine": "^5.1.0",
+    "karma-mocha-reporter": "^2.2.5",
     "karma-sourcemap-loader": "^0.4.0",
     "karma-typescript": "^5.5.4",
     "karma-webpack": "^5.0.0",

diff --git a/packages/client-common/__tests__/fixtures/read_only_user.ts b/packages/client-common/__tests__/fixtures/read_only_user.ts
@@ -14,7 +14,22 @@ export async function createReadOnlyUser(client: ClickHouseClient) {
   let createUser: string
   let grant: string
   switch (env) {
-    case TestEnv.Cloud: // we do not need 'ON CLUSTER' in the cloud
+    // requires select_sequential_consistency = 1 for immediate selects after inserts
+    case TestEnv.CloudSMT:
+      createUser = `
+          CREATE USER ${username}
+          IDENTIFIED WITH sha256_password BY '${password}'
+          DEFAULT DATABASE ${database}
+          SETTINGS readonly = 1, select_sequential_consistency = 1
+        `
+      grant = `
+          GRANT SHOW TABLES, SELECT
+          ON ${database}.*
+          TO ${username}
+        `
+      break
+    // we do not need 'ON CLUSTER' in the cloud, so it's the same as a local docker
+    case TestEnv.Cloud:
     case TestEnv.LocalSingleNode:
       createUser = `
           CREATE USER ${username}

diff --git a/packages/client-common/__tests__/fixtures/simple_table.ts b/packages/client-common/__tests__/fixtures/simple_table.ts
@@ -24,6 +24,7 @@ export function createSimpleTable<Stream = unknown>(
       // ENGINE can be omitted in the cloud statements:
       // it will use ReplicatedMergeTree and will add ON CLUSTER as well
       case TestEnv.Cloud:
+      case TestEnv.CloudSMT:
         return `
           CREATE TABLE ${tableName}
           (id UInt64, name String, sku Array(UInt8))
@@ -53,6 +54,7 @@ export function createSimpleTable<Stream = unknown>(
 function filterSettingsBasedOnEnv(settings: MergeTreeSettings, env: TestEnv) {
   switch (env) {
     case TestEnv.Cloud:
+    case TestEnv.CloudSMT:
       // ClickHouse Cloud does not like this particular one
       // Local cluster, however, does.
       if ('non_replicated_deduplication_window' in settings) {

diff --git a/packages/client-common/__tests__/fixtures/table_with_fields.ts b/packages/client-common/__tests__/fixtures/table_with_fields.ts
@@ -17,6 +17,7 @@ export async function createTableWithFields(
         // ENGINE can be omitted in the cloud statements:
         // it will use ReplicatedMergeTree and will add ON CLUSTER as well
         case TestEnv.Cloud:
+        case TestEnv.CloudSMT:
           return `
             CREATE TABLE ${tableName}
             (id UInt32, ${fields})