Update ClickHouse test workflow and remove engine types

Signed-off-by: Marcel Coetzee <[email protected]>

.github/workflows/test_destination_clickhouse.yml

@@ -20,7 +20,7 @@ env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   ACTIVE_DESTINATIONS: "[\"clickhouse\"]"
-  ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
+  ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]"
 
 jobs:
   get_docs_changes:
@@ -45,7 +45,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10.x"
+          python-version: "3.11.x"
 
       - name: Install Poetry
         uses: snok/[email protected]
@@ -76,3 +76,21 @@ jobs:
           poetry run pytest tests/load
         name: Run all tests Linux
         if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}
+
+      - name: Start ClickHouse
+        run: docker-compose -f ".github/clickhouse-compose.yml" up -d
+
+      - run: poetry run pytest tests/load && poetry run pytest tests/cli
+        name: Run all local tests Linux
+        env:
+          DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost
+          DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data
+          DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader
+          DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader
+          DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000
+          DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123
+          DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0
+
+      - name: Stop ClickHouse
+        if: always()
+        run: docker-compose -f ".github/clickhouse-compose.yml" down -v

.github/workflows/test_local_destinations.yml

@@ -1,7 +1,7 @@
 # Tests destinations that can run without credentials.
-# i.e. postgres, duckdb, weaviate, clickhouse and filesystem (with local fs/memory bucket)
+# i.e. local postgres, duckdb, filesystem (with local fs/memory bucket)
 
-name: dest | postgres, duckdb, weaviate, clickhouse and fs local tests
+name: dest | postgres, duckdb and fs
 
 on:
   pull_request:
@@ -21,7 +21,7 @@ env:
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
-  ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\", \"weaviate\", \"clickhouse\"]"
+  ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\", \"weaviate\"]"
   ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]"
 
   DESTINATION__WEAVIATE__VECTORIZER: text2vec-contextionary
@@ -33,7 +33,7 @@ jobs:
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_loader:
-    name: dest | postgres, duckdb, weaviate, clickhouse and fs local tests
+    name: dest | postgres, duckdb and fs local tests
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
@@ -70,9 +70,6 @@ jobs:
       - name: Start weaviate
         run: docker-compose -f ".github/weaviate-compose.yml" up -d
 
-      - name: Start ClickHouse
-        run: docker-compose -f ".github/clickhouse-compose.yml" up -d
-
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
@@ -93,29 +90,17 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate -E clickhouse --with sentry-sdk --with pipeline -E deltalake
+        run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline -E deltalake
 
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
-      # Always run full suite, also on branches.
+      # always run full suite, also on branches
       - run: poetry run pytest tests/load && poetry run pytest tests/cli
         name: Run tests Linux
         env:
           DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data
-          DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost
-          DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data
-          DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader
-          DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader
-          DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000
-          DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123
-          DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0
-
 
       - name: Stop weaviate
         if: always()
         run: docker-compose -f ".github/weaviate-compose.yml" down -v
-
-      - name: Stop ClickHouse
-        if: always()
-        run: docker-compose -f ".github/clickhouse-compose.yml" down -v

dlt/destinations/impl/clickhouse/clickhouse.py

@@ -66,8 +66,6 @@
     "merge_tree": "MergeTree",
     "shared_merge_tree": "SharedMergeTree",
     "replicated_merge_tree": "ReplicatedMergeTree",
-    "stripe_log": "StripeLog",
-    "tiny_log": "TinyLog",
 }
 
 

dlt/destinations/impl/clickhouse/clickhouse_adapter.py

@@ -6,7 +6,7 @@
 
 
 TTableEngineType = Literal[
-    "merge_tree", "shared_merge_tree", "replicated_merge_tree", "stripe_log", "tiny_log"
+    "merge_tree", "shared_merge_tree", "replicated_merge_tree",
 ]
 
 """