diff --git a/.github/workflows/test_destination_clickhouse.yml b/.github/workflows/test_destination_clickhouse.yml index d834df6b28..6b782219a6 100644 --- a/.github/workflows/test_destination_clickhouse.yml +++ b/.github/workflows/test_destination_clickhouse.yml @@ -20,7 +20,7 @@ env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} ACTIVE_DESTINATIONS: "[\"clickhouse\"]" - ALL_FILESYSTEM_DRIVERS: "[\"memory\"]" + ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]" jobs: get_docs_changes: @@ -45,7 +45,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10.x" + python-version: "3.11.x" - name: Install Poetry uses: snok/install-poetry@v1.3.2 @@ -76,3 +76,21 @@ jobs: poetry run pytest tests/load name: Run all tests Linux if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} + + - name: Start ClickHouse + run: docker-compose -f ".github/clickhouse-compose.yml" up -d + + - run: poetry run pytest tests/load && poetry run pytest tests/cli + name: Run all local tests Linux + env: + DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost + DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data + DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader + DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader + DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000 + DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123 + DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0 + + - name: Stop ClickHouse + if: always() + run: docker-compose -f ".github/clickhouse-compose.yml" down -v diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index a30e7d2317..263d3f588c 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -1,7 +1,7 @@ # Tests destinations that can run without credentials. -# i.e. postgres, duckdb, weaviate, clickhouse and filesystem (with local fs/memory bucket) +# i.e. local postgres, duckdb, filesystem (with local fs/memory bucket) -name: dest | postgres, duckdb, weaviate, clickhouse and fs local tests +name: dest | postgres, duckdb and fs on: pull_request: @@ -21,7 +21,7 @@ env: RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} - ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\", \"weaviate\", \"clickhouse\"]" + ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\", \"weaviate\"]" ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]" DESTINATION__WEAVIATE__VECTORIZER: text2vec-contextionary @@ -33,7 +33,7 @@ jobs: uses: ./.github/workflows/get_docs_changes.yml run_loader: - name: dest | postgres, duckdb, weaviate, clickhouse and fs local tests + name: dest | postgres, duckdb and fs local tests needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' strategy: @@ -70,9 +70,6 @@ jobs: - name: Start weaviate run: docker-compose -f ".github/weaviate-compose.yml" up -d - - name: Start ClickHouse - run: docker-compose -f ".github/clickhouse-compose.yml" up -d - - name: Setup Python uses: actions/setup-python@v4 with: @@ -93,29 +90,17 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations - name: Install dependencies - run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate -E clickhouse --with sentry-sdk --with pipeline -E deltalake + run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline -E deltalake - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - # Always run full suite, also on branches. + # always run full suite, also on branches - run: poetry run pytest tests/load && poetry run pytest tests/cli name: Run tests Linux env: DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data - DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost - DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data - DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader - DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader - DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000 - DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123 - DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0 - - name: Stop weaviate if: always() run: docker-compose -f ".github/weaviate-compose.yml" down -v - - - name: Stop ClickHouse - if: always() - run: docker-compose -f ".github/clickhouse-compose.yml" down -v diff --git a/dlt/destinations/impl/clickhouse/clickhouse.py b/dlt/destinations/impl/clickhouse/clickhouse.py index c2955af550..70c2749085 100644 --- a/dlt/destinations/impl/clickhouse/clickhouse.py +++ b/dlt/destinations/impl/clickhouse/clickhouse.py @@ -66,8 +66,6 @@ "merge_tree": "MergeTree", "shared_merge_tree": "SharedMergeTree", "replicated_merge_tree": "ReplicatedMergeTree", - "stripe_log": "StripeLog", - "tiny_log": "TinyLog", } diff --git a/dlt/destinations/impl/clickhouse/clickhouse_adapter.py b/dlt/destinations/impl/clickhouse/clickhouse_adapter.py index 97e163ed62..413b3007b7 100644 --- a/dlt/destinations/impl/clickhouse/clickhouse_adapter.py +++ b/dlt/destinations/impl/clickhouse/clickhouse_adapter.py @@ -6,7 +6,7 @@ TTableEngineType = Literal[ - "merge_tree", "shared_merge_tree", "replicated_merge_tree", "stripe_log", "tiny_log" + "merge_tree", "shared_merge_tree", "replicated_merge_tree", ] """