From 4d5205437d01099964cf9af417ed9dac62e4e774 Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Fri, 11 Oct 2024 16:37:22 +0200 Subject: [PATCH 01/12] fix (OutputManifest): don't overwrite source with destination --- src/FileDumper/OutputManifest/OutputManifest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/FileDumper/OutputManifest/OutputManifest.php b/src/FileDumper/OutputManifest/OutputManifest.php index eb7d4f8..d7c3585 100644 --- a/src/FileDumper/OutputManifest/OutputManifest.php +++ b/src/FileDumper/OutputManifest/OutputManifest.php @@ -87,19 +87,19 @@ protected function processTableDefinition( array $configuredOutputTables, ): void { $tableName = $tableDef->getTableName(); - $destinationTableName = $tableName; +// $destinationTableName = $tableName; $configuredPrimaryKeys = []; if ($configuredOutputTables !== []) { foreach ($configuredOutputTables as $configuredOutputTable) { if (isset($configuredOutputTable['source']) && $configuredOutputTable['source'] === $tableName) { - $destinationTableName = $configuredOutputTable['destination']; +// $destinationTableName = $configuredOutputTable['destination']; $configuredPrimaryKeys = $configuredOutputTable['primary_key'] ?? []; break; } } } - $realTableName = $this->getRealTableName($destinationTableName); + $realTableName = $this->getRealTableName($tableName); $dbtColumnsMetadata = $dbtMetadata[$tableName]['column_metadata'] ?? []; if ($configuredPrimaryKeys !== []) { From 3b2e5315aeafc67bb9b28c89288b05775ca2c5c0 Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Fri, 11 Oct 2024 20:08:52 +0200 Subject: [PATCH 02/12] fix (OutputManiest): set destination from OM config --- .../OutputManifest/OutputManifest.php | 17 ++++++++++++++--- tests/phpunit/FileDumper/OutputManifestTest.php | 4 ++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/FileDumper/OutputManifest/OutputManifest.php b/src/FileDumper/OutputManifest/OutputManifest.php index d7c3585..b13b1a2 100644 --- a/src/FileDumper/OutputManifest/OutputManifest.php +++ b/src/FileDumper/OutputManifest/OutputManifest.php @@ -87,12 +87,12 @@ protected function processTableDefinition( array $configuredOutputTables, ): void { $tableName = $tableDef->getTableName(); -// $destinationTableName = $tableName; + $destinationTableName = null; $configuredPrimaryKeys = []; if ($configuredOutputTables !== []) { foreach ($configuredOutputTables as $configuredOutputTable) { if (isset($configuredOutputTable['source']) && $configuredOutputTable['source'] === $tableName) { -// $destinationTableName = $configuredOutputTable['destination']; + $destinationTableName = $configuredOutputTable['destination']; $configuredPrimaryKeys = $configuredOutputTable['primary_key'] ?? []; break; } @@ -111,7 +111,14 @@ protected function processTableDefinition( $columnsMetadata = $this->getColumnsMetadata($tableDef, $dbtColumnsMetadata); $tableMetadata = $this->getTableMetadata($tableName, $realTableName, $dbtMetadata); - $this->createAndWriteTableManifest($tableDef, $realTableName, $tableMetadata, $columnsMetadata, $dbtPrimaryKey); + $this->createAndWriteTableManifest( + $tableDef, + $realTableName, + $tableMetadata, + $columnsMetadata, + $dbtPrimaryKey, + $destinationTableName + ); } /** @@ -160,6 +167,7 @@ protected function createAndWriteTableManifest( array $tableMetadata, object $columnsMetadata, array $dbtPrimaryKey, + ?string $destination = null, ): void { $tableManifestOptions = new ManifestOptions(); $primaryKeys = self::getPrimaryKeyColumnNames( @@ -205,6 +213,9 @@ protected function createAndWriteTableManifest( } $tableManifestOptions->setSchema($schema); $tableManifestOptions->setTableMetadata($tableMetadataKeyValue); + if ($destination) { + $tableManifestOptions->setDestination($destination); + } $this->manifestManager->writeTableManifest($realTableName, $tableManifestOptions, $this->legacyFormat); } diff --git a/tests/phpunit/FileDumper/OutputManifestTest.php b/tests/phpunit/FileDumper/OutputManifestTest.php index 5db438a..8d0098b 100644 --- a/tests/phpunit/FileDumper/OutputManifestTest.php +++ b/tests/phpunit/FileDumper/OutputManifestTest.php @@ -777,7 +777,7 @@ public function testDumpJsonWithOutputTables(): void ], ]); - $tableManifestPath1 = $this->dataDir . '/out/tables/OUT.TEST.BEERS_WITH_BREWERIES.manifest'; + $tableManifestPath1 = $this->dataDir . '/out/tables/BEERS_WITH_BREWERIES.manifest'; self::assertFileExists($tableManifestPath1); $tableManifestPath2 = $this->dataDir . '/out/tables/beers.manifest'; @@ -816,7 +816,7 @@ public function testDumpJsonWithOutputTables(): void ], [ 'key' => 'KBC.name', - 'value' => 'OUT.TEST.BEERS_WITH_BREWERIES', + 'value' => 'BEERS_WITH_BREWERIES', ], [ 'key' => 'KBC.datatype.backend', From 8789a55d744e5afbf840b7610ea1ad38c61a65fa Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Fri, 11 Oct 2024 20:09:57 +0200 Subject: [PATCH 03/12] fix cs --- src/FileDumper/OutputManifest/OutputManifest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FileDumper/OutputManifest/OutputManifest.php b/src/FileDumper/OutputManifest/OutputManifest.php index b13b1a2..b788e4d 100644 --- a/src/FileDumper/OutputManifest/OutputManifest.php +++ b/src/FileDumper/OutputManifest/OutputManifest.php @@ -117,7 +117,7 @@ protected function processTableDefinition( $tableMetadata, $columnsMetadata, $dbtPrimaryKey, - $destinationTableName + $destinationTableName, ); } From 65ef55972c1812ae1ac0bf4c74af7cd273e91e57 Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Mon, 14 Oct 2024 12:40:07 +0200 Subject: [PATCH 04/12] tests (run-action-with-output-table-mapping): out.c-test.fct_model.manifest -> fct_model.manifest --- .../tables/{out.c-test.fct_model.manifest => fct_model.manifest} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/functional/run-action-with-output-table-mapping/expected/data/out/tables/{out.c-test.fct_model.manifest => fct_model.manifest} (100%) diff --git a/tests/functional/run-action-with-output-table-mapping/expected/data/out/tables/out.c-test.fct_model.manifest b/tests/functional/run-action-with-output-table-mapping/expected/data/out/tables/fct_model.manifest similarity index 100% rename from tests/functional/run-action-with-output-table-mapping/expected/data/out/tables/out.c-test.fct_model.manifest rename to tests/functional/run-action-with-output-table-mapping/expected/data/out/tables/fct_model.manifest From 4995ba3105c2c8cbc6d640c92b9e214cdae37482 Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Tue, 15 Oct 2024 10:03:18 +0200 Subject: [PATCH 05/12] tests (OutputManifest): test that destination is in the manifest --- .../expected/data/out/tables/fct_model.manifest | 5 +++-- tests/phpunit/FileDumper/OutputManifestTest.php | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/functional/run-action-with-output-table-mapping/expected/data/out/tables/fct_model.manifest b/tests/functional/run-action-with-output-table-mapping/expected/data/out/tables/fct_model.manifest index d6c822f..eae1e3c 100644 --- a/tests/functional/run-action-with-output-table-mapping/expected/data/out/tables/fct_model.manifest +++ b/tests/functional/run-action-with-output-table-mapping/expected/data/out/tables/fct_model.manifest @@ -1,4 +1,5 @@ { + "destination": "out.c-test.fct_model", "metadata": [ { "key": "KBC.description", @@ -6,7 +7,7 @@ }, { "key": "KBC.name", - "value": "out.c-test.fct_model" + "value": "fct_model" }, { "key": "KBC.datatype.backend", @@ -96,4 +97,4 @@ } ] } -} \ No newline at end of file +} diff --git a/tests/phpunit/FileDumper/OutputManifestTest.php b/tests/phpunit/FileDumper/OutputManifestTest.php index 8d0098b..2f66c9c 100644 --- a/tests/phpunit/FileDumper/OutputManifestTest.php +++ b/tests/phpunit/FileDumper/OutputManifestTest.php @@ -784,6 +784,7 @@ public function testDumpJsonWithOutputTables(): void self::assertFileDoesNotExist($tableManifestPath2); /** @var array{ + * 'destination': string, * 'primary_key': array, * 'columns': array, * 'metadata': array>, @@ -792,6 +793,7 @@ public function testDumpJsonWithOutputTables(): void */ $manifest1 = (array) json_decode((string) file_get_contents($tableManifestPath1), true); + self::assertSame('out.test.beers_with_breweries', $manifest1['destination']); self::assertEquals([ 'brewery_id', 'beer_id', From 44f9b006a636aa83c5eb26fe6bbd9651d248eece Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Mon, 21 Oct 2024 11:27:43 +0200 Subject: [PATCH 06/12] ci: add integration test - run job in kbc project --- .github/workflows/push.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 81fe36b..bc1376e 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -69,9 +69,8 @@ env: DOCKERHUB_USER: '${{ secrets.DOCKERHUB_USER }}' DOCKERHUB_TOKEN: '${{ secrets.DOCKERHUB_TOKEN }}' - KBC_STORAGE_TOKEN: '${{ secrets.KBC_STORAGE_TOKEN }}' - KBC_TEST_PROJECT_URL: '' - KBC_TEST_PROJECT_CONFIGS: '' + KBC_TEST_PROJECT_URL: 'https://connection.keboola.com/admin/projects/9317' + KBC_TEST_PROJECT_CONFIGS: '1192751064' jobs: build: runs-on: ubuntu-latest @@ -210,12 +209,12 @@ jobs: steps: - name: 'Run KBC test jobs' - if: 'env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS' + if: 'env.KBC_TOKEN && env.KBC_TEST_PROJECT_CONFIGS' uses: keboola/action-run-configs-parallel@master with: - token: '${{ env.KBC_STORAGE_TOKEN }}' + token: '${{ env.KBC_TOKEN }}' componentId: '${{ env.KBC_DEVELOPERPORTAL_APP }}' - tag: '${{ needs.build.outputs.app_image_tag }}-1.5.0' + tag: '${{ needs.build.outputs.app_image_tag }}-1.8.6' configs: '${{ env.KBC_TEST_PROJECT_CONFIGS }}' deploy: needs: From 384d2818c0f90670f715ffc2b10f2f5357779e70 Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Tue, 22 Oct 2024 13:37:46 +0200 Subject: [PATCH 07/12] ci/tests: integration test for GCP BQ, fixed tests coz new tables added to test projects --- .github/workflows/push.yml | 12 ++++++++++++ tests/phpunit/Service/DbtServiceTest.php | 9 +++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bc1376e..fe12d6c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -71,6 +71,9 @@ env: DOCKERHUB_TOKEN: '${{ secrets.DOCKERHUB_TOKEN }}' KBC_TEST_PROJECT_URL: 'https://connection.keboola.com/admin/projects/9317' KBC_TEST_PROJECT_CONFIGS: '1192751064' + GCP_KBC_TEST_PROJECT_URL: 'https://connection.europe-west3.gcp.keboola.com/admin/projects/55' + GCP_KBC_TEST_PROJECT_CONFIGS: '49197' + jobs: build: runs-on: ubuntu-latest @@ -216,6 +219,15 @@ jobs: componentId: '${{ env.KBC_DEVELOPERPORTAL_APP }}' tag: '${{ needs.build.outputs.app_image_tag }}-1.8.6' configs: '${{ env.KBC_TEST_PROJECT_CONFIGS }}' + - + name: 'Run KBC test jobs (GCP BQ)' + if: 'env.GCP_KBC_TOKEN && env.GCP_KBC_TEST_PROJECT_CONFIGS' + uses: keboola/action-run-configs-parallel@master + with: + token: '${{ env.GCP_KBC_TOKEN }}' + componentId: 'keboola.dbt-transformation-local-bigquery' + tag: '${{ needs.build.outputs.app_image_tag }}-1.8.6' + configs: '${{ env.GCP_KBC_TEST_PROJECT_CONFIGS }}' deploy: needs: - build diff --git a/tests/phpunit/Service/DbtServiceTest.php b/tests/phpunit/Service/DbtServiceTest.php index a4afa1b..48ed0a6 100644 --- a/tests/phpunit/Service/DbtServiceTest.php +++ b/tests/phpunit/Service/DbtServiceTest.php @@ -165,24 +165,25 @@ public function testDbtCompile(string $backend, string $firstSql, string $second $stringsToFind = [ '/Starting full parse./', - '/Found 2 models, 2 (data )?tests/', + '/Found 2 models/', ]; - $foundedCount = 0; + $foundCount = 0; foreach ($parsedOutput as $line) { foreach ($stringsToFind as $stringToFind) { if (preg_match($stringToFind, $line) === 1) { - $foundedCount++; + $foundCount++; }; } } - if ($foundedCount !== count($stringsToFind)) { + if ($foundCount !== count($stringsToFind)) { self::fail('Not all strings were found in output'); } $compiledSql = DbtCompileHelper::getCompiledSqlFilesContent($this->getProjectPath() . '/target'); $keys = array_keys($compiledSql); + self::assertContains('source_not_null_in.c-test-bucket_test__id_.sql', $keys); self::assertContains('source_unique_in.c-test-bucket_test__id_.sql', $keys); self::assertContains('fct_model.sql', $keys); From 657c18951afed30163d4af5a871e959e871421db Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Tue, 22 Oct 2024 18:35:27 +0200 Subject: [PATCH 08/12] tests (functional): stdout of run-action-with-multiple-execute-steps updated --- .gitignore | 3 +- .../expected-stdout | 36 ++++++++++++++++--- .../source/data/config.json | 9 +++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index f317bbb..d678424 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ +/env /vendor -/.idea .env +/.idea /.phpunit.result.cache diff --git a/tests/functional/run-action-with-multiple-execute-steps/expected-stdout b/tests/functional/run-action-with-multiple-execute-steps/expected-stdout index 6acca7a..8d30699 100644 --- a/tests/functional/run-action-with-multiple-execute-steps/expected-stdout +++ b/tests/functional/run-action-with-multiple-execute-steps/expected-stdout @@ -2,11 +2,15 @@ Successfully cloned repository https://github.com/keboola/dbt-test-project-publi Executing command "dbt deps" Running with dbt=%A Warning: No packages were found in packages.yml -%AExecuting command "dbt build" +Warning: No packages were found in packages.yml +Executing command "dbt build" Running with dbt=%A +Registered adapter: snowflake=%A %a Starting full parse. Found 2%S + %AConcurrency: 1 threads (target='kbc_prod')%A + 1 of 4 START test source_not_null_in.c-test-bucket_test__id_ %s [RUN] 1 of 4 PASS source_not_null_in.c-test-bucket_test__id_ %s [PASS in %ss] 2 of 4 START test source_unique_in.c-test-bucket_test__id_ %s [RUN] @@ -15,40 +19,64 @@ Found 2%S 3 of 4 OK created %Sview model WORKSPACE_%s.stg_model %s [SUCCESS 1 in %ss] 4 of 4 START %Sview model WORKSPACE_%s.fct_model %s [RUN] 4 of 4 OK created %Sview model WORKSPACE_%s.fct_model %s [SUCCESS 1 in %ss] + %AFinished running 2 %a + Completed successfully%A + Done. PASS=4 WARN=0 ERROR=0 SKIP=0 TOTAL=4 Executing command "dbt run" Running with dbt=%A +Registered adapter: snowflake=%A Found 2 models%S + %AConcurrency: 1 threads (target='kbc_prod')%A + 1 of 2 START %Sview model %s.stg_model %s [RUN] 1 of 2 OK created %Sview model %s.stg_model %s [SUCCESS 1 in %ss] 2 of 2 START %Sview model %s.fct_model %s [RUN] 2 of 2 OK created %Sview model %s.fct_model %s [SUCCESS 1 in %ss] + %AFinished running 2 view models in %s hours %s minutes and %s seconds (%ss).%A + Completed successfully%A + Done. PASS=2 WARN=0 ERROR=0 SKIP=0 TOTAL=2 Executing command "dbt docs generate" Running with dbt=%A +Registered adapter: snowflake=%A Found 2 models%S + %AConcurrency: 1 threads (target='kbc_prod')%A + Building catalog -Catalog written to /tmp/%s/dbt-project/target/catalog.json +Catalog written to /tmp/%a/dbt-project/target/catalog.json Executing command "dbt test" Running with dbt=%A +Registered adapter: snowflake=%A Found 2 models%S + %AConcurrency: 1 threads (target='kbc_prod')%A + 1 of 2 START test source_not_null_in.c-test-bucket_test__id_ %s [RUN] 1 of 2 PASS source_not_null_in.c-test-bucket_test__id_ %s [PASS in %ss] 2 of 2 START test source_unique_in.c-test-bucket_test__id_ %s [RUN] 2 of 2 PASS source_unique_in.c-test-bucket_test__id_ %s [PASS in %ss] + %AFinished running 2 %a + Completed successfully%A + Done. PASS=2 WARN=0 ERROR=0 SKIP=0 TOTAL=2 Executing command "dbt source freshness" Running with dbt=%A +Registered adapter: snowflake=%A Found 2 models%S + +Pulling freshness from warehouse metadata tables for 0 sources %AConcurrency: 1 threads (target='kbc_prod')%A -1 of 2 START freshness of in.c-test-bucket.test %s [RUN]%A -%aDone. \ No newline at end of file + +1 of 1 START freshness of in.c-test-bucket.test %s [RUN] + +%AFinished running 1 source %A +Done. diff --git a/tests/functional/run-action-with-multiple-execute-steps/source/data/config.json b/tests/functional/run-action-with-multiple-execute-steps/source/data/config.json index 2d2bdc4..93b77de 100644 --- a/tests/functional/run-action-with-multiple-execute-steps/source/data/config.json +++ b/tests/functional/run-action-with-multiple-execute-steps/source/data/config.json @@ -24,6 +24,15 @@ "freshness": { "warn_after": {"active": true, "count": 1, "period": "day"} } + }, + "storage": { + "input": { + "tables": [ + { + "source": "in.c-test-bucket.test" + } + ] + } } } } From 841ba0b4cf347fbad0af4884214496e74ff3a0a2 Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Tue, 22 Oct 2024 19:30:56 +0200 Subject: [PATCH 09/12] tests (functional): IM added to run-action-with-expired-freshness --- .../source/data/config.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/functional/run-action-with-expired-freshness/source/data/config.json b/tests/functional/run-action-with-expired-freshness/source/data/config.json index 70f4929..86db7aa 100644 --- a/tests/functional/run-action-with-expired-freshness/source/data/config.json +++ b/tests/functional/run-action-with-expired-freshness/source/data/config.json @@ -20,6 +20,15 @@ "warn_after": {"active": true, "count": 1, "period": "minute"}, "error_after": {"active": true, "count": 2, "period": "minute"} } + }, + "storage": { + "input": { + "tables": [ + { + "source": "in.c-test-bucket.test" + } + ] + } } } } From 9417149986415ed916eab31d7426701b7eb92a80 Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Tue, 22 Oct 2024 20:56:07 +0200 Subject: [PATCH 10/12] tests (functional): to run-action-with-expired-freshness stdout update --- .../run-action-with-expired-freshness/expected-stderr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/run-action-with-expired-freshness/expected-stderr b/tests/functional/run-action-with-expired-freshness/expected-stderr index 8af01c8..edddb30 100644 --- a/tests/functional/run-action-with-expired-freshness/expected-stderr +++ b/tests/functional/run-action-with-expired-freshness/expected-stderr @@ -1 +1 @@ -1 of 2 ERROR STALE freshness of in.c-test-bucket.test .......................... [ERROR STALE in %ss] \ No newline at end of file +1 of 1 ERROR STALE freshness of in.c-test-bucket.test .......................... [ERROR STALE in %ss] From 3373bde46a2b132c4699194eb7b112d6c72a37b0 Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Wed, 23 Oct 2024 14:12:16 +0200 Subject: [PATCH 11/12] tests (run-action-with-multiple-execute-steps): fix ups try --- .../expected-stdout | 36 +++---------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/tests/functional/run-action-with-multiple-execute-steps/expected-stdout b/tests/functional/run-action-with-multiple-execute-steps/expected-stdout index 8d30699..b1fe881 100644 --- a/tests/functional/run-action-with-multiple-execute-steps/expected-stdout +++ b/tests/functional/run-action-with-multiple-execute-steps/expected-stdout @@ -2,15 +2,11 @@ Successfully cloned repository https://github.com/keboola/dbt-test-project-publi Executing command "dbt deps" Running with dbt=%A Warning: No packages were found in packages.yml -Warning: No packages were found in packages.yml -Executing command "dbt build" +%AExecuting command "dbt build" Running with dbt=%A -Registered adapter: snowflake=%A %a Starting full parse. Found 2%S - %AConcurrency: 1 threads (target='kbc_prod')%A - 1 of 4 START test source_not_null_in.c-test-bucket_test__id_ %s [RUN] 1 of 4 PASS source_not_null_in.c-test-bucket_test__id_ %s [PASS in %ss] 2 of 4 START test source_unique_in.c-test-bucket_test__id_ %s [RUN] @@ -19,64 +15,40 @@ Found 2%S 3 of 4 OK created %Sview model WORKSPACE_%s.stg_model %s [SUCCESS 1 in %ss] 4 of 4 START %Sview model WORKSPACE_%s.fct_model %s [RUN] 4 of 4 OK created %Sview model WORKSPACE_%s.fct_model %s [SUCCESS 1 in %ss] - %AFinished running 2 %a - Completed successfully%A - Done. PASS=4 WARN=0 ERROR=0 SKIP=0 TOTAL=4 Executing command "dbt run" Running with dbt=%A -Registered adapter: snowflake=%A Found 2 models%S - %AConcurrency: 1 threads (target='kbc_prod')%A - 1 of 2 START %Sview model %s.stg_model %s [RUN] 1 of 2 OK created %Sview model %s.stg_model %s [SUCCESS 1 in %ss] 2 of 2 START %Sview model %s.fct_model %s [RUN] 2 of 2 OK created %Sview model %s.fct_model %s [SUCCESS 1 in %ss] - %AFinished running 2 view models in %s hours %s minutes and %s seconds (%ss).%A - Completed successfully%A - Done. PASS=2 WARN=0 ERROR=0 SKIP=0 TOTAL=2 Executing command "dbt docs generate" Running with dbt=%A -Registered adapter: snowflake=%A Found 2 models%S - %AConcurrency: 1 threads (target='kbc_prod')%A - Building catalog -Catalog written to /tmp/%a/dbt-project/target/catalog.json +Catalog written to /tmp/%s/dbt-project/target/catalog.json Executing command "dbt test" Running with dbt=%A -Registered adapter: snowflake=%A Found 2 models%S - %AConcurrency: 1 threads (target='kbc_prod')%A - 1 of 2 START test source_not_null_in.c-test-bucket_test__id_ %s [RUN] 1 of 2 PASS source_not_null_in.c-test-bucket_test__id_ %s [PASS in %ss] 2 of 2 START test source_unique_in.c-test-bucket_test__id_ %s [RUN] 2 of 2 PASS source_unique_in.c-test-bucket_test__id_ %s [PASS in %ss] - %AFinished running 2 %a - Completed successfully%A - Done. PASS=2 WARN=0 ERROR=0 SKIP=0 TOTAL=2 Executing command "dbt source freshness" Running with dbt=%A -Registered adapter: snowflake=%A Found 2 models%S - -Pulling freshness from warehouse metadata tables for 0 sources %AConcurrency: 1 threads (target='kbc_prod')%A - -1 of 1 START freshness of in.c-test-bucket.test %s [RUN] - -%AFinished running 1 source %A -Done. +1 of 1 START freshness of in.c-test-bucket.test %s [RUN]%A +%ADone. From ef6c03160b09abee270852eb823d0019d27d3ac1 Mon Sep 17 00:00:00 2001 From: MiroCillik Date: Thu, 24 Oct 2024 13:04:27 +0200 Subject: [PATCH 12/12] ci (kbc tests): fix host --- .github/workflows/push.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index fe12d6c..7769ccf 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -224,10 +224,11 @@ jobs: if: 'env.GCP_KBC_TOKEN && env.GCP_KBC_TEST_PROJECT_CONFIGS' uses: keboola/action-run-configs-parallel@master with: - token: '${{ env.GCP_KBC_TOKEN }}' - componentId: 'keboola.dbt-transformation-local-bigquery' - tag: '${{ needs.build.outputs.app_image_tag }}-1.8.6' - configs: '${{ env.GCP_KBC_TEST_PROJECT_CONFIGS }}' + host: '${{ env.GCP_KBC_URL }}' + token: '${{ env.GCP_KBC_TOKEN }}' + componentId: 'keboola.dbt-transformation-local-bigquery' + tag: '${{ needs.build.outputs.app_image_tag }}-1.8.6' + configs: '${{ env.GCP_KBC_TEST_PROJECT_CONFIGS }}' deploy: needs: - build