Merge pull request #71 from keboola/adamvyborny-DBT-16-generate-sourc…

…es-optionaly

Added parameter for not generating the sources and its tests

README.md

@@ -10,7 +10,7 @@ The configuration `config.json` contains following properties in `parameters` ke
     - `#password` - string (optional): GitHub Private Access Token if repository is private. Both or none of couple `username` and `password` must be specified.
     - `branch` - string (optional): Specify git branch if you want to clone project from specific branch.
 - `dbt` - object (required): Configuration of DBT
-    - `executeSteps` - array of strings (required): Enum values of dbt steps you want to run. Available values are `dbt run`, `dbt docs generate`, `dbt test`, `dbt source freshness`, `dbt debug`, `dbt compile` and `dbt seed`. At least one value required.
+    - `executeSteps` - array of strings (required): Enum values of dbt steps you want to run. Available values are `dbt build`, `dbt run`, `dbt docs generate`, `dbt test`, `dbt source freshness`, `dbt debug`, `dbt compile` and `dbt seed`. At least one value required.
     - `modelNames` - array of strings (optional): If you want to run DBT only with certain models, you can specify their names here. Otherwise, all models will be run.
     - `threads` - integer 1 - 8, default 4 (optional): Maximum number of paths through the graph dbt may work on at once.
     - `freshness` - object (required): Configuration of freshness.
@@ -21,6 +21,9 @@ The configuration `config.json` contains following properties in `parameters` ke
         - `period` - positive integer: Number of periods where a data source is still considered "fresh".
         - `count` - string enum: The time period used in the freshness calculation. One of `minute`, `hour` or `day`.
 - `showExecutedSqls` - boolean (optional): Default `false`, if set to `true` SQL queries executed by DBT transformation are printed to output.
+- `generateSources` - boolean (optional): Default `true`
+  - If `true` sources YAML files are generated. 
+  - If `false` generating of sources file is skipped. 
 
 Example:
 ```json
@@ -41,7 +44,8 @@ Example:
       "error_after": {"count": 1, "period": "day"}
     }
   },
-  "showExecutedSqls": true
+  "showExecutedSqls": true,
+  "generateSources": true
 }
 ```
 

src/Config.php

@@ -47,6 +47,11 @@ public function showSqls(): bool
         return $this->getValue(['parameters', 'showExecutedSqls']);
     }
 
+    public function generateSources(): bool
+    {
+        return $this->getValue(['parameters', 'generateSources']);
+    }
+
     /**
      * @return array<string>
      */

src/Configuration/ConfigDefinition.php

@@ -34,6 +34,13 @@ protected function getParametersDefinition(): ArrayNodeDefinition
             ->defaultFalse()
             ->end();
 
+        /** @noinspection NullPointerExceptionInspection */
+        $parametersNode
+            ->children()
+            ->booleanNode('generateSources')
+            ->defaultTrue()
+            ->end();
+
         // @formatter:on
         return $parametersNode;
     }

src/DwhProvider/LocalSnowflakeProvider.php

@@ -48,6 +48,10 @@ public function createDbtYamlFiles(array $configurationNames = []): void
         );
         $this->setEnvVars();
 
+        if (!$this->config->generateSources()) {
+            return;
+        }
+
         $client = new Client([
             'url' => $this->config->getStorageApiUrl(),
             'token' => $this->config->getStorageApiToken(),

tests/functional/run-action-fail-without-sources/expected-code

@@ -0,0 +1 @@
+1

tests/functional/run-action-fail-without-sources/expected-stderr

@@ -0,0 +1 @@
+Encountered an error: Compilation Error in model stg_model (models/example/stg_model.sql)   Model 'model.my_new_project.stg_model' (models/example/stg_model.sql) depends on a source named 'in.c-test-bucket.test' which was not found Traceback (most recent call last):   File "/usr/local/lib/python3.9/dist-packages/dbt/main.py", line 129, in main     results, succeeded = handle_and_check(args)   File "/usr/local/lib/python3.9/dist-packages/dbt/main.py", line 191, in handle_and_check     task, res = run_from_args(parsed)   File "/usr/local/lib/python3.9/dist-packages/dbt/main.py", line 238, in run_from_args     results = task.run()   File "/usr/local/lib/python3.9/dist-packages/dbt/task/runnable.py", line 451, in run     self._runtime_initialize()   File "/usr/local/lib/python3.9/dist-packages/dbt/task/runnable.py", line 159, in _runtime_initialize     super()._runtime_initialize()   File "/usr/local/lib/python3.9/dist-packages/dbt/task/runnable.py", line 92, in _runtime_initialize     self.load_manifest()   File "/usr/local/lib/python3.9/dist-packages/dbt/task/runnable.py", line 81, in load_manifest     self.manifest = ManifestLoader.get_full_manifest(self.config)   File "/usr/local/lib/python3.9/dist-packages/dbt/parser/manifest.py", line 219, in get_full_manifest     manifest = loader.load()   File "/usr/local/lib/python3.9/dist-packages/dbt/parser/manifest.py", line 390, in load     self.process_sources(self.root_project.project_name)   File "/usr/local/lib/python3.9/dist-packages/dbt/parser/manifest.py", line 921, in process_sources     _process_sources_for_node(self.manifest, current_project, node)   File "/usr/local/lib/python3.9/dist-packages/dbt/parser/manifest.py", line 1302, in _process_sources_for_node     invalid_source_fail_unless_test(   File "/usr/local/lib/python3.9/dist-packages/dbt/parser/manifest.py", line 955, in invalid_source_fail_unless_test     source_target_not_found(node, target_name, target_table_name, disabled=disabled)   File "/usr/local/lib/python3.9/dist-packages/dbt/exceptions.py", line 656, in source_target_not_found     raise_compiler_error(msg, model)   File "/usr/local/lib/python3.9/dist-packages/dbt/exceptions.py", line 445, in raise_compiler_error     raise CompilationException(msg, node) dbt.exceptions.CompilationException: Compilation Error in model stg_model (models/example/stg_model.sql)   Model 'model.my_new_project.stg_model' (models/example/stg_model.sql) depends on a source named 'in.c-test-bucket.test' which was not found

tests/functional/run-action-fail-without-sources/source/data/config.json

@@ -0,0 +1,22 @@
+{
+  "authorization": {
+    "workspace": {
+      "host": "%env(string:SNOWFLAKE_HOST)%",
+      "warehouse": "%env(string:SNOWFLAKE_WAREHOUSE)%",
+      "database": "%env(string:SNOWFLAKE_DATABASE)%",
+      "schema": "%env(string:SNOWFLAKE_SCHEMA)%",
+      "user": "%env(string:SNOWFLAKE_USER)%",
+      "password": "%env(string:SNOWFLAKE_PASSWORD)%"
+    }
+  },
+  "parameters": {
+    "git": {
+      "repo": "https://github.com/keboola/dbt-test-project-public.git"
+    },
+    "dbt": {
+      "executeSteps": ["dbt run"],
+      "threads": 1
+    },
+    "generateSources": false
+  }
+}

tests/functional/run-action-without-generating-sources/expected-code

@@ -0,0 +1 @@
+0

tests/functional/run-action-without-generating-sources/expected-stdout

@@ -0,0 +1,16 @@
+Successfully cloned repository https://github.com/keboola/dbt-test-project-public.git from branch branch-with-sources (%s)
+Executing command "dbt deps"
+Running with dbt=%s
+Warning: No packages were found in packages.yml
+Executing command "dbt run"
+Running with dbt=%s
+Partial parse save file not found. Starting full parse.
+Found 2 models, 0 tests, 0 snapshots, 0 analyses, %s macros, 0 operations, 0 seed files, 1 source, 0 exposures, 0 metrics
+Concurrency: 1 threads (target='kbc_prod')
+1 of 2 START view model %s.stg_model %s [RUN]
+1 of 2 OK created view model %s.stg_model %s [SUCCESS 1 in %ss]
+2 of 2 START view model %s.fct_model %s [RUN]
+2 of 2 OK created view model %s.fct_model %s [SUCCESS 1 in %ss]
+Finished running 2 view models in %s hours %s minutes and %s seconds (%ss).
+Completed successfully
+Done. PASS=2 WARN=0 ERROR=0 SKIP=0 TOTAL=2

tests/functional/run-action-without-generating-sources/source/data/config.json

@@ -0,0 +1,23 @@
+{
+  "authorization": {
+    "workspace": {
+      "host": "%env(string:SNOWFLAKE_HOST)%",
+      "warehouse": "%env(string:SNOWFLAKE_WAREHOUSE)%",
+      "database": "%env(string:SNOWFLAKE_DATABASE)%",
+      "schema": "%env(string:SNOWFLAKE_SCHEMA)%",
+      "user": "%env(string:SNOWFLAKE_USER)%",
+      "password": "%env(string:SNOWFLAKE_PASSWORD)%"
+    }
+  },
+  "parameters": {
+    "git": {
+      "repo": "https://github.com/keboola/dbt-test-project-public.git",
+      "branch": "branch-with-sources"
+    },
+    "dbt": {
+      "executeSteps": ["dbt run"],
+      "threads": 1
+    },
+    "generateSources": false
+  }
+}

tests/phpunit/ConfigDefinition/ConfigDefinitionTest.php

@@ -333,6 +333,21 @@ public function validConfigsData(): Generator
                 ],
             ],
         ];
+
+        yield 'config without generating sources' => [
+            'configData' => [
+                'action' => 'run',
+                'parameters' => [
+                    'git' => [
+                        'repo' => 'https://github.com/my-repo',
+                    ],
+                    'dbt' => [
+                        'executeSteps' => ['dbt run'],
+                    ],
+                    'generateSources' => false,
+                ],
+            ],
+        ];
     }
 
     /**
@@ -699,6 +714,10 @@ protected function addDefaultValues(array $configData): array
             $configData['parameters']['remoteDwh']['threads'] = 4;
         }
 
+        if (!array_key_exists('generateSources', $configData['parameters'])) {
+            $configData['parameters']['generateSources'] = true;
+        }
+
         return $configData;
     }
 }