From 2c3f9eaf3d7d1ccd7cb9d308802d76accbe845ea Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Thu, 29 Jun 2023 22:45:30 +0100 Subject: [PATCH 01/14] dbt run successful --- dbt_project.yml | 3 ++- integration_test_project/dbt_project.yml | 4 ++-- macros/generate_surrogate_key.sql | 15 ++++++++++++++- macros/type_helpers.sql | 8 ++++++++ models/dim_dbt__snapshots.sql | 4 ++-- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 24da069d..073e7896 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -19,4 +19,5 @@ models: +full_refresh: false +persist_docs: # Databricks doesn't offer column-level support for persisting docs - columns: '{{ target.name != "databricks" }}' + columns: '{{ target.name not in ["databricks", "dremio"] }}' + relation: '{{ target.name not in ["dremio"] }}' diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index a6550009..baca7fc6 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -24,8 +24,8 @@ vars: models: +persist_docs: - relation: true - columns: true + columns: '{{ target.name != "dremio" }}' + relation: '{{ target.name != "dremio" }}' seeds: +quote_columns: false diff --git a/macros/generate_surrogate_key.sql b/macros/generate_surrogate_key.sql index 9bcc2686..7eec3ec7 100644 --- a/macros/generate_surrogate_key.sql +++ b/macros/generate_surrogate_key.sql @@ -37,6 +37,19 @@ {%- endfor -%} -{{ dbt.hash(dbt.concat(fields)) }} +{{ hash(dbt.concat(fields)) }} {%- endmacro -%} + +{%- macro hash(field) -%} + {{ adapter.dispatch('hash', 'dbt_artifacts')(field) }} +{%- endmacro -%} + +{%- macro default__hash(field) -%} + {{ dbt.hash(field) }} +{%- endmacro -%} + +-- FIXME: See https://github.com/dremio/dbt-dremio/issues/189 +{%- macro dremio__hash(field) -%} + md5(cast({{ field }} as varchar)) +{%- endmacro -%} diff --git a/macros/type_helpers.sql b/macros/type_helpers.sql index 19c3a718..3dd59ae2 100644 --- a/macros/type_helpers.sql +++ b/macros/type_helpers.sql @@ -26,6 +26,10 @@ JSON {% endmacro %} +{%- macro dremio__type_json() -%} + VARCHAR +{%- endmacro -%} + {#- ARRAY -#} {% macro type_array() %} @@ -43,3 +47,7 @@ {% macro bigquery__type_array() %} ARRAY {% endmacro %} + +{%- macro dremio__type_array() -%} + VARCHAR +{%- endmacro -%} diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql index 13f05d0e..3cf319bb 100644 --- a/models/dim_dbt__snapshots.sql +++ b/models/dim_dbt__snapshots.sql @@ -5,7 +5,7 @@ with base as ( ), -snapshots as ( +"snapshots" as ( select snapshot_execution_id, @@ -26,4 +26,4 @@ snapshots as ( ) -select * from snapshots +select * from "snapshots" From 25f69675235c28abe7ac799b6cf874679ada264b Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Fri, 30 Jun 2023 17:07:29 +0100 Subject: [PATCH 02/14] on-run-end hooks working --- macros/insert_into_metadata_table.sql | 9 ++++ macros/string.sql | 11 +++++ macros/timestamp.sql | 12 ++++++ macros/upload_exposures.sql | 31 ++++++++++++++ macros/upload_invocations.sql | 55 +++++++++++++++++++++++++ macros/upload_model_executions.sql | 59 +++++++++++++++++++++++++++ macros/upload_models.sql | 33 +++++++++++++++ macros/upload_seed_executions.sql | 58 ++++++++++++++++++++++++++ macros/upload_seeds.sql | 29 +++++++++++++ macros/upload_snapshot_executions.sql | 58 ++++++++++++++++++++++++++ macros/upload_snapshots.sql | 31 ++++++++++++++ macros/upload_sources.sql | 41 ++++++++++++++++--- macros/upload_test_executions.sql | 55 +++++++++++++++++++++++++ macros/upload_tests.sql | 27 ++++++++++++ 14 files changed, 503 insertions(+), 6 deletions(-) create mode 100644 macros/string.sql create mode 100644 macros/timestamp.sql diff --git a/macros/insert_into_metadata_table.sql b/macros/insert_into_metadata_table.sql index 235d2326..9faa0787 100644 --- a/macros/insert_into_metadata_table.sql +++ b/macros/insert_into_metadata_table.sql @@ -34,5 +34,14 @@ {%- endmacro %} +{% macro dremio__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} + {% set insert_into_table_query %} + insert into "{{database_name}}"."{{ schema_name }}"."{{ table_name }}" + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} +{%- endmacro %} + {% macro default__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} {%- endmacro %} diff --git a/macros/string.sql b/macros/string.sql new file mode 100644 index 00000000..d950e8d2 --- /dev/null +++ b/macros/string.sql @@ -0,0 +1,11 @@ +{%- macro escape_string(field) -%} + {{ return(adapter.dispatch('escape_string', 'dbt_artifacts')(field)) }} +{%- endmacro -%} + +{%- macro default__escape_string(field) -%} + {{ field | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }} +{%- endmacro -%} + +{%- macro dremio__escape_string(field) -%} + {{ field | replace("'", "''") }} +{%- endmacro -%} diff --git a/macros/timestamp.sql b/macros/timestamp.sql new file mode 100644 index 00000000..d0812322 --- /dev/null +++ b/macros/timestamp.sql @@ -0,0 +1,12 @@ +{%- macro truncate_timestamp(field) -%} + {{ return(adapter.dispatch('truncate_timestamp', 'dbt_artifacts')(field)) }} +{%- endmacro -%} + +{%- macro default__truncate_timestamp(field) -%} + {{ field }} +{%- endmacro -%} + +{%- macro dremio__truncate_timestamp(field) -%} + {%- set pattern = '^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{0,3})?)\d*(\+\d{2}:\d{2}|\-\d{2}:\d{2}|Z|[A-Z]{3}|)$' -%} + concat(regexp_extract('{{ field }}', '{{ pattern }}', 1), regexp_extract('{{ field }}', '{{ pattern }}', 3)) +{%- endmacro -%} diff --git a/macros/upload_exposures.sql b/macros/upload_exposures.sql index daa86fea..f68e8f11 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_exposures.sql @@ -80,3 +80,34 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro dremio__get_exposures_dml_sql(exposures) -%} + + {% if exposures != [] %} + {% set exposure_values %} + values + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_artifacts.escape_string(exposure.unique_id) }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + '{{ dbt_artifacts.escape_string(exposure.name) }}', {# name #} + '{{ exposure.type }}', {# type #} + '{{ tojson(exposure.owner) }}', {# owner #} + '{{ exposure.maturity }}', {# maturity #} + '{{ dbt_artifacts.escape_string(exposure.original_file_path) }}', {# path #} + '{{ dbt_artifacts.escape_string(exposure.description) }}', {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + '{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ tojson(exposure.tags) }}', {# tags #} + '{{ dbt_artifacts.escape_string(tojson(exposure)) }}' {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ exposure_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_invocations.sql b/macros/upload_invocations.sql index b1d6e7bc..d9e0b087 100644 --- a/macros/upload_invocations.sql +++ b/macros/upload_invocations.sql @@ -159,3 +159,58 @@ {{ invocation_values }} {% endmacro -%} + +{% macro dremio__get_invocations_dml_sql() -%} + {% set invocation_values %} + values + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + '{{ flags.FULL_REFRESH }}', {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + nullif('{{ env_var('DBT_CLOUD_PROJECT_ID', '') }}', ''), {# dbt_cloud_project_id #} + nullif('{{ env_var('DBT_CLOUD_JOB_ID', '') }}', ''), {# dbt_cloud_job_id #} + nullif('{{ env_var('DBT_CLOUD_RUN_ID', '') }}', ''), {# dbt_cloud_run_id #} + nullif('{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', ''), {# dbt_cloud_run_reason_category #} + nullif('{{ dbt_artifacts.escape_string(env_var('DBT_CLOUD_RUN_REASON', '')) }}', ''), {# dbt_cloud_run_reason #} + + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, '') | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(env_vars_dict) }}', {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} + + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: (dbt_artifacts.escape_string(var(dbt_var, '')))}) %} + {% endfor %} + '{{ tojson(dbt_vars_dict) }}', {# dbt_vars #} + {% else %} + null, {# dbt_vars #} + {% endif %} + + '{{ dbt_artifacts.escape_string(tojson(invocation_args_dict)) }}', {# invocation_args #} + + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: (dbt_artifacts.escape_string(value))}) %} + {% endfor %} + '{{ dbt_artifacts.escape_string(tojson(metadata_env)) }}' {# dbt_custom_envs #} + + ) + {% endset %} + {{ invocation_values }} + +{% endmacro -%} diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index 14917d1a..db6a3a71 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -216,3 +216,62 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro dremio__get_model_executions_dml_sql(models) -%} + {% if models != [] %} + {% set model_execution_values %} + values + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + {{ dbt_artifacts.truncate_timestamp(stage.started_at) }}, {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + {{ dbt_artifacts.truncate_timestamp(stage.completed_at) }}, {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ dbt_artifacts.escape_string(model.message) }}', {# message #} + '{{ dbt_artifacts.escape_string(tojson(model.adapter_response)) }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} + diff --git a/macros/upload_models.sql b/macros/upload_models.sql index b8fce69e..05ed51dd 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -81,3 +81,36 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro dremio__get_models_dml_sql(models) -%} + + {% if models != [] %} + {% set model_values %} + values + {% for model in models -%} + {% do model.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.unique_id }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + '{{ model.database }}', {# database #} + '{{ model.schema }}', {# schema #} + '{{ model.name }}', {# name #} + '{{ dbt_artifacts.escape_string(tojson(model.depends_on.nodes)) }}', {# depends_on_nodes #} + '{{ model.package_name }}', {# package_name #} + '{{ dbt_artifacts.escape_string(model.original_file_path) }}', {# path #} + '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.config.materialized }}', {# materialization #} + '{{ tojson(model.tags) }}', {# tags #} + '{{ dbt_artifacts.escape_string(tojson(model.config.meta)) }}', {# meta #} + '{{ model.alias }}', {# alias #} + '{{ dbt_artifacts.escape_string(tojson(model)) }}' {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index a7ada664..9dbcdf54 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -214,3 +214,61 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro dremio__get_seed_executions_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_execution_values %} + values + {% for model in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }},, {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + {{ dbt_artifacts.truncate_timestamp(stage.started_at) }}, {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + {{ dbt_artifacts.truncate_timestamp(stage.completed_at) }}, {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ dbt_artifacts.escape_string(model.message) }}', {# message #} + '{{ dbt_artifacts.escape_string(tojson(model.adapter_response)) }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ seed_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index 61683f75..ca7169ba 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -74,3 +74,32 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro dremio__get_seeds_dml_sql(seeds) -%} + + {% if seeds != [] %} + {% set seed_values %} + values + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ dbt_artifacts.escape_string(seed.original_file_path) }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + '{{ dbt_artifacts.escape_string(tojson(seed.config.meta)) }}', {# meta #} + '{{ seed.alias }}', {# alias #} + '{{ dbt_artifacts.escape_string(tojson(seed)) }}' {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ seed_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index 4903f48e..3785bccb 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -214,3 +214,61 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro dremio__get_snapshot_executions_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_execution_values %} + values + {% for model in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + {{ dbt_artifacts.truncate_timestamp(stage.started_at) }}, {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + {{ dbt_artifacts.truncate_timestamp(stage.completed_at) }}, {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ dbt_artifacts.escape_string(model.message) }}', {# message #} + '{{ dbt_artifacts.escape_string(tojson(model.adapter_response)) }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ snapshot_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index cac08abc..6e555321 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -81,3 +81,34 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro dremio__get_snapshots_dml_sql(snapshots) -%} + + {% if snapshots != [] %} + {% set snapshot_values %} + values + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + '{{ tojson(snapshot.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ dbt_artifacts.escape_string(snapshot.original_file_path) }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + '{{ dbt_artifacts.escape_string(tojson(snapshot.config.meta)) }}', {# meta #} + '{{ snapshot.alias }}', {# alias #} + '{{ dbt_artifacts.escape_string(tojson(snapshot)) }}' {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ snapshot_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index a624ea4b..3d20bcd1 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -31,9 +31,9 @@ '{{ source.loader }}', {# loader #} '{{ source.name }}', {# name #} '{{ source.identifier }}', {# identifier #} - '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} - '{{ tojson(source.freshness) | replace("'","\\'") }}', {# freshness #} - '{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} + '{{ dbt_artifacts.escape_string(source.loaded_at_field) }}', {# loaded_at_field #} + '{{ dbt_artifacts.escape_string(tojson(source.freshness)) }}', {# freshness #} + '{{ dbt_artifacts.escape_string(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'")) }}' {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -58,9 +58,9 @@ '{{ source.loader }}', {# loader #} '{{ source.name }}', {# name #} '{{ source.identifier }}', {# identifier #} - '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} - parse_json('{{ tojson(source.freshness) | replace("'","\\'") }}'), {# freshness #} - parse_json('{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# all_results #} + '{{ dbt_artifacts.escape_string(source.loaded_at_field) }}', {# loaded_at_field #} + parse_json('{{ dbt_artifacts.escape_string(tojson(source.freshness)) }}'), {# freshness #} + parse_json('{{ dbt_artifacts.escape_string(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'")) }}') {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -70,3 +70,32 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro dremio__get_sources_dml_sql(sources) -%} + + {% if sources != [] %} + {% set source_values %} + values + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + '{{ dbt_artifacts.escape_string(source.loaded_at_field) }}', {# loaded_at_field #} + '{{ dbt_artifacts.escape_string(tojson(source.freshness)) }}', {# freshness #} + '{{ dbt_artifacts.escape_string(tojson(source)) }}' {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ source_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_test_executions.sql b/macros/upload_test_executions.sql index 1f0f43d7..dd8addb9 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_test_executions.sql @@ -131,3 +131,58 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro dremio__get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_execution_values %} + values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% if test.timing != [] %} + {% for stage in test.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + {{ dbt_artifacts.truncate_timestamp(stage.started_at) }}, {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in test.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + {{ dbt_artifacts.truncate_timestamp(stage.completed_at) }}, {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + cast({{ test.execution_time }} as float), {# total_node_runtime #} + null, {# rows_affected not available in Databricks #} + {{ 'null' if test.failures is none else test.failures }}, {# failures #} + '{{ dbt_artifacts.escape_string(test.message) }}', {# message #} + '{{ dbt_artifacts.escape_string(tojson(test.adapter_response)) }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ test_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index 5d7c268d..20c28333 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -61,3 +61,30 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro dremio__get_tests_dml_sql(tests) -%} + + {% if tests != [] %} + {% set test_values %} + values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + '{{ test.name }}', {# name #} + '{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ dbt_artifacts.escape_string(test.original_file_path) }}', {# test_path #} + '{{ tojson(test.tags) }}', {# tags #} + '{{ dbt_artifacts.escape_string(tojson(test)) }}' {# all_fields #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ test_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} + From 0f2ccca8b83717b1256d3c01a8752b82b13e23cf Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Fri, 30 Jun 2023 22:42:06 +0100 Subject: [PATCH 03/14] cast model_execution_time to float --- macros/upload_model_executions.sql | 2 +- macros/upload_seed_executions.sql | 2 +- macros/upload_snapshot_executions.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index db6a3a71..36c93451 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -257,7 +257,7 @@ null, {# query_completed_at #} {% endif %} - {{ model.execution_time }}, {# total_node_runtime #} + cast({{ model.execution_time }} as float), {# total_node_runtime #} null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index 9dbcdf54..f1c323df 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -255,7 +255,7 @@ null, {# query_completed_at #} {% endif %} - {{ model.execution_time }}, {# total_node_runtime #} + cast({{ model.execution_time }} as float), {# total_node_runtime #} null, -- rows_affected not available {# Only available in Snowflake #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index 3785bccb..3e18c25d 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -255,7 +255,7 @@ null, {# query_completed_at #} {% endif %} - {{ model.execution_time }}, {# total_node_runtime #} + cast({{ model.execution_time }} as float), {# total_node_runtime #} null, -- rows_affected not available {# Only available in Snowflake #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} From a9a936e9193f9044325aed9f8563855648a80644 Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Sat, 1 Jul 2023 16:20:03 +0100 Subject: [PATCH 04/14] Fix upload_seed_executions.sql and add configuration to tox.ini --- README.md | 1 + integration_test_project/example-env.sh | 4 ++++ integration_test_project/profiles.yml | 19 +++++++++++++++---- macros/upload_seed_executions.sql | 2 +- tox.ini | 7 +++++++ 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 94853de2..b49b0641 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The package currently supports - Spark :white_check_mark: - Snowflake :white_check_mark: - Google BigQuery :white_check_mark: +- Dremio :white_check_mark: Models included: diff --git a/integration_test_project/example-env.sh b/integration_test_project/example-env.sh index 51450cf3..61d72070 100755 --- a/integration_test_project/example-env.sh +++ b/integration_test_project/example-env.sh @@ -16,6 +16,10 @@ export DBT_ENV_SECRET_DATABRICKS_TOKEN= export DBT_ENV_SECRET_GCP_PROJECT= export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dylib on a Mac export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path +export DBT_ENV_DREMIO_USER= +export DBT_ENV_DREMIO_PASSWORD= +export DBT_ENV_DREMIO_SOFTWARE_HOST= +export DBT_ENV_OBJECT_STORAGE_SOURCE= # dbt environment variables, change these export DBT_CLOUD_PROJECT_ID= diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index e8548e3f..314ee2d5 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -16,11 +16,11 @@ dbt_artifacts: role: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE') }}" database: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE') }}" warehouse: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE') }}" - schema: dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} + schema: &schema dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} threads: 8 databricks: type: databricks - schema: dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} + schema: *schema host: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_HOST') }}" http_path: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_HTTP_PATH') }}" token: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_TOKEN') }}" @@ -28,7 +28,7 @@ dbt_artifacts: spark: type: spark method: odbc - schema: dbt_artifacts_test_commit_spark_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} + schema: *schema host: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_HOST') }}" driver: "{{ env_var('DBT_ENV_SPARK_DRIVER_PATH') }}" endpoint: "{{ env_var('DBT_ENV_SPARK_ENDPOINT') }}" @@ -38,8 +38,19 @@ dbt_artifacts: type: bigquery method: oauth project: "{{ env_var('DBT_ENV_SECRET_GCP_PROJECT') }}" - dataset: dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} + dataset: *schema threads: 8 timeout_seconds: 300 priority: interactive retries: 1 + dremio: + type: dremio + user: "{{ env_var('DBT_ENV_DREMIO_USER') }}" + password: "{{ env_var('DBT_ENV_DREMIO_PASSWORD') }}" + software_host: "{{ env_var('DBT_ENV_DREMIO_SOFTWARE_HOST') }}" + port: 443 + threads: 8 + use_ssl: true + dremio_space: *schema + object_storage_path: *schema + object_storage_source: "{{ env_var('DBT_ENV_OBJECT_STORAGE_SOURCE') }}" diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index f1c323df..c73ad1dd 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -223,7 +223,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.node.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }},, {# run_started_at #} + {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} {% set config_full_refresh = model.node.config.full_refresh %} {% if config_full_refresh is none %} diff --git a/tox.ini b/tox.ini index f311d76d..929dcb82 100644 --- a/tox.ini +++ b/tox.ini @@ -226,3 +226,10 @@ commands = dbt deps dbt build --exclude snapshot --target spark +[testenv:integration_dremio] +changedir = integration_test_project +deps = dbt-dremio~=1.5.0 +commands = + dbt clean + dbt deps + dbt build --exclude snapshot --target dremio From 3810624c0ed751b0c325c1db3c179c45088c18bc Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Mon, 3 Jul 2023 13:34:05 +0100 Subject: [PATCH 05/14] Fix concatenation bug when timestamp column is null --- macros/timestamp.sql | 14 +++++++++----- macros/upload_exposures.sql | 2 +- macros/upload_invocations.sql | 2 +- macros/upload_model_executions.sql | 14 +++++++------- macros/upload_models.sql | 2 +- macros/upload_seed_executions.sql | 14 +++++++------- macros/upload_seeds.sql | 2 +- macros/upload_snapshot_executions.sql | 14 +++++++------- macros/upload_snapshots.sql | 2 +- macros/upload_sources.sql | 2 +- macros/upload_test_executions.sql | 14 +++++++------- macros/upload_tests.sql | 2 +- 12 files changed, 44 insertions(+), 40 deletions(-) diff --git a/macros/timestamp.sql b/macros/timestamp.sql index d0812322..58612506 100644 --- a/macros/timestamp.sql +++ b/macros/timestamp.sql @@ -1,12 +1,16 @@ -{%- macro truncate_timestamp(field) -%} - {{ return(adapter.dispatch('truncate_timestamp', 'dbt_artifacts')(field)) }} +{%- macro cast_as_timestamp(field) -%} + {{ return(adapter.dispatch('cast_as_timestamp', 'dbt_artifacts')(field)) }} +{%- endmacro -%} + +{%- macro default__cast_as_timestamp(field) -%} + cast({{ field }} as timestamp) {%- endmacro -%} -{%- macro default__truncate_timestamp(field) -%} - {{ field }} +{%- macro dremio__cast_as_timestamp(field) -%} + cast({{ dbt_artifacts.truncate_timestamp(field) }} as timestamp) {%- endmacro -%} -{%- macro dremio__truncate_timestamp(field) -%} +{%- macro truncate_timestamp(field) -%} {%- set pattern = '^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{0,3})?)\d*(\+\d{2}:\d{2}|\-\d{2}:\d{2}|Z|[A-Z]{3}|)$' -%} concat(regexp_extract('{{ field }}', '{{ pattern }}', 1), regexp_extract('{{ field }}', '{{ pattern }}', 3)) {%- endmacro -%} diff --git a/macros/upload_exposures.sql b/macros/upload_exposures.sql index f68e8f11..5cf357de 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_exposures.sql @@ -90,7 +90,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ dbt_artifacts.escape_string(exposure.unique_id) }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} '{{ dbt_artifacts.escape_string(exposure.name) }}', {# name #} '{{ exposure.type }}', {# type #} '{{ tojson(exposure.owner) }}', {# owner #} diff --git a/macros/upload_invocations.sql b/macros/upload_invocations.sql index d9e0b087..000ebdd6 100644 --- a/macros/upload_invocations.sql +++ b/macros/upload_invocations.sql @@ -167,7 +167,7 @@ '{{ invocation_id }}', {# command_invocation_id #} '{{ dbt_version }}', {# dbt_version #} '{{ project_name }}', {# project_name #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} '{{ flags.WHICH }}', {# dbt_command #} '{{ flags.FULL_REFRESH }}', {# full_refresh_flag #} '{{ target.profile_name }}', {# target_profile_name #} diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index 36c93451..4af31c1a 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -225,7 +225,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.node.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} {% set config_full_refresh = model.node.config.full_refresh %} {% if config_full_refresh is none %} @@ -239,22 +239,22 @@ {% if model.timing != [] %} {% for stage in model.timing if stage.name == "compile" %} {% if loop.length == 0 %} - null, {# compile_started_at #} + cast(null as timestamp), {# compile_started_at #} {% else %} - {{ dbt_artifacts.truncate_timestamp(stage.started_at) }}, {# compile_started_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} {% endif %} {% endfor %} {% for stage in model.timing if stage.name == "execute" %} {% if loop.length == 0 %} - null, {# query_completed_at #} + cast(null as timestamp), {# query_completed_at #} {% else %} - {{ dbt_artifacts.truncate_timestamp(stage.completed_at) }}, {# query_completed_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} {% endif %} {% endfor %} {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} + cast(null as timestamp), {# compile_started_at #} + cast(null as timestamp), {# query_completed_at #} {% endif %} cast({{ model.execution_time }} as float), {# total_node_runtime #} diff --git a/macros/upload_models.sql b/macros/upload_models.sql index 05ed51dd..91f34a5d 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -92,7 +92,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} '{{ model.database }}', {# database #} '{{ model.schema }}', {# schema #} '{{ model.name }}', {# name #} diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index c73ad1dd..1d64490c 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -223,7 +223,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.node.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} {% set config_full_refresh = model.node.config.full_refresh %} {% if config_full_refresh is none %} @@ -237,22 +237,22 @@ {% if model.timing != [] %} {% for stage in model.timing if stage.name == "compile" %} {% if loop.length == 0 %} - null, {# compile_started_at #} + cast(null as timestamp), {# compile_started_at #} {% else %} - {{ dbt_artifacts.truncate_timestamp(stage.started_at) }}, {# compile_started_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} {% endif %} {% endfor %} {% for stage in model.timing if stage.name == "execute" %} {% if loop.length == 0 %} - null, {# query_completed_at #} + cast(null as timestamp), {# query_completed_at #} {% else %} - {{ dbt_artifacts.truncate_timestamp(stage.completed_at) }}, {# query_completed_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} {% endif %} {% endfor %} {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} + cast(null as timestamp), {# compile_started_at #} + cast(null as timestamp), {# query_completed_at #} {% endif %} cast({{ model.execution_time }} as float), {# total_node_runtime #} diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index ca7169ba..966d8204 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -84,7 +84,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ seed.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} '{{ seed.database }}', {# database #} '{{ seed.schema }}', {# schema #} '{{ seed.name }}', {# name #} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index 3e18c25d..4e1b749b 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -223,7 +223,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.node.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} {% set config_full_refresh = model.node.config.full_refresh %} {% if config_full_refresh is none %} @@ -237,22 +237,22 @@ {% if model.timing != [] %} {% for stage in model.timing if stage.name == "compile" %} {% if loop.length == 0 %} - null, {# compile_started_at #} + cast(null as timestamp), {# compile_started_at #} {% else %} - {{ dbt_artifacts.truncate_timestamp(stage.started_at) }}, {# compile_started_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} {% endif %} {% endfor %} {% for stage in model.timing if stage.name == "execute" %} {% if loop.length == 0 %} - null, {# query_completed_at #} + cast(null as timestamp), {# query_completed_at #} {% else %} - {{ dbt_artifacts.truncate_timestamp(stage.completed_at) }}, {# query_completed_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} {% endif %} {% endfor %} {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} + cast(null as timestamp), {# compile_started_at #} + cast(null as timestamp), {# query_completed_at #} {% endif %} cast({{ model.execution_time }} as float), {# total_node_runtime #} diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index 6e555321..91d01b98 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -91,7 +91,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ snapshot.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} '{{ snapshot.database }}', {# database #} '{{ snapshot.schema }}', {# schema #} '{{ snapshot.name }}', {# name #} diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index 3d20bcd1..db4a4ee7 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -80,7 +80,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ source.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} '{{ source.database }}', {# database #} '{{ source.schema }}', {# schema #} '{{ source.source_name }}', {# source_name #} diff --git a/macros/upload_test_executions.sql b/macros/upload_test_executions.sql index dd8addb9..ea75b8f9 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_test_executions.sql @@ -140,7 +140,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ test.node.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} {% set config_full_refresh = test.node.config.full_refresh %} {% if config_full_refresh is none %} @@ -154,22 +154,22 @@ {% if test.timing != [] %} {% for stage in test.timing if stage.name == "compile" %} {% if loop.length == 0 %} - null, {# compile_started_at #} + cast(null as timestamp), {# compile_started_at #} {% else %} - {{ dbt_artifacts.truncate_timestamp(stage.started_at) }}, {# compile_started_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} {% endif %} {% endfor %} {% for stage in test.timing if stage.name == "execute" %} {% if loop.length == 0 %} - null, {# query_completed_at #} + cast(null as timestamp), {# query_completed_at #} {% else %} - {{ dbt_artifacts.truncate_timestamp(stage.completed_at) }}, {# query_completed_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} {% endif %} {% endfor %} {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} + cast(null as timestamp), {# compile_started_at #} + cast(null as timestamp), {# query_completed_at #} {% endif %} cast({{ test.execution_time }} as float), {# total_node_runtime #} diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index 20c28333..cce3e3db 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -71,7 +71,7 @@ ( '{{ invocation_id }}', {# command_invocation_id #} '{{ test.unique_id }}', {# node_id #} - {{ dbt_artifacts.truncate_timestamp(run_started_at) }}, {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} '{{ test.name }}', {# name #} '{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #} '{{ test.package_name }}', {# package_name #} From d9d9e2f2636b3d07a08cc99d3bb148c6466e92d3 Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Tue, 25 Jul 2023 10:21:35 +0100 Subject: [PATCH 06/14] Fix regression --- macros/upload_sources.sql | 12 ++++++------ models/dim_dbt__snapshots.sql | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index db4a4ee7..5fc17d3a 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -31,9 +31,9 @@ '{{ source.loader }}', {# loader #} '{{ source.name }}', {# name #} '{{ source.identifier }}', {# identifier #} - '{{ dbt_artifacts.escape_string(source.loaded_at_field) }}', {# loaded_at_field #} - '{{ dbt_artifacts.escape_string(tojson(source.freshness)) }}', {# freshness #} - '{{ dbt_artifacts.escape_string(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'")) }}' {# all_results #} + '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} + '{{ tojson(source.freshness) | replace("'","\\'") }}', {# freshness #} + '{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -58,9 +58,9 @@ '{{ source.loader }}', {# loader #} '{{ source.name }}', {# name #} '{{ source.identifier }}', {# identifier #} - '{{ dbt_artifacts.escape_string(source.loaded_at_field) }}', {# loaded_at_field #} - parse_json('{{ dbt_artifacts.escape_string(tojson(source.freshness)) }}'), {# freshness #} - parse_json('{{ dbt_artifacts.escape_string(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'")) }}') {# all_results #} + '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} + parse_json('{{ tojson(source.freshness) | replace("'","\\'") }}'), {# freshness #} + parse_json('{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql index 3cf319bb..24b31659 100644 --- a/models/dim_dbt__snapshots.sql +++ b/models/dim_dbt__snapshots.sql @@ -5,7 +5,7 @@ with base as ( ), -"snapshots" as ( +dbt_snapshots as ( select snapshot_execution_id, @@ -26,4 +26,4 @@ with base as ( ) -select * from "snapshots" +select * from dbt_snapshots From 8fac6e28a9485143858f5dbe989a64854cfb9858 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 29 Sep 2023 16:20:34 +0100 Subject: [PATCH 07/14] Update to reflect recent changes --- .../upload_exposures.sql | 65 +++++----- .../upload_invocations.sql | 1 - .../upload_model_executions.sql | 109 ++++++++-------- .../upload_models.sql | 69 +++++----- .../upload_seed_executions.sql | 121 +++++++++--------- .../upload_seeds.sql | 61 ++++----- .../upload_snapshot_executions.sql | 86 ++++++------- .../upload_snapshots.sql | 64 ++++----- .../upload_sources.sql | 60 ++++----- .../upload_test_executions.sql | 51 ++++---- .../upload_tests.sql | 55 ++++---- .../insert_into_metadata_table.sql | 3 +- 12 files changed, 379 insertions(+), 366 deletions(-) diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql index de77ffab..ea2bd767 100644 --- a/macros/upload_individual_datasets/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -85,6 +85,40 @@ {% endif %} {%- endmacro %} +{% macro dremio__get_exposures_dml_sql(exposures) -%} + + {% if exposures != [] %} + {% set exposure_values %} + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_artifacts.escape_string(exposure.unique_id) }}', {# node_id #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ dbt_artifacts.escape_string(exposure.name) }}', {# name #} + '{{ exposure.type }}', {# type #} + '{{ tojson(exposure.owner) }}', {# owner #} + '{{ exposure.maturity }}', {# maturity #} + '{{ dbt_artifacts.escape_string(exposure.original_file_path) }}', {# path #} + '{{ dbt_artifacts.escape_string(exposure.description) }}', {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + '{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ tojson(exposure.tags) }}', {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ dbt_artifacts.escape_string(tojson(exposure)) }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ exposure_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} + {% macro postgres__get_exposures_dml_sql(exposures) -%} {% if exposures != [] %} @@ -118,34 +152,3 @@ {{ return("") }} {% endif %} {%- endmacro %} - -{% macro dremio__get_exposures_dml_sql(exposures) -%} - - {% if exposures != [] %} - {% set exposure_values %} - values - {% for exposure in exposures -%} - ( - '{{ invocation_id }}', {# command_invocation_id #} - '{{ dbt_artifacts.escape_string(exposure.unique_id) }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} - '{{ dbt_artifacts.escape_string(exposure.name) }}', {# name #} - '{{ exposure.type }}', {# type #} - '{{ tojson(exposure.owner) }}', {# owner #} - '{{ exposure.maturity }}', {# maturity #} - '{{ dbt_artifacts.escape_string(exposure.original_file_path) }}', {# path #} - '{{ dbt_artifacts.escape_string(exposure.description) }}', {# description #} - '{{ exposure.url }}', {# url #} - '{{ exposure.package_name }}', {# package_name #} - '{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #} - '{{ tojson(exposure.tags) }}', {# tags #} - '{{ dbt_artifacts.escape_string(tojson(exposure)) }}' {# all_results #} - ) - {%- if not loop.last %},{%- endif %} - {%- endfor %} - {% endset %} - {{ exposure_values }} - {% else %} - {{ return("") }} - {% endif %} -{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql index 7412d3ab..9abd1de1 100644 --- a/macros/upload_individual_datasets/upload_invocations.sql +++ b/macros/upload_individual_datasets/upload_invocations.sql @@ -162,7 +162,6 @@ {% macro dremio__get_invocations_dml_sql() -%} {% set invocation_values %} - values ( '{{ invocation_id }}', {# command_invocation_id #} '{{ dbt_version }}', {# dbt_version #} diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index 0a515d9b..ae289255 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -104,32 +104,14 @@ {% endif %} {%- endmacro %} -{% macro snowflake__get_model_executions_dml_sql(models) -%} +{% macro dremio__get_model_executions_dml_sql(models) -%} {% if models != [] %} {% set model_execution_values %} - select - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }} - from values {% for model in models -%} ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.node.unique_id }}', {# node_id #} - '{{ run_started_at }}', {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} {% set config_full_refresh = model.node.config.full_refresh %} {% if config_full_refresh is none %} @@ -140,19 +122,35 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} - {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} - {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} - {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + cast(null as timestamp), {# compile_started_at #} + {% else %} + {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} + {% endif %} + {% endfor %} - {{ model.execution_time }}, {# total_node_runtime #} - try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + cast(null as timestamp), {# query_completed_at #} + {% else %} + {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + cast(null as timestamp), {# compile_started_at #} + cast(null as timestamp), {# query_completed_at #} + {% endif %} + + cast({{ model.execution_time }} as float), {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} - '{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} + '{{ dbt_artifacts.escape_string(model.message) }}', {# message #} + '{{ dbt_artifacts.escape_string(tojson(model.adapter_response)) }}' {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -204,15 +202,32 @@ {% endif %} {%- endmacro %} -{% macro dremio__get_model_executions_dml_sql(models) -%} +{% macro snowflake__get_model_executions_dml_sql(models) -%} {% if models != [] %} {% set model_execution_values %} - values + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }} + from values {% for model in models -%} ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.node.unique_id }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ run_started_at }}', {# run_started_at #} {% set config_full_refresh = model.node.config.full_refresh %} {% if config_full_refresh is none %} @@ -223,35 +238,19 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - cast(null as timestamp), {# compile_started_at #} - {% else %} - {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - cast(null as timestamp), {# query_completed_at #} - {% else %} - {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - cast(null as timestamp), {# compile_started_at #} - cast(null as timestamp), {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} - cast({{ model.execution_time }} as float), {# total_node_runtime #} - null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} + {{ model.execution_time }}, {# total_node_runtime #} + try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ dbt_artifacts.escape_string(model.message) }}', {# message #} - '{{ dbt_artifacts.escape_string(tojson(model.adapter_response)) }}' {# adapter_response #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} + '{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index acc49bc6..1f015772 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -90,6 +90,42 @@ {% endif %} {%- endmacro %} +{% macro dremio__get_models_dml_sql(models) -%} + + {% if models != [] %} + {% set model_values %} + {% for model in models -%} + {% do model.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.unique_id }}', {# node_id #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ model.database }}', {# database #} + '{{ model.schema }}', {# schema #} + '{{ model.name }}', {# name #} + '{{ dbt_artifacts.escape_string(tojson(model.depends_on.nodes)) }}', {# depends_on_nodes #} + '{{ model.package_name }}', {# package_name #} + '{{ dbt_artifacts.escape_string(model.original_file_path) }}', {# path #} + '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.config.materialized }}', {# materialization #} + '{{ tojson(model.tags) }}', {# tags #} + '{{ dbt_artifacts.escape_string(tojson(model.config.meta)) }}', {# meta #} + '{{ model.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ dbt_artifacts.escape_string(tojson(model)) }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} + {% macro postgres__get_models_dml_sql(models) -%} {% if models != [] %} {% set model_values %} @@ -124,36 +160,3 @@ {{ return("") }} {% endif %} {%- endmacro %} - -{% macro dremio__get_models_dml_sql(models) -%} - - {% if models != [] %} - {% set model_values %} - values - {% for model in models -%} - {% do model.pop('raw_code', None) %} - ( - '{{ invocation_id }}', {# command_invocation_id #} - '{{ model.unique_id }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} - '{{ model.database }}', {# database #} - '{{ model.schema }}', {# schema #} - '{{ model.name }}', {# name #} - '{{ dbt_artifacts.escape_string(tojson(model.depends_on.nodes)) }}', {# depends_on_nodes #} - '{{ model.package_name }}', {# package_name #} - '{{ dbt_artifacts.escape_string(model.original_file_path) }}', {# path #} - '{{ model.checksum.checksum }}', {# checksum #} - '{{ model.config.materialized }}', {# materialization #} - '{{ tojson(model.tags) }}', {# tags #} - '{{ dbt_artifacts.escape_string(tojson(model.config.meta)) }}', {# meta #} - '{{ model.alias }}', {# alias #} - '{{ dbt_artifacts.escape_string(tojson(model)) }}' {# all_results #} - ) - {%- if not loop.last %},{%- endif %} - {%- endfor %} - {% endset %} - {{ model_values }} - {% else %} - {{ return("") }} - {% endif %} -{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index 11fa508a..66b91c74 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -134,27 +134,41 @@ {% endif %} {% endmacro -%} -{% macro snowflake__get_seed_executions_dml_sql(seeds) -%} +{% macro dremio__get_models_dml_sql(models) -%} + + {% if models != [] %} + {% set model_values %} + {% for model in models -%} + {% do model.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.unique_id }}', {# node_id #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ model.database }}', {# database #} + '{{ model.schema }}', {# schema #} + '{{ model.name }}', {# name #} + '{{ dbt_artifacts.escape_string(tojson(model.depends_on.nodes)) }}', {# depends_on_nodes #} + '{{ model.package_name }}', {# package_name #} + '{{ dbt_artifacts.escape_string(model.original_file_path) }}', {# path #} + '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.config.materialized }}', {# materialization #} + '{{ tojson(model.tags) }}', {# tags #} + '{{ dbt_artifacts.escape_string(tojson(model.config.meta)) }}', {# meta #} + '{{ model.alias }}', {# alias #} + '{{ dbt_artifacts.escape_string(tojson(model)) }}' {# all_results #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} + +{% macro postgres__get_seed_executions_dml_sql(seeds) -%} {% if seeds != [] %} {% set seed_execution_values %} - select - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }} - from values {% for model in seeds -%} ( '{{ invocation_id }}', {# command_invocation_id #} @@ -165,7 +179,7 @@ {% if config_full_refresh is none %} {% set config_full_refresh = flags.FULL_REFRESH %} {% endif %} - '{{ config_full_refresh }}', {# was_full_refresh #} + {{ config_full_refresh }}, {# was_full_refresh #} '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} @@ -192,13 +206,13 @@ {% endif %} {{ model.execution_time }}, {# total_node_runtime #} - try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} + null, -- rows_affected not available {# Databricks #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} - '{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} + $${{ model.message }}$$, {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -209,9 +223,27 @@ {% endif %} {% endmacro -%} -{% macro postgres__get_seed_executions_dml_sql(seeds) -%} +{% macro snowflake__get_seed_executions_dml_sql(seeds) -%} {% if seeds != [] %} {% set seed_execution_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }} + from values {% for model in seeds -%} ( '{{ invocation_id }}', {# command_invocation_id #} @@ -222,7 +254,7 @@ {% if config_full_refresh is none %} {% set config_full_refresh = flags.FULL_REFRESH %} {% endif %} - {{ config_full_refresh }}, {# was_full_refresh #} + '{{ config_full_refresh }}', {# was_full_refresh #} '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} @@ -249,13 +281,13 @@ {% endif %} {{ model.execution_time }}, {# total_node_runtime #} - null, -- rows_affected not available {# Databricks #} + try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - $${{ model.message }}$$, {# message #} - $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} + '{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -265,36 +297,3 @@ {{ return("") }} {% endif %} {% endmacro -%} - -{% macro dremio__get_models_dml_sql(models) -%} - - {% if models != [] %} - {% set model_values %} - values - {% for model in models -%} - {% do model.pop('raw_code', None) %} - ( - '{{ invocation_id }}', {# command_invocation_id #} - '{{ model.unique_id }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} - '{{ model.database }}', {# database #} - '{{ model.schema }}', {# schema #} - '{{ model.name }}', {# name #} - '{{ dbt_artifacts.escape_string(tojson(model.depends_on.nodes)) }}', {# depends_on_nodes #} - '{{ model.package_name }}', {# package_name #} - '{{ dbt_artifacts.escape_string(model.original_file_path) }}', {# path #} - '{{ model.checksum.checksum }}', {# checksum #} - '{{ model.config.materialized }}', {# materialization #} - '{{ tojson(model.tags) }}', {# tags #} - '{{ dbt_artifacts.escape_string(tojson(model.config.meta)) }}', {# meta #} - '{{ model.alias }}', {# alias #} - '{{ dbt_artifacts.escape_string(tojson(model)) }}' {# all_results #} - ) - {%- if not loop.last %},{%- endif %} - {%- endfor %} - {% endset %} - {{ model_values }} - {% else %} - {{ return("") }} - {% endif %} -{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql index 8f2d86eb..d05896ef 100644 --- a/macros/upload_individual_datasets/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -79,6 +79,38 @@ {% endif %} {%- endmacro %} +{% macro dremio__get_seeds_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_values %} + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ dbt_artifacts.escape_string(seed.original_file_path) }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + '{{ dbt_artifacts.escape_string(tojson(seed.config.meta)) }}', {# meta #} + '{{ seed.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ dbt_artifacts.escape_string(tojson(seed)) }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ seed_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} + + {% macro postgres__get_seeds_dml_sql(seeds) -%} {% if seeds != [] %} {% set seed_values %} @@ -109,32 +141,3 @@ {{ return("") }} {% endif %} {%- endmacro %} - -{% macro dremio__get_seeds_dml_sql(seeds) -%} - - {% if seeds != [] %} - {% set seed_values %} - values - {% for seed in seeds -%} - ( - '{{ invocation_id }}', {# command_invocation_id #} - '{{ seed.unique_id }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} - '{{ seed.database }}', {# database #} - '{{ seed.schema }}', {# schema #} - '{{ seed.name }}', {# name #} - '{{ seed.package_name }}', {# package_name #} - '{{ dbt_artifacts.escape_string(seed.original_file_path) }}', {# path #} - '{{ seed.checksum.checksum }}', {# checksum #} - '{{ dbt_artifacts.escape_string(tojson(seed.config.meta)) }}', {# meta #} - '{{ seed.alias }}', {# alias #} - '{{ dbt_artifacts.escape_string(tojson(seed)) }}' {# all_results #} - ) - {%- if not loop.last %},{%- endif %} - {%- endfor %} - {% endset %} - {{ seed_values }} - {% else %} - {{ return("") }} - {% endif %} -{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index d6eaf0a3..c2849c5f 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -134,32 +134,14 @@ {% endif %} {% endmacro -%} -{% macro snowflake__get_snapshot_executions_dml_sql(snapshots) -%} +{% macro dremio__get_snapshot_executions_dml_sql(snapshots) -%} {% if snapshots != [] %} {% set snapshot_execution_values %} - select - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }} - from values {% for model in snapshots -%} ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.node.unique_id }}', {# node_id #} - '{{ run_started_at }}', {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} {% set config_full_refresh = model.node.config.full_refresh %} {% if config_full_refresh is none %} @@ -173,32 +155,32 @@ {% if model.timing != [] %} {% for stage in model.timing if stage.name == "compile" %} {% if loop.length == 0 %} - null, {# compile_started_at #} + cast(null as timestamp), {# compile_started_at #} {% else %} - '{{ stage.started_at }}', {# compile_started_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} {% endif %} {% endfor %} {% for stage in model.timing if stage.name == "execute" %} {% if loop.length == 0 %} - null, {# query_completed_at #} + cast(null as timestamp), {# query_completed_at #} {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} {% endif %} {% endfor %} {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} + cast(null as timestamp), {# compile_started_at #} + cast(null as timestamp), {# query_completed_at #} {% endif %} - {{ model.execution_time }}, {# total_node_runtime #} - try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} + cast({{ model.execution_time }} as float), {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} - '{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} + '{{ dbt_artifacts.escape_string(model.message) }}', {# message #} + '{{ dbt_artifacts.escape_string(tojson(model.adapter_response)) }}' {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -266,15 +248,33 @@ {% endif %} {% endmacro -%} -{% macro dremio__get_snapshot_executions_dml_sql(snapshots) -%} + +{% macro snowflake__get_snapshot_executions_dml_sql(snapshots) -%} {% if snapshots != [] %} {% set snapshot_execution_values %} - values + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }} + from values {% for model in snapshots -%} ( '{{ invocation_id }}', {# command_invocation_id #} '{{ model.node.unique_id }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ run_started_at }}', {# run_started_at #} {% set config_full_refresh = model.node.config.full_refresh %} {% if config_full_refresh is none %} @@ -288,32 +288,32 @@ {% if model.timing != [] %} {% for stage in model.timing if stage.name == "compile" %} {% if loop.length == 0 %} - cast(null as timestamp), {# compile_started_at #} + null, {# compile_started_at #} {% else %} - {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} + '{{ stage.started_at }}', {# compile_started_at #} {% endif %} {% endfor %} {% for stage in model.timing if stage.name == "execute" %} {% if loop.length == 0 %} - cast(null as timestamp), {# query_completed_at #} + null, {# query_completed_at #} {% else %} - {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} + '{{ stage.completed_at }}', {# query_completed_at #} {% endif %} {% endfor %} {% else %} - cast(null as timestamp), {# compile_started_at #} - cast(null as timestamp), {# query_completed_at #} + null, {# compile_started_at #} + null, {# query_completed_at #} {% endif %} - cast({{ model.execution_time }} as float), {# total_node_runtime #} - null, -- rows_affected not available {# Only available in Snowflake #} + {{ model.execution_time }}, {# total_node_runtime #} + try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} '{{ model.node.config.materialized }}', {# materialization #} '{{ model.node.schema }}', {# schema #} '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} - '{{ dbt_artifacts.escape_string(model.message) }}', {# message #} - '{{ dbt_artifacts.escape_string(tojson(model.adapter_response)) }}' {# adapter_response #} + '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} + '{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_individual_datasets/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql index fb52c992..44e80922 100644 --- a/macros/upload_individual_datasets/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -87,6 +87,39 @@ {% endif %} {%- endmacro %} +{% macro dremio__get_snapshots_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_values %} + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + '{{ tojson(snapshot.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ dbt_artifacts.escape_string(snapshot.original_file_path) }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + '{{ dbt_artifacts.escape_string(tojson(snapshot.config.meta)) }}', {# meta #} + '{{ snapshot.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ dbt_artifacts.escape_string(tojson(snapshot)) }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ snapshot_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} + {% macro postgres__get_snapshots_dml_sql(snapshots) -%} {% if snapshots != [] %} {% set snapshot_values %} @@ -119,34 +152,3 @@ {{ return("") }} {% endif %} {%- endmacro %} - -{% macro dremio__get_snapshots_dml_sql(snapshots) -%} - - {% if snapshots != [] %} - {% set snapshot_values %} - values - {% for snapshot in snapshots -%} - ( - '{{ invocation_id }}', {# command_invocation_id #} - '{{ snapshot.unique_id }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} - '{{ snapshot.database }}', {# database #} - '{{ snapshot.schema }}', {# schema #} - '{{ snapshot.name }}', {# name #} - '{{ tojson(snapshot.depends_on.nodes) }}', {# depends_on_nodes #} - '{{ snapshot.package_name }}', {# package_name #} - '{{ dbt_artifacts.escape_string(snapshot.original_file_path) }}', {# path #} - '{{ snapshot.checksum.checksum }}', {# checksum #} - '{{ snapshot.config.strategy }}', {# strategy #} - '{{ dbt_artifacts.escape_string(tojson(snapshot.config.meta)) }}', {# meta #} - '{{ snapshot.alias }}', {# alias #} - '{{ dbt_artifacts.escape_string(tojson(snapshot)) }}' {# all_results #} - ) - {%- if not loop.last %},{%- endif %} - {%- endfor %} - {% endset %} - {{ snapshot_values }} - {% else %} - {{ return("") }} - {% endif %} -{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql index c0e99716..3d751f53 100644 --- a/macros/upload_individual_datasets/upload_sources.sql +++ b/macros/upload_individual_datasets/upload_sources.sql @@ -79,6 +79,37 @@ {% endif %} {%- endmacro %} +{% macro dremio__get_sources_dml_sql(sources) -%} + {% if sources != [] %} + {% set source_values %} + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + '{{ dbt_artifacts.escape_string(source.loaded_at_field) }}', {# loaded_at_field #} + '{{ dbt_artifacts.escape_string(tojson(source.freshness)) }}', {# freshness #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ dbt_artifacts.escape_string(tojson(source)) }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ source_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} + {% macro postgres__get_sources_dml_sql(sources) -%} {% if sources != [] %} {% set source_values %} @@ -109,32 +140,3 @@ {{ return("") }} {% endif %} {%- endmacro %} - -{% macro dremio__get_sources_dml_sql(sources) -%} - - {% if sources != [] %} - {% set source_values %} - values - {% for source in sources -%} - ( - '{{ invocation_id }}', {# command_invocation_id #} - '{{ source.unique_id }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} - '{{ source.database }}', {# database #} - '{{ source.schema }}', {# schema #} - '{{ source.source_name }}', {# source_name #} - '{{ source.loader }}', {# loader #} - '{{ source.name }}', {# name #} - '{{ source.identifier }}', {# identifier #} - '{{ dbt_artifacts.escape_string(source.loaded_at_field) }}', {# loaded_at_field #} - '{{ dbt_artifacts.escape_string(tojson(source.freshness)) }}', {# freshness #} - '{{ dbt_artifacts.escape_string(tojson(source)) }}' {# all_results #} - ) - {%- if not loop.last %},{%- endif %} - {%- endfor %} - {% endset %} - {{ source_values }} - {% else %} - {{ return("") }} - {% endif %} -{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index 1a132ee4..092c940e 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -126,20 +126,20 @@ {% endif %} {% endmacro -%} -{% macro postgres__get_test_executions_dml_sql(tests) -%} +{% macro dremio__get_test_executions_dml_sql(tests) -%} {% if tests != [] %} {% set test_execution_values %} {% for test in tests -%} ( '{{ invocation_id }}', {# command_invocation_id #} '{{ test.node.unique_id }}', {# node_id #} - '{{ run_started_at }}', {# run_started_at #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} {% set config_full_refresh = test.node.config.full_refresh %} {% if config_full_refresh is none %} {% set config_full_refresh = flags.FULL_REFRESH %} {% endif %} - {{ config_full_refresh }}, {# was_full_refresh #} + '{{ config_full_refresh }}', {# was_full_refresh #} '{{ test.thread_id }}', {# thread_id #} '{{ test.status }}', {# status #} @@ -147,32 +147,31 @@ {% if test.timing != [] %} {% for stage in test.timing if stage.name == "compile" %} {% if loop.length == 0 %} - null, {# compile_started_at #} + cast(null as timestamp), {# compile_started_at #} {% else %} - '{{ stage.started_at }}', {# compile_started_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} {% endif %} {% endfor %} {% for stage in test.timing if stage.name == "execute" %} {% if loop.length == 0 %} - null, {# query_completed_at #} + cast(null as timestamp), {# query_completed_at #} {% else %} - '{{ stage.completed_at }}', {# query_completed_at #} + {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} {% endif %} {% endfor %} {% else %} - null, {# compile_started_at #} - null, {# query_completed_at #} + cast(null as timestamp), {# compile_started_at #} + cast(null as timestamp), {# query_completed_at #} {% endif %} - {{ test.execution_time }}, {# total_node_runtime #} + cast({{ test.execution_time }} as float), {# total_node_runtime #} null, {# rows_affected not available in Databricks #} {{ 'null' if test.failures is none else test.failures }}, {# failures #} - $${{ test.message }}$$, {# message #} - $${{ tojson(test.adapter_response) }}$$ {# adapter_response #} + '{{ dbt_artifacts.escape_string(test.message) }}', {# message #} + '{{ dbt_artifacts.escape_string(tojson(test.adapter_response)) }}' {# adapter_response #} ) {%- if not loop.last %},{%- endif %} - {%- endfor %} {% endset %} {{ test_execution_values }} @@ -181,21 +180,20 @@ {% endif %} {% endmacro -%} -{% macro dremio__get_test_executions_dml_sql(tests) -%} +{% macro postgres__get_test_executions_dml_sql(tests) -%} {% if tests != [] %} {% set test_execution_values %} - values {% for test in tests -%} ( '{{ invocation_id }}', {# command_invocation_id #} '{{ test.node.unique_id }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ run_started_at }}', {# run_started_at #} {% set config_full_refresh = test.node.config.full_refresh %} {% if config_full_refresh is none %} {% set config_full_refresh = flags.FULL_REFRESH %} {% endif %} - '{{ config_full_refresh }}', {# was_full_refresh #} + {{ config_full_refresh }}, {# was_full_refresh #} '{{ test.thread_id }}', {# thread_id #} '{{ test.status }}', {# status #} @@ -203,31 +201,32 @@ {% if test.timing != [] %} {% for stage in test.timing if stage.name == "compile" %} {% if loop.length == 0 %} - cast(null as timestamp), {# compile_started_at #} + null, {# compile_started_at #} {% else %} - {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} + '{{ stage.started_at }}', {# compile_started_at #} {% endif %} {% endfor %} {% for stage in test.timing if stage.name == "execute" %} {% if loop.length == 0 %} - cast(null as timestamp), {# query_completed_at #} + null, {# query_completed_at #} {% else %} - {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} + '{{ stage.completed_at }}', {# query_completed_at #} {% endif %} {% endfor %} {% else %} - cast(null as timestamp), {# compile_started_at #} - cast(null as timestamp), {# query_completed_at #} + null, {# compile_started_at #} + null, {# query_completed_at #} {% endif %} - cast({{ test.execution_time }} as float), {# total_node_runtime #} + {{ test.execution_time }}, {# total_node_runtime #} null, {# rows_affected not available in Databricks #} {{ 'null' if test.failures is none else test.failures }}, {# failures #} - '{{ dbt_artifacts.escape_string(test.message) }}', {# message #} - '{{ dbt_artifacts.escape_string(tojson(test.adapter_response)) }}' {# adapter_response #} + $${{ test.message }}$$, {# message #} + $${{ tojson(test.adapter_response) }}$$ {# adapter_response #} ) {%- if not loop.last %},{%- endif %} + {%- endfor %} {% endset %} {{ test_execution_values }} diff --git a/macros/upload_individual_datasets/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql index bb9701c5..f66a5a03 100644 --- a/macros/upload_individual_datasets/upload_tests.sql +++ b/macros/upload_individual_datasets/upload_tests.sql @@ -70,6 +70,35 @@ {% endif %} {%- endmacro %} +{% macro dremio__get_tests_dml_sql(tests) -%} + + {% if tests != [] %} + {% set test_values %} + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} + '{{ test.name }}', {# name #} + '{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ dbt_artifacts.escape_string(test.original_file_path) }}', {# test_path #} + '{{ tojson(test.tags) }}', {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ dbt_artifacts.escape_string(tojson(test)) }}' {# all_fields #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ test_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} + {% macro postgres__get_tests_dml_sql(tests) -%} {% if tests != [] %} {% set test_values %} @@ -98,29 +127,3 @@ {% endif %} {%- endmacro %} -{% macro dremio__get_tests_dml_sql(tests) -%} - - {% if tests != [] %} - {% set test_values %} - values - {% for test in tests -%} - ( - '{{ invocation_id }}', {# command_invocation_id #} - '{{ test.unique_id }}', {# node_id #} - {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} - '{{ test.name }}', {# name #} - '{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #} - '{{ test.package_name }}', {# package_name #} - '{{ dbt_artifacts.escape_string(test.original_file_path) }}', {# test_path #} - '{{ tojson(test.tags) }}', {# tags #} - '{{ dbt_artifacts.escape_string(tojson(test)) }}' {# all_fields #} - ) - {%- if not loop.last %},{%- endif %} - {%- endfor %} - {% endset %} - {{ test_values }} - {% else %} - {{ return("") }} - {% endif %} -{% endmacro -%} - diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 4bee88e6..4d367f12 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -61,7 +61,8 @@ {% macro dremio__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} {% set insert_into_table_query %} - insert into "{{database_name}}"."{{ schema_name }}"."{{ table_name }}" + insert into {{ relation }} {{ fields }} + values {{ content }} {% endset %} From 80bc228647f1782fd6c49236e1b25fd9da8483fc Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Fri, 29 Sep 2023 16:43:19 +0100 Subject: [PATCH 08/14] Reinstate intsert_into_metadata_table macro --- integration_test_project/dbt_project.yml | 4 ++-- macros/upload_results/insert_into_metadata_table.sql | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index b9f39973..14d870e0 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -25,8 +25,8 @@ vars: models: +persist_docs: - columns: '{{ target.name != "dremio" }}' - relation: '{{ target.name != "dremio" }}' + columns: '{{ target.type != "dremio" }}' + relation: '{{ target.type != "dremio" }}' seeds: +quote_columns: false diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 2ec4d5d5..f225cd57 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -45,5 +45,14 @@ {%- endmacro %} +{% macro dremio__insert_into_metadata_table(relation, fields, content) -%} + {% set insert_into_table_query %} + insert into {{ relation }} + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} +{%- endmacro %} + {% macro default__insert_into_metadata_table(relation, fields, content) -%} {%- endmacro %} From ab5cdb53b58ce07784bb78ddd2918a0901fa0a77 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 29 Sep 2023 17:06:13 +0100 Subject: [PATCH 09/14] Minor formatting --- macros/database_specific_helpers/type_helpers.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/database_specific_helpers/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql index de9ce96f..5c235cde 100644 --- a/macros/database_specific_helpers/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -27,7 +27,7 @@ {% endmacro %} {%- macro dremio__type_json() -%} - VARCHAR + varchar {%- endmacro -%} {#- ARRAY -#} @@ -49,5 +49,5 @@ {% endmacro %} {%- macro dremio__type_array() -%} - VARCHAR + varchar {%- endmacro -%} From f67042b61e12dd2110c9ba101360d9e88e1a17a2 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 29 Sep 2023 17:10:54 +0100 Subject: [PATCH 10/14] Add Dremio to PR template --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3ce860d8..ca46c45a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -29,5 +29,5 @@ - [ ] Snowflake - [ ] Google BigQuery - [ ] Databricks -- [ ] Spark +- [ ] Dremio - [ ] N/A From 4454abf97961b96e40d5a5d6b080bf9c4d0291e2 Mon Sep 17 00:00:00 2001 From: Gemma Down Date: Fri, 29 Sep 2023 17:22:56 +0100 Subject: [PATCH 11/14] Bug squashing --- .../upload_model_executions.sql | 24 ++++--------------- .../upload_seed_executions.sql | 2 +- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index ae289255..60d11088 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -122,26 +122,10 @@ '{{ model.thread_id }}', {# thread_id #} '{{ model.status }}', {# status #} - {% if model.timing != [] %} - {% for stage in model.timing if stage.name == "compile" %} - {% if loop.length == 0 %} - cast(null as timestamp), {# compile_started_at #} - {% else %} - {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} - {% endif %} - {% endfor %} - - {% for stage in model.timing if stage.name == "execute" %} - {% if loop.length == 0 %} - cast(null as timestamp), {# query_completed_at #} - {% else %} - {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} - {% endif %} - {% endfor %} - {% else %} - cast(null as timestamp), {# compile_started_at #} - cast(null as timestamp), {# query_completed_at #} - {% endif %} + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}cast(null as timestamp){% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}cast(null as timestamp){% endif %}, {# query_completed_at #} cast({{ model.execution_time }} as float), {# total_node_runtime #} null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index 66b91c74..bbd9127c 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -134,7 +134,7 @@ {% endif %} {% endmacro -%} -{% macro dremio__get_models_dml_sql(models) -%} +{% macro dremio__get_seed_executions_dml_sql(models) -%} {% if models != [] %} {% set model_values %} From 99301ea83eb1b2817b6a3022dc86e70874ac3f25 Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Fri, 29 Sep 2023 21:32:30 +0100 Subject: [PATCH 12/14] Fix regressions --- .../upload_model_executions.sql | 4 +- .../upload_seed_executions.sql | 54 ++++++++++++++----- .../insert_into_metadata_table.sql | 4 +- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index 60d11088..71f03783 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -123,9 +123,9 @@ '{{ model.status }}', {# status #} {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} - {% if compile_started_at %}'{{ compile_started_at }}'{% else %}cast(null as timestamp){% endif %}, {# compile_started_at #} + {% if compile_started_at %}{{ dbt_artifacts.cast_as_timestamp(compile_started_at) }}{% else %}cast(null as timestamp){% endif %}, {# compile_started_at #} {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} - {% if query_completed_at %}'{{ query_completed_at }}'{% else %}cast(null as timestamp){% endif %}, {# query_completed_at #} + {% if query_completed_at %}{{ dbt_artifacts.cast_as_timestamp(query_completed_at) }}{% else %}cast(null as timestamp){% endif %}, {# query_completed_at #} cast({{ model.execution_time }} as float), {# total_node_runtime #} null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index bbd9127c..fb674fe4 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -139,23 +139,49 @@ {% if models != [] %} {% set model_values %} {% for model in models -%} - {% do model.pop('raw_code', None) %} ( '{{ invocation_id }}', {# command_invocation_id #} - '{{ model.unique_id }}', {# node_id #} + '{{ model.node.unique_id }}', {# node_id #} {{ dbt_artifacts.cast_as_timestamp(run_started_at) }}, {# run_started_at #} - '{{ model.database }}', {# database #} - '{{ model.schema }}', {# schema #} - '{{ model.name }}', {# name #} - '{{ dbt_artifacts.escape_string(tojson(model.depends_on.nodes)) }}', {# depends_on_nodes #} - '{{ model.package_name }}', {# package_name #} - '{{ dbt_artifacts.escape_string(model.original_file_path) }}', {# path #} - '{{ model.checksum.checksum }}', {# checksum #} - '{{ model.config.materialized }}', {# materialization #} - '{{ tojson(model.tags) }}', {# tags #} - '{{ dbt_artifacts.escape_string(tojson(model.config.meta)) }}', {# meta #} - '{{ model.alias }}', {# alias #} - '{{ dbt_artifacts.escape_string(tojson(model)) }}' {# all_results #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + {{ dbt_artifacts.cast_as_timestamp(stage.started_at) }}, {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + {{ dbt_artifacts.cast_as_timestamp(stage.completed_at) }}, {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ dbt_artifacts.escape_string(model.message) }}', {# message #} + '{{ dbt_artifacts.escape_string(tojson(model.adapter_response)) }}' {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 4d367f12..db0af15c 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -59,9 +59,9 @@ {%- endmacro %} -{% macro dremio__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} +{% macro dremio__insert_into_metadata_table(relation, fields, content) -%} {% set insert_into_table_query %} - insert into {{ relation }} {{ fields }} + insert into {{ relation }} values {{ content }} {% endset %} From 4eb7a9807839e4eddc76bbd8f323d187d74b8035 Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Tue, 7 May 2024 10:20:42 +0100 Subject: [PATCH 13/14] Pass all env vars to tox --- .gitignore | 2 ++ integration_test_project/profiles.yml | 2 +- tox.ini | 15 +++++++++++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index eb7e5d18..772a0893 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ env.sh .venv .DS_Store + +integration_test_project/package-lock.yml diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index 4987588c..8f8a8674 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -61,5 +61,5 @@ dbt_artifacts: threads: 8 use_ssl: true dremio_space: *schema - object_storage_path: *schema + object_storage_path: "{{ env_var('DBT_ENV_OBJECT_STORAGE_PATH') }}" object_storage_source: "{{ env_var('DBT_ENV_OBJECT_STORAGE_SOURCE') }}" diff --git a/tox.ini b/tox.ini index 14b6a8c1..25825bfb 100644 --- a/tox.ini +++ b/tox.ini @@ -95,6 +95,11 @@ passenv = TEST_ENV_VAR_NUMBER TEST_ENV_VAR_EMPTY TEST_ENV_VAR_WITH_QUOTE + DBT_ENV_DREMIO_USER + DBT_ENV_DREMIO_PASSWORD + DBT_ENV_DREMIO_SOFTWARE_HOST + DBT_ENV_OBJECT_STORAGE_SOURCE + DBT_ENV_OBJECT_STORAGE_PATH [testenv:lint] deps = {[sqlfluff]deps} @@ -318,10 +323,16 @@ commands = [testenv:integration_dremio_1_5_0] changedir = integration_test_project -deps = dbt-dremio~=1.5.0 +deps = dbt-dremio==1.5.1 commands = dbt clean dbt deps dbt build --exclude snapshot --target dremio -# Note: Dremio is not yet supported on dbt v1.6.0 +[testenv:integration_dremio_1_7_0] +changedir = integration_test_project +deps = dbt-dremio~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --exclude snapshot --target dremio From fccd05ae6aed8bd061197273011b1affabbbcbfe Mon Sep 17 00:00:00 2001 From: "Firman, Max" Date: Tue, 7 May 2024 10:47:11 +0100 Subject: [PATCH 14/14] Pin to 1.5.0 --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 4ff722f1..6a38181e 100644 --- a/tox.ini +++ b/tox.ini @@ -330,7 +330,7 @@ commands = # Dremio integration test [testenv:integration_dremio] changedir = integration_test_project -deps = dbt-dremio~=1.5.0 +deps = dbt-dremio~=1.7.0 commands = dbt clean dbt deps @@ -354,7 +354,7 @@ commands = [testenv:integration_dremio_1_5_0] changedir = integration_test_project -deps = dbt-dremio==1.5.1 +deps = dbt-dremio==1.5.0 commands = dbt clean dbt deps