diff --git a/.github/workflows/ci-integration-tests-csae.yml b/.github/workflows/ci-integration-tests-csae.yml index e282cdf9..3a6166e8 100644 --- a/.github/workflows/ci-integration-tests-csae.yml +++ b/.github/workflows/ci-integration-tests-csae.yml @@ -178,11 +178,26 @@ jobs: timeout_seconds: 300 priority: interactive retries: 1 + dbt_catalog_test_qvci: + type: teradata + host: $DBT_TERADATA_SERVER_NAME_QVCI + user: $DBT_TERADATA_USERNAME_QVCI + password: $DBT_TERADATA_PASSWORD_QVCI + logmech: TD2 + schema: dbt_catalog_test + tmode: ANSI + threads: 1 + timeout_seconds: 300 + priority: interactive + retries: 1 EOF env: DBT_TERADATA_SERVER_NAME: ${{ steps.create-csae-environments.outputs.teradata-server-name }} DBT_TERADATA_USERNAME: dbc DBT_TERADATA_PASSWORD: ${{ secrets.CSAE_ENV_PASSWORD }} + DBT_TERADATA_SERVER_NAME_QVCI: ${{ steps.create-csae-environments.outputs.teradata-server-name-qvci }} + DBT_TERADATA_USERNAME_QVCI: dbc + DBT_TERADATA_PASSWORD_QVCI: ${{ secrets.CSAE_ENV_PASSWORD }} - name: Run performance tests with ANSI run: | @@ -191,6 +206,8 @@ jobs: - name: Run catalog tests with ANSI run: | + # cd $GITHUB_WORKSPACE/test/catalog/with_qvci + # ./run.sh cd $GITHUB_WORKSPACE/test/catalog/without_qvci ./run.sh @@ -224,6 +241,18 @@ jobs: timeout_seconds: 300 priority: interactive retries: 1 + dbt_catalog_test_qvci: + type: teradata + host: $DBT_TERADATA_SERVER_NAME_QVCI + user: $DBT_TERADATA_USERNAME_QVCI + password: $DBT_TERADATA_PASSWORD_QVCI + logmech: TD2 + schema: dbt_catalog_test + tmode: TERA + threads: 1 + timeout_seconds: 300 + priority: interactive + retries: 1 dbt_test_valid_history: type: teradata host: $DBT_TERADATA_SERVER_NAME @@ -241,6 +270,9 @@ jobs: DBT_TERADATA_SERVER_NAME: ${{ steps.create-csae-environments.outputs.teradata-server-name }} DBT_TERADATA_USERNAME: dbc DBT_TERADATA_PASSWORD: ${{ secrets.CSAE_ENV_PASSWORD }} + DBT_TERADATA_SERVER_NAME_QVCI: ${{ steps.create-csae-environments.outputs.teradata-server-name-qvci }} + DBT_TERADATA_USERNAME_QVCI: dbc + DBT_TERADATA_PASSWORD_QVCI: ${{ secrets.CSAE_ENV_PASSWORD }} - name: Run performance tests with TERA run: | @@ -249,6 +281,8 @@ jobs: - name: Run catalog tests with TERA run: | + # cd $GITHUB_WORKSPACE/test/catalog/with_qvci + # ./run.sh cd $GITHUB_WORKSPACE/test/catalog/without_qvci ./run.sh diff --git a/.github/workflows/scripts/createTestEnvironments.sh b/.github/workflows/scripts/createTestEnvironments.sh index 921f8a6f..f8dcc22f 100755 --- a/.github/workflows/scripts/createTestEnvironments.sh +++ b/.github/workflows/scripts/createTestEnvironments.sh @@ -44,6 +44,21 @@ createRegularEnv() { echo "teradata-server-name=$TERADATA_SERVER_NAME" >> $GITHUB_OUTPUT } +createEnvWithQVCI() { + local RESULT=$(curl -s -L 'https://api.clearscape.teradata.com/environments' \ + -H "Authorization: Bearer $CSAE_TOKEN" \ + -H 'Content-Type: application/json' \ + --data-raw "{ + \"name\": \"$CSAE_ENV_NAME-qvci\", + \"region\": \"us-central\", + \"password\": \"$CSAE_ENV_PASSWORD\", + \"startupScript\": \"#!/bin/bash\n\nfunction withRetry {\n local RETRIES=\$1; shift 1\n local SLEEP=\$1; shift 1\n for i in \$(seq 1 \$RETRIES)\n do\n echo \\\"Attempt \$i: Running command \$@\\\"\n \$@ && s=0 && break || s=\$? && sleep \$SLEEP\n done\n return \$s\n}\n\nfunction ensureVantageIsUp {\n pdestate -a\n pdestate -a | grep \\\"DBS state is [45]\\\"\n}\n\ndbscontrol << EOF\nm i 551=false\nW\nEOF\ntpareset -y changing dbscontrol\nwithRetry 40 2 ensureVantageIsUp\" + }") + local TERADATA_SERVER_NAME=$(echo $RESULT | jq -r '.dnsName') + echo "teradata-server-name-qvci=$TERADATA_SERVER_NAME" >> $GITHUB_OUTPUT +} + background createRegularEnv +background createEnvWithQVCI reap diff --git a/.github/workflows/scripts/deleteTestEnvironments.sh b/.github/workflows/scripts/deleteTestEnvironments.sh index d3445daa..7755f3e3 100755 --- a/.github/workflows/scripts/deleteTestEnvironments.sh +++ b/.github/workflows/scripts/deleteTestEnvironments.sh @@ -35,7 +35,13 @@ deleteRegularEnv() { -H "Authorization: Bearer $CSAE_TOKEN" } +deleteEnvWithQVCI() { + curl -s -L --request DELETE "https://api.clearscape.teradata.com/environments/$CSAE_ENV_NAME-qvci" \ + -H "Authorization: Bearer $CSAE_TOKEN" +} + background deleteRegularEnv +background deleteEnvWithQVCI reap @@ -76,6 +82,12 @@ deleteRegularEnv() { -H "Authorization: Bearer $CSAE_TOKEN" } +deleteEnvWithQVCI() { + curl -s -L --request DELETE "https://api.clearscape.teradata.com/environments/$CSAE_ENV_NAME-qvci" \ + -H "Authorization: Bearer $CSAE_TOKEN" +} + background deleteRegularEnv +background deleteEnvWithQVCI reap diff --git a/README.md b/README.md index 756788fd..665367ba 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ my-teradata-db-profile: tmode: ANSI ``` -At a minimum, you need to specify `host`, `user`, `password`, `schema` (database). +At a minimum, you need to specify `host`, `user`, `password`, `schema` (database), `tmode`. ## Python compatibility @@ -343,6 +343,31 @@ All dbt commands are supported. ### Custom configurations +#### General + +* *Enable view column types in docs* - Teradata Vantage has a dbscontrol configuration flag called `DisableQVCI` (QVCI - Queryable View Column Index). This flag instructs the database to build `DBC.ColumnsJQV` with view column type definitions. + + > :information_source: Existing customers, please see [KB0022230](https://support.teradata.com/knowledge?id=kb_article_view&sys_kb_id=d066248b1b0000187361c8415b4bcb48) for more information about enabling QVCI. + + To enable this functionality you need to: + 1. Enable QVCI mode in Vantage. Use `dbscontrol` utility and then restart Teradata. Run these commands as a privileged user on a Teradata node: + ```bash + # option 551 is DisableQVCI. Setting it to false enables QVCI. + dbscontrol << EOF + M internal 551=false + W + EOF + + # restart Teradata + tpareset -y Enable QVCI + ``` + 2. Instruct `dbt` to use `QVCI` mode. Include the following variable in your `dbt_project.yml`: + ```yaml + vars: + use_qvci: true + ``` + For example configuration, see `test/catalog/with_qvci/dbt_project.yml`. + #### Models ##### Table @@ -754,6 +779,15 @@ If no query_band is set by user, default query_band will come in play that is : * Unit testing is supported in dbt-teradata, allowing users to write and execute unit tests using the dbt test command. * For detailed guidance, refer to the dbt documentation. +* QVCI must be enabled in the database to run unit tests for views. + * Additional details on enabling QVCI can be found in the General section. + * Without QVCI enabled, unit test support for views will be limited. + * Users might encounter the following database error when testing views without QVCI enabled: + ``` + * [Teradata Database] [Error 3706] Syntax error: Data Type "N" does not match a Defined Type name. + ``` + + ## Credits The adapter was originally created by [Doug Beatty](https://github.com/dbeatty10). Teradata took over the adapter in January 2022. We are grateful to Doug for founding the project and accelerating the integration of dbt + Teradata. diff --git a/dbt/include/teradata/macros/adapters.sql b/dbt/include/teradata/macros/adapters.sql index b36f9484..174928eb 100644 --- a/dbt/include/teradata/macros/adapters.sql +++ b/dbt/include/teradata/macros/adapters.sql @@ -118,23 +118,8 @@ {% endmacro %} {% macro teradata__get_columns_in_relation(relation) -%} - {% call statement('check_table_or_view', fetch_result=True) %} - SELECT TableKind FROM DBC.TablesV WHERE DatabaseName = '{{ relation.schema }}' AND TableName = '{{ relation.identifier }}' - {% endcall %} - - {% set table_kind = load_result('check_table_or_view').table.columns['TableKind'].values()[0] | trim %} - - {%- if table_kind == 'V' -%} - {% set temp_relation_for_view = relation.identifier ~ '_tmp_viw_tbl' %} - {% call statement('drop_existing_table', fetch_result=False) %} - DROP table /*+ IF EXISTS */ "{{ relation.schema }}"."{{ temp_relation_for_view }}"; - {% endcall %} - load_result('drop_existing_table') - {% call statement('creating_table_from_view', fetch_result=False) %} - CREATE TABLE "{{ relation.schema }}"."{{ temp_relation_for_view }}" AS (SELECT * FROM "{{ relation.schema }}"."{{ relation.identifier }}") WITH NO DATA; - {% endcall %} - load_result('creating_table_from_view') - {% endif %} + {% set use_qvci = var("use_qvci", "false") | as_bool %} + {{ log("use_qvci set to : " ~ use_qvci) }} {% call statement('get_columns_in_relation', fetch_result=True) %} SELECT ColumnsV.ColumnName AS "column", @@ -193,41 +178,31 @@ NULL AS table_database, ColumnsV.DatabaseName AS table_schema, ColumnsV.TableName AS table_name, - {%- if table_kind == 'V' -%} - 'view' AS table_type, - {%- else -%} CASE WHEN TablesV.TableKind = 'T' THEN 'table' WHEN TablesV.TableKind = 'O' THEN 'table' + WHEN TablesV.TableKind = 'V' THEN 'view' ELSE TablesV.TableKind END AS table_type, - {%- endif -%} ColumnsV.ColumnID AS column_index FROM + {% if use_qvci == True -%} + {{ information_schema_name(relation.schema) }}.ColumnsJQV AS ColumnsV + {% else %} {{ information_schema_name(relation.schema) }}.ColumnsV + {%- endif %} LEFT OUTER JOIN {{ information_schema_name(relation.schema) }}.TablesV ON ColumnsV.DatabaseName = TablesV.DatabaseName AND ColumnsV.TableName = TablesV.TableName WHERE TablesV.TableKind IN ('T', 'V', 'O') AND ColumnsV.DatabaseName = '{{ relation.schema }}' (NOT CASESPECIFIC) - {%- if table_kind == 'V' -%} - AND ColumnsV.TableName = '{{ temp_relation_for_view }}' (NOT CASESPECIFIC) - {%- else -%} AND ColumnsV.TableName = '{{ relation.identifier }}' (NOT CASESPECIFIC) - {%- endif -%} ORDER BY ColumnsV.ColumnID {% endcall %} {% set table = load_result('get_columns_in_relation').table %} - {%- if table_kind == 'V' -%} - {% call statement('drop_table_from_view', fetch_result=False) %} - DROP table /*+ IF EXISTS */ "{{ relation.schema }}"."{{ temp_relation_for_view }}"; - {% endcall %} - load_result('drop_table_from_view') - {% endif %} - {{ return(sql_convert_columns_in_relation(table)) }} {% endmacro %} diff --git a/dbt/include/teradata/macros/catalog.sql b/dbt/include/teradata/macros/catalog.sql index be3c4ad8..90866f7c 100644 --- a/dbt/include/teradata/macros/catalog.sql +++ b/dbt/include/teradata/macros/catalog.sql @@ -1,49 +1,7 @@ -{% macro teradata__get_views_from_relations(relations) -%} - {% set view_relations = [] %} - {%- for relation in relations -%} - {% if relation.schema and relation.identifier %} - {% call statement('check_table_or_view', fetch_result=True) %} - SELECT TableKind FROM DBC.TablesV WHERE DatabaseName = '{{ relation.schema }}' AND TableName = '{{ relation.identifier }}' - {% endcall %} - {% set table_kind = load_result('check_table_or_view').table.columns['TableKind'].values()[0] | trim %} - {%- if table_kind == 'V' -%} - {% do view_relations.append(relation) %} - {%- endif -%} - {% endif %} - {%- endfor %} - {{ return(view_relations) }} -{%- endmacro %} - -{% macro teradata__create_tmp_tables_of_views(view_relations) -%} - {% set view_tmp_tables_mapping = {} %} - {%- for relation in view_relations -%} - {% set temp_relation_for_view = relation.identifier ~ '_tmp_viw_tbl' %} - {% set view_tmp_tables_mapping = view_tmp_tables_mapping.update({relation: temp_relation_for_view}) %} - {%- endfor %} - - {{ teradata__drop_tmp_tables_of_views(view_tmp_tables_mapping) }} - - {%- for relation, temp_relation_for_view in view_tmp_tables_mapping.items() %} - {% call statement('creating_table_from_view', fetch_result=False) %} - CREATE TABLE "{{ relation.schema }}"."{{ temp_relation_for_view }}" AS (SELECT * FROM "{{ relation.schema }}"."{{ relation.identifier }}") WITH NO DATA; - {% endcall %} - load_result('creating_table_from_view') - {%- endfor %} - {{ return(view_tmp_tables_mapping) }} -{%- endmacro %} - -{% macro teradata__drop_tmp_tables_of_views(view_tmp_tables_mapping) -%} - {% for relation, temp_table in view_tmp_tables_mapping.items() %} - {% call statement('drop_existing_table', fetch_result=False) %} - DROP table /*+ IF EXISTS */ "{{ relation.schema }}"."{{ temp_table }}"; - {% endcall %} - load_result('drop_existing_table') - {% endfor %} -{%- endmacro %} - - --Decompose the existing get_catalog() macro in order to minimize redundancy with body of get_catalog_relations(). This introduces some additional macros {% macro teradata__get_catalog(information_schema, schemas) -%} + {% set use_qvci = var("use_qvci", "false") | as_bool %} + {{ log("use_qvci set to : " ~ use_qvci) }} {%- call statement('catalog', fetch_result=True) -%} with tables as ( @@ -54,33 +12,28 @@ {{ teradata__get_catalog_columns_sql(information_schema) }} {{ teradata__get_catalog_schemas_where_clause_sql(schemas) }} ) - {{ teradata__get_catalog_results_sql(none) }} + {{ teradata__get_catalog_results_sql() }} {%- endcall -%} {{ return(load_result('catalog').table) }} {%- endmacro %} --A new macro, get_catalog_relations() which accepts a list of relations, rather than a list of schemas. {% macro teradata__get_catalog_relations(information_schema, relations) -%} - {% set view_relations = teradata__get_views_from_relations(relations) %} - {% set view_tmp_tables_mapping = teradata__create_tmp_tables_of_views(view_relations) %} - + {% set use_qvci = var("use_qvci", "false") | as_bool %} + {{ log("use_qvci set to : " ~ use_qvci) }} + {%- call statement('catalog', fetch_result=True) -%} with tables as ( {{ teradata__get_catalog_tables_sql(information_schema) }} - {{ teradata__get_catalog_relations_where_clause_sql(relations, view_tmp_tables_mapping) }} + {{ teradata__get_catalog_relations_where_clause_sql(relations) }} ), columns as ( {{ teradata__get_catalog_columns_sql(information_schema) }} - {{ teradata__get_catalog_relations_where_clause_sql(relations, view_tmp_tables_mapping) }} + {{ teradata__get_catalog_relations_where_clause_sql(relations) }} ) - {{ teradata__get_catalog_results_sql(view_tmp_tables_mapping) }} + {{ teradata__get_catalog_results_sql() }} {%- endcall -%} - - {% set catalog_table = load_result('catalog').table %} - - {{ teradata__drop_tmp_tables_of_views(view_tmp_tables_mapping) }} - - {{ return(catalog_table) }} + {{ return(load_result('catalog').table) }} {%- endmacro %} --get_catalog_tables_sql() copied straight from pre-existing get_catalog() everything you would normally fetch from DBC.tablesV @@ -90,7 +43,7 @@ DatabaseName AS table_schema, TableName AS table_name, CASE WHEN TableKind = 'T' THEN 'table' - WHEN TableKind = 'O' THEN 'table' + WHEN TableKind = 'O' THEN 'view' WHEN TableKind = 'V' THEN 'view' ELSE TableKind END AS table_type, @@ -98,8 +51,9 @@ FROM {{ information_schema_name(schema) }}.tablesV {%- endmacro %} ---get_catalog_columns_sql() copied straight from pre-existing get_catalog() everything you would normally fetch from DBC.ColumnsV +--get_catalog_columns_sql() copied straight from pre-existing get_catalog() everything you would normally fetch from DBC.ColumnsJQV and DBC.ColumnsV {% macro teradata__get_catalog_columns_sql(information_schema) -%} + {% set use_qvci = var("use_qvci", "false") | as_bool %} SELECT NULL AS table_database, DatabaseName AS table_schema, @@ -109,10 +63,15 @@ ColumnID AS column_index, ColumnType AS column_type, CommentString AS column_comment - FROM {{ information_schema_name(schema) }}.ColumnsV + + {% if use_qvci == True -%} + FROM {{ information_schema_name(schema) }}.ColumnsJQV + {% else -%} + FROM {{ information_schema_name(schema) }}.ColumnsV + {% endif -%} {%- endmacro %} -{% macro teradata__get_catalog_results_sql(view_tmp_tables_mapping) -%} +{% macro teradata__get_catalog_results_sql() -%} , columns_transformed AS ( SELECT @@ -175,26 +134,8 @@ SELECT columns_transformed.table_database, columns_transformed.table_schema, - {% if view_tmp_tables_mapping is not none and view_tmp_tables_mapping|length > 0 %} - CASE - {% for relation, temp_table in view_tmp_tables_mapping.items() %} - WHEN columns_transformed.table_name = '{{ temp_table }}' THEN '{{ relation.identifier }}' - {% endfor %} - ELSE columns_transformed.table_name - END AS table_name, - {% else %} - columns_transformed.table_name, - {% endif %} - {% if view_tmp_tables_mapping is not none and view_tmp_tables_mapping|length > 0 %} - CASE - {% for relation, temp_table in view_tmp_tables_mapping.items() %} - WHEN columns_transformed.table_name = '{{ temp_table }}' THEN 'view' - {% endfor %} - ELSE tables.table_type - END AS table_type, - {% else %} - tables.table_type, - {% endif %} + columns_transformed.table_name, + tables.table_type, columns_transformed.table_comment, tables.table_owner, columns_transformed.column_name, @@ -234,22 +175,13 @@ -- 3. else throw exception. Houston we do not have a something we can use. -- The result of the above is that dbt can, with "laser-precision" fetch metadata for only the relations it needs, rather than the superset of "all the relations in all the schemas in which dbt has relations". -{% macro teradata__get_catalog_relations_where_clause_sql(relations, view_tmp_tables_mapping) -%} +{% macro teradata__get_catalog_relations_where_clause_sql(relations) -%} where ( {%- for relation in relations -%} {% if relation.schema and relation.identifier %} ( upper("table_schema") = upper('{{ relation.schema }}') - {% if view_tmp_tables_mapping is not none and view_tmp_tables_mapping|length > 0 %} - {% set temp_table = view_tmp_tables_mapping[relation] %} - {% if not temp_table %} - and upper("table_name") = upper('{{ relation.identifier }}') - {% else %} - and upper("table_name") = upper('{{ temp_table }}') - {% endif %} - {% else %} - and upper("table_name") = upper('{{ relation.identifier }}') - {% endif %} + and upper("table_name") = upper('{{ relation.identifier }}') ) {% elif relation.schema %} ( diff --git a/dbt/include/teradata/macros/materializations/unit/unit.sql b/dbt/include/teradata/macros/materializations/unit/unit.sql index 7abcf85c..a55193ce 100644 --- a/dbt/include/teradata/macros/materializations/unit/unit.sql +++ b/dbt/include/teradata/macros/materializations/unit/unit.sql @@ -3,7 +3,33 @@ -- calling the macro set_query_band() which will set the query_band for this materialization as per the user_configuration {% do set_query_band() %} - {% set relations = materialization_unit_default() %} -- calling the default test materialization from dbt-core + {% set relations = [] %} + + {% set expected_rows = config.get('expected_rows') %} + {% set expected_sql = config.get('expected_sql') %} + {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %} + + {%- set target_relation = this.incorporate(type='table') -%} + {%- set temp_relation = make_temp_relation(target_relation)-%} + {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %} + {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%} + {%- set column_name_to_data_types = {} -%} + {%- for column in columns_in_relation -%} + {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%} + {%- endfor -%} + + {% if not expected_sql %} + {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %} + {% endif %} + {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %} + + {% call statement('main', fetch_result=True) -%} + + {{ unit_test_sql }} + + {%- endcall %} + + {% do adapter.drop_relation(temp_relation) %} {{ return({'relations': relations}) }} diff --git a/test/catalog/with_qvci/.gitignore b/test/catalog/with_qvci/.gitignore new file mode 100644 index 00000000..da5e1839 --- /dev/null +++ b/test/catalog/with_qvci/.gitignore @@ -0,0 +1,3 @@ +logs/ +target/ +dbt_modules/ diff --git a/test/catalog/with_qvci/data/test_data_types.csv b/test/catalog/with_qvci/data/test_data_types.csv new file mode 100644 index 00000000..705922b6 --- /dev/null +++ b/test/catalog/with_qvci/data/test_data_types.csv @@ -0,0 +1,11 @@ +id,timestamp_column,date_column,float_column,integer_column,boolean_column +1,2022-01-13 13:04:34,2022-01-13,10.03,10234234,true +2,2022-02-13 13:14:34,2022-02-13,11.03,10234234,true +3,2022-03-13 13:24:34,2022-03-13,12.03,10234234,true +4,2022-04-13 13:34:34,2022-04-13,13.03,10234234,true +5,2022-05-13 13:44:34,2022-05-13,14.03,10234234,true +6,2022-06-13 13:54:34,2022-06-13,15.03,10234234,false +7,2022-07-13 14:04:34,2022-07-13,16.03,10234234,false +8,2022-08-13 14:24:34,2022-08-13,17.03,10234234,false +9,2022-09-13 14:54:34,2022-09-13,18.0,10234234,false +0,2022-10-13 16:24:34,2022-10-13,19.0343,10234234,false diff --git a/test/catalog/with_qvci/dbt_project.yml b/test/catalog/with_qvci/dbt_project.yml new file mode 100644 index 00000000..4d6bdebf --- /dev/null +++ b/test/catalog/with_qvci/dbt_project.yml @@ -0,0 +1,41 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'teradata' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'teradata' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_modules" + +#on-run-start: +# - "{{ install_dbt_drop_relation_if_exists() }}" + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ directory +# as tables. These settings can be overridden in the individual model files +# using the `{{ config(...) }}` macro. +models: + teradata: + +materialized: view + +vars: + use_qvci: true diff --git a/test/catalog/with_qvci/models/sources.yml b/test/catalog/with_qvci/models/sources.yml new file mode 100644 index 00000000..4ecf1ee5 --- /dev/null +++ b/test/catalog/with_qvci/models/sources.yml @@ -0,0 +1,7 @@ +version: 2 +sources: + - name: alias_source_schema + schema: dbt_catalog_test + tables: + - name: alias_source_table + identifier: test_table diff --git a/test/catalog/with_qvci/models/table_from_source.sql b/test/catalog/with_qvci/models/table_from_source.sql new file mode 100644 index 00000000..526a93cd --- /dev/null +++ b/test/catalog/with_qvci/models/table_from_source.sql @@ -0,0 +1,6 @@ +{{ + config( + materialized='table' + ) +}} +SELECT * FROM {{ source('alias_source_schema', 'alias_source_table') }} diff --git a/test/catalog/with_qvci/models/view_from_source.sql b/test/catalog/with_qvci/models/view_from_source.sql new file mode 100644 index 00000000..f148ff6f --- /dev/null +++ b/test/catalog/with_qvci/models/view_from_source.sql @@ -0,0 +1,6 @@ +{{ + config( + materialized='view' + ) +}} +SELECT * FROM {{ source('alias_source_schema', 'alias_source_table') }} diff --git a/test/catalog/with_qvci/models/view_from_test_data_types.sql b/test/catalog/with_qvci/models/view_from_test_data_types.sql new file mode 100644 index 00000000..4ce36d31 --- /dev/null +++ b/test/catalog/with_qvci/models/view_from_test_data_types.sql @@ -0,0 +1,6 @@ +{{ + config( + materialized='view' + ) +}} +SELECT * FROM {{ ref('test_data_types') }} diff --git a/test/catalog/with_qvci/run.sh b/test/catalog/with_qvci/run.sh new file mode 100755 index 00000000..e9bc04d9 --- /dev/null +++ b/test/catalog/with_qvci/run.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -e + +dbt -d seed --target dbt_catalog_test_qvci --full-refresh +dbt -d run --target dbt_catalog_test_qvci +dbt -d docs generate --target dbt_catalog_test_qvci + + +assertJsonElementEquals() { + VALUE_IN_JSON=`cat target/catalog.json| jq -r $2` + if [ "$1" == "$VALUE_IN_JSON" ]; then + echo "Assert successful for '$1' and '$2'" + return + else + echo "Assert failed for $2 . Was $VALUE_IN_JSON, but expected $1." + exit 1 + fi +} + +# assert on target/catalog.json +assertJsonElementEquals CHARACTER '.nodes["seed.teradata.test_table"].columns.animal.type' +assertJsonElementEquals CHARACTER '.nodes["seed.teradata.test_table"].columns.superpower.type' +assertJsonElementEquals INTEGER '.nodes["seed.teradata.test_table"].columns.magic_index.type' + +assertJsonElementEquals INTEGER '.nodes["seed.teradata.test_data_types"].columns.id.type' +assertJsonElementEquals TIMESTAMP '.nodes["seed.teradata.test_data_types"].columns.timestamp_column.type' +assertJsonElementEquals DATE '.nodes["seed.teradata.test_data_types"].columns.date_column.type' +assertJsonElementEquals 'DOUBLE PRECISION' '.nodes["seed.teradata.test_data_types"].columns.float_column.type' +assertJsonElementEquals INTEGER '.nodes["seed.teradata.test_data_types"].columns.integer_column.type' +assertJsonElementEquals BYTEINT '.nodes["seed.teradata.test_data_types"].columns.boolean_column.type' + +assertJsonElementEquals CHARACTER '.nodes["model.teradata.table_from_source"].columns.animal.type' +assertJsonElementEquals CHARACTER '.nodes["model.teradata.table_from_source"].columns.superpower.type' +assertJsonElementEquals INTEGER '.nodes["model.teradata.table_from_source"].columns.magic_index.type' + +assertJsonElementEquals CHARACTER '.nodes["model.teradata.view_from_source"].columns.animal.type' +assertJsonElementEquals CHARACTER '.nodes["model.teradata.view_from_source"].columns.superpower.type' +assertJsonElementEquals INTEGER '.nodes["model.teradata.view_from_source"].columns.magic_index.type' + +assertJsonElementEquals INTEGER '.nodes["model.teradata.view_from_test_data_types"].columns.id.type' +assertJsonElementEquals TIMESTAMP '.nodes["model.teradata.view_from_test_data_types"].columns.timestamp_column.type' +assertJsonElementEquals DATE '.nodes["model.teradata.view_from_test_data_types"].columns.date_column.type' +assertJsonElementEquals 'DOUBLE PRECISION' '.nodes["model.teradata.view_from_test_data_types"].columns.float_column.type' +assertJsonElementEquals INTEGER '.nodes["model.teradata.view_from_test_data_types"].columns.integer_column.type' +assertJsonElementEquals BYTEINT '.nodes["model.teradata.view_from_test_data_types"].columns.boolean_column.type' + diff --git a/test/catalog/with_qvci/seeds/test_data_types.csv b/test/catalog/with_qvci/seeds/test_data_types.csv new file mode 100644 index 00000000..705922b6 --- /dev/null +++ b/test/catalog/with_qvci/seeds/test_data_types.csv @@ -0,0 +1,11 @@ +id,timestamp_column,date_column,float_column,integer_column,boolean_column +1,2022-01-13 13:04:34,2022-01-13,10.03,10234234,true +2,2022-02-13 13:14:34,2022-02-13,11.03,10234234,true +3,2022-03-13 13:24:34,2022-03-13,12.03,10234234,true +4,2022-04-13 13:34:34,2022-04-13,13.03,10234234,true +5,2022-05-13 13:44:34,2022-05-13,14.03,10234234,true +6,2022-06-13 13:54:34,2022-06-13,15.03,10234234,false +7,2022-07-13 14:04:34,2022-07-13,16.03,10234234,false +8,2022-08-13 14:24:34,2022-08-13,17.03,10234234,false +9,2022-09-13 14:54:34,2022-09-13,18.0,10234234,false +0,2022-10-13 16:24:34,2022-10-13,19.0343,10234234,false diff --git a/test/catalog/with_qvci/seeds/test_table.csv b/test/catalog/with_qvci/seeds/test_table.csv new file mode 100644 index 00000000..d0de5750 --- /dev/null +++ b/test/catalog/with_qvci/seeds/test_table.csv @@ -0,0 +1,3 @@ +animal,superpower,magic_index +unicorn,magic,100 +pegasus,flying,500 diff --git a/test/catalog/without_qvci/run.sh b/test/catalog/without_qvci/run.sh index 7a1d9642..c763a9f1 100755 --- a/test/catalog/without_qvci/run.sh +++ b/test/catalog/without_qvci/run.sh @@ -33,14 +33,14 @@ assertJsonElementEquals CHARACTER '.nodes["model.teradata.table_from_source"].co assertJsonElementEquals CHARACTER '.nodes["model.teradata.table_from_source"].columns.superpower.type' assertJsonElementEquals INTEGER '.nodes["model.teradata.table_from_source"].columns.magic_index.type' -assertJsonElementEquals CHARACTER '.nodes["model.teradata.view_from_source"].columns.animal.type' -assertJsonElementEquals CHARACTER '.nodes["model.teradata.view_from_source"].columns.superpower.type' -assertJsonElementEquals INTEGER '.nodes["model.teradata.view_from_source"].columns.magic_index.type' - -assertJsonElementEquals INTEGER '.nodes["model.teradata.view_from_test_data_types"].columns.id.type' -assertJsonElementEquals TIMESTAMP '.nodes["model.teradata.view_from_test_data_types"].columns.timestamp_column.type' -assertJsonElementEquals DATE '.nodes["model.teradata.view_from_test_data_types"].columns.date_column.type' -assertJsonElementEquals 'DOUBLE PRECISION' '.nodes["model.teradata.view_from_test_data_types"].columns.float_column.type' -assertJsonElementEquals INTEGER '.nodes["model.teradata.view_from_test_data_types"].columns.integer_column.type' -assertJsonElementEquals BYTEINT '.nodes["model.teradata.view_from_test_data_types"].columns.boolean_column.type' +assertJsonElementEquals 'N/A' '.nodes["model.teradata.view_from_source"].columns.animal.type' +assertJsonElementEquals 'N/A' '.nodes["model.teradata.view_from_source"].columns.superpower.type' +assertJsonElementEquals 'N/A' '.nodes["model.teradata.view_from_source"].columns.magic_index.type' + +assertJsonElementEquals 'N/A' '.nodes["model.teradata.view_from_test_data_types"].columns.id.type' +assertJsonElementEquals 'N/A' '.nodes["model.teradata.view_from_test_data_types"].columns.timestamp_column.type' +assertJsonElementEquals 'N/A' '.nodes["model.teradata.view_from_test_data_types"].columns.date_column.type' +assertJsonElementEquals 'N/A' '.nodes["model.teradata.view_from_test_data_types"].columns.float_column.type' +assertJsonElementEquals 'N/A' '.nodes["model.teradata.view_from_test_data_types"].columns.integer_column.type' +assertJsonElementEquals 'N/A' '.nodes["model.teradata.view_from_test_data_types"].columns.boolean_column.type' diff --git a/tests/functional/adapter/test_unit_tests.py b/tests/functional/adapter/test_unit_tests.py index 28fb6e67..10a7295d 100644 --- a/tests/functional/adapter/test_unit_tests.py +++ b/tests/functional/adapter/test_unit_tests.py @@ -32,6 +32,16 @@ class TestTestingTypesTeradata(BaseUnitTestingTypes): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "test_testing_types", + "models": { + "test_testing_types": { + "materialized": "table" + } + } + } @pytest.fixture def data_types(self): # sql_value, yaml_value @@ -51,12 +61,30 @@ def data_types(self): class TestUnitTestCaseInsensitivityTeradata(BaseUnitTestCaseInsensivity): - pass + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "test_case_insensitivity", + "models":{ + "test_case_insensitivity":{ + "materialized": "table" + } + } + } class TestUnitTestInvalidInput(BaseUnitTestInvalidInput): - pass + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "test_unit_test_invalid_input", + "models": { + "test_unit_test_invalid_input": { + "materialized": "table" + } + } + } class TestSafeCast():