Merge pull request #205 from Teradata/revert-202-mt255026_view_metadata

Revert "IDE-24702: Replacing QVCI in dbt-teradata"

.github/workflows/ci-integration-tests-csae.yml

@@ -178,11 +178,26 @@ jobs:
                 timeout_seconds: 300
                 priority: interactive
                 retries: 1
+              dbt_catalog_test_qvci:
+                type: teradata
+                host: $DBT_TERADATA_SERVER_NAME_QVCI
+                user: $DBT_TERADATA_USERNAME_QVCI
+                password: $DBT_TERADATA_PASSWORD_QVCI
+                logmech: TD2
+                schema: dbt_catalog_test
+                tmode: ANSI
+                threads: 1
+                timeout_seconds: 300
+                priority: interactive
+                retries: 1
           EOF
         env:
           DBT_TERADATA_SERVER_NAME: ${{ steps.create-csae-environments.outputs.teradata-server-name }}
           DBT_TERADATA_USERNAME: dbc
           DBT_TERADATA_PASSWORD: ${{ secrets.CSAE_ENV_PASSWORD }}
+          DBT_TERADATA_SERVER_NAME_QVCI: ${{ steps.create-csae-environments.outputs.teradata-server-name-qvci }}
+          DBT_TERADATA_USERNAME_QVCI: dbc
+          DBT_TERADATA_PASSWORD_QVCI: ${{ secrets.CSAE_ENV_PASSWORD }}
 
       - name: Run performance tests with ANSI
         run: |
@@ -191,6 +206,8 @@ jobs:
 
       - name: Run catalog tests with ANSI
         run: |
+          # cd $GITHUB_WORKSPACE/test/catalog/with_qvci
+          # ./run.sh
           cd $GITHUB_WORKSPACE/test/catalog/without_qvci
           ./run.sh
 
@@ -224,6 +241,18 @@ jobs:
                 timeout_seconds: 300
                 priority: interactive
                 retries: 1
+              dbt_catalog_test_qvci:
+                type: teradata
+                host: $DBT_TERADATA_SERVER_NAME_QVCI
+                user: $DBT_TERADATA_USERNAME_QVCI
+                password: $DBT_TERADATA_PASSWORD_QVCI
+                logmech: TD2
+                schema: dbt_catalog_test
+                tmode: TERA
+                threads: 1
+                timeout_seconds: 300
+                priority: interactive
+                retries: 1
               dbt_test_valid_history:
                 type: teradata
                 host: $DBT_TERADATA_SERVER_NAME
@@ -241,6 +270,9 @@ jobs:
           DBT_TERADATA_SERVER_NAME: ${{ steps.create-csae-environments.outputs.teradata-server-name }}
           DBT_TERADATA_USERNAME: dbc
           DBT_TERADATA_PASSWORD: ${{ secrets.CSAE_ENV_PASSWORD }}
+          DBT_TERADATA_SERVER_NAME_QVCI: ${{ steps.create-csae-environments.outputs.teradata-server-name-qvci }}
+          DBT_TERADATA_USERNAME_QVCI: dbc
+          DBT_TERADATA_PASSWORD_QVCI: ${{ secrets.CSAE_ENV_PASSWORD }}
 
       - name: Run performance tests with TERA
         run: |
@@ -249,6 +281,8 @@ jobs:
 
       - name: Run catalog tests with TERA
         run: |
+          # cd $GITHUB_WORKSPACE/test/catalog/with_qvci
+          # ./run.sh
           cd $GITHUB_WORKSPACE/test/catalog/without_qvci
           ./run.sh
 

.github/workflows/scripts/createTestEnvironments.sh

@@ -44,6 +44,21 @@ createRegularEnv() {
   echo "teradata-server-name=$TERADATA_SERVER_NAME" >> $GITHUB_OUTPUT
 }
 
+createEnvWithQVCI() {
+  local RESULT=$(curl -s -L 'https://api.clearscape.teradata.com/environments' \
+  -H "Authorization: Bearer $CSAE_TOKEN" \
+  -H 'Content-Type: application/json' \
+  --data-raw "{
+      \"name\": \"$CSAE_ENV_NAME-qvci\",
+      \"region\": \"us-central\",
+      \"password\": \"$CSAE_ENV_PASSWORD\",
+      \"startupScript\": \"#!/bin/bash\n\nfunction withRetry {\n  local RETRIES=\$1; shift 1\n  local SLEEP=\$1; shift 1\n  for i in \$(seq 1 \$RETRIES)\n  do\n    echo \\\"Attempt \$i: Running command \$@\\\"\n    \$@ && s=0 && break || s=\$? && sleep \$SLEEP\n  done\n  return \$s\n}\n\nfunction ensureVantageIsUp {\n  pdestate -a\n  pdestate -a | grep \\\"DBS state is [45]\\\"\n}\n\ndbscontrol << EOF\nm i 551=false\nW\nEOF\ntpareset -y changing dbscontrol\nwithRetry 40 2 ensureVantageIsUp\"
+  }")
+  local TERADATA_SERVER_NAME=$(echo $RESULT | jq -r '.dnsName')
+  echo "teradata-server-name-qvci=$TERADATA_SERVER_NAME" >> $GITHUB_OUTPUT
+}
+
 background createRegularEnv
+background createEnvWithQVCI
 
 reap

.github/workflows/scripts/deleteTestEnvironments.sh

@@ -35,7 +35,13 @@ deleteRegularEnv() {
     -H "Authorization: Bearer $CSAE_TOKEN"
 }
 
+deleteEnvWithQVCI() {
+  curl -s -L --request DELETE "https://api.clearscape.teradata.com/environments/$CSAE_ENV_NAME-qvci" \
+    -H "Authorization: Bearer $CSAE_TOKEN"
+}
+
 background deleteRegularEnv
+background deleteEnvWithQVCI
 
 reap
 
@@ -76,6 +82,12 @@ deleteRegularEnv() {
     -H "Authorization: Bearer $CSAE_TOKEN"
 }
 
+deleteEnvWithQVCI() {
+  curl -s -L --request DELETE "https://api.clearscape.teradata.com/environments/$CSAE_ENV_NAME-qvci" \
+    -H "Authorization: Bearer $CSAE_TOKEN"
+}
+
 background deleteRegularEnv
+background deleteEnvWithQVCI
 
 reap

README.md

@@ -41,7 +41,7 @@ my-teradata-db-profile:
       tmode: ANSI
 ```
 
-At a minimum, you need to specify `host`, `user`, `password`, `schema` (database).
+At a minimum, you need to specify `host`, `user`, `password`, `schema` (database), `tmode`.
 
 ## Python compatibility
 
@@ -343,6 +343,31 @@ All dbt commands are supported.
 
 ### Custom configurations
 
+#### General
+
+* *Enable view column types in docs* -  Teradata Vantage has a dbscontrol configuration flag called `DisableQVCI` (QVCI - Queryable View Column Index). This flag instructs the database to build `DBC.ColumnsJQV` with view column type definitions. 
+
+    > :information_source: Existing customers, please see [KB0022230](https://support.teradata.com/knowledge?id=kb_article_view&sys_kb_id=d066248b1b0000187361c8415b4bcb48) for more information about enabling QVCI.
+
+  To enable this functionality you need to:
+  1. Enable QVCI mode in Vantage. Use `dbscontrol` utility and then restart Teradata. Run these commands as a privileged user on a Teradata node:
+      ```bash
+      # option 551 is DisableQVCI. Setting it to false enables QVCI.
+      dbscontrol << EOF
+      M internal 551=false
+      W
+      EOF
+
+      # restart Teradata
+      tpareset -y Enable QVCI
+      ```
+  2. Instruct `dbt` to use `QVCI` mode. Include the following variable in your `dbt_project.yml`:
+      ```yaml
+      vars:
+        use_qvci: true
+      ```
+      For example configuration, see `test/catalog/with_qvci/dbt_project.yml`.
+
 #### Models
 
 ##### Table
@@ -754,6 +779,15 @@ If no query_band is set by user, default query_band will come in play that is :
 * Unit testing is supported in dbt-teradata, allowing users to write and execute unit tests using the dbt test command.
   * For detailed guidance, refer to the dbt documentation.
 
+* QVCI must be enabled in the database to run unit tests for views.
+  * Additional details on enabling QVCI can be found in the General section.
+  * Without QVCI enabled, unit test support for views will be limited.
+  * Users might encounter the following database error when testing views without QVCI enabled:
+    ```
+    * [Teradata Database] [Error 3706] Syntax error: Data Type "N" does not match a Defined Type name.
+    ```
+
+
 ## Credits
 
 The adapter was originally created by [Doug Beatty](https://github.com/dbeatty10). Teradata took over the adapter in January 2022. We are grateful to Doug for founding the project and accelerating the integration of dbt + Teradata.

dbt/include/teradata/macros/adapters.sql

@@ -118,23 +118,8 @@
 {% endmacro %}
 
 {% macro teradata__get_columns_in_relation(relation) -%}
-    {% call statement('check_table_or_view', fetch_result=True) %}
-        SELECT TableKind FROM DBC.TablesV WHERE DatabaseName = '{{ relation.schema }}' AND TableName = '{{ relation.identifier }}'
-    {% endcall %}
-
-    {% set table_kind = load_result('check_table_or_view').table.columns['TableKind'].values()[0] | trim %}
-
-    {%- if table_kind == 'V' -%}
-        {% set temp_relation_for_view = relation.identifier ~ '_tmp_viw_tbl' %}
-        {% call statement('drop_existing_table', fetch_result=False) %}
-            DROP table /*+ IF EXISTS */ "{{ relation.schema }}"."{{ temp_relation_for_view }}";
-        {% endcall %}
-        load_result('drop_existing_table')
-        {% call statement('creating_table_from_view', fetch_result=False) %}
-            CREATE TABLE "{{ relation.schema }}"."{{ temp_relation_for_view }}" AS (SELECT * FROM "{{ relation.schema }}"."{{ relation.identifier }}") WITH NO DATA;
-        {% endcall %}
-        load_result('creating_table_from_view')
-    {% endif %}
+    {% set use_qvci = var("use_qvci", "false") | as_bool %}
+     {{ log("use_qvci set to : " ~ use_qvci) }}
     {% call statement('get_columns_in_relation', fetch_result=True) %}
     SELECT
       ColumnsV.ColumnName AS "column",
@@ -193,41 +178,31 @@
       NULL AS table_database,
       ColumnsV.DatabaseName AS table_schema,
       ColumnsV.TableName AS table_name,
-      {%- if table_kind == 'V' -%}
-        'view' AS table_type,
-      {%- else -%}
       CASE WHEN TablesV.TableKind = 'T' THEN 'table'
         WHEN TablesV.TableKind = 'O' THEN 'table'
+        WHEN TablesV.TableKind = 'V' THEN 'view'
         ELSE TablesV.TableKind
       END AS table_type,
-      {%- endif -%}
       ColumnsV.ColumnID AS column_index
     FROM
+    {% if use_qvci == True -%}
+      {{ information_schema_name(relation.schema) }}.ColumnsJQV AS ColumnsV
+    {% else %}
       {{ information_schema_name(relation.schema) }}.ColumnsV
+    {%- endif %}
     LEFT OUTER JOIN {{ information_schema_name(relation.schema) }}.TablesV
       ON ColumnsV.DatabaseName = TablesV.DatabaseName
       AND ColumnsV.TableName = TablesV.TableName
     WHERE
       TablesV.TableKind IN ('T', 'V', 'O')
       AND ColumnsV.DatabaseName = '{{ relation.schema }}' (NOT CASESPECIFIC)
-    {%- if table_kind == 'V' -%}
-      AND ColumnsV.TableName = '{{ temp_relation_for_view }}' (NOT CASESPECIFIC)
-    {%- else -%}
       AND ColumnsV.TableName = '{{ relation.identifier }}' (NOT CASESPECIFIC)
-    {%- endif -%}
     ORDER BY
       ColumnsV.ColumnID
     {% endcall %}
 
     {% set table = load_result('get_columns_in_relation').table %}
 
-    {%- if table_kind == 'V' -%}
-        {% call statement('drop_table_from_view', fetch_result=False) %}
-            DROP table /*+ IF EXISTS */ "{{ relation.schema }}"."{{ temp_relation_for_view }}";
-        {% endcall %}
-        load_result('drop_table_from_view')
-    {% endif %}
-
     {{ return(sql_convert_columns_in_relation(table)) }}
 {% endmacro %}