PR initial review comments addressed (#36)

Teradata · May 2, 2024 · 73ca455 · 73ca455
1 parent 1596bee
commit 73ca455
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 46 deletions.
diff --git a/airflow/providers/teradata/transfers/azure_blob_to_teradata.py b/airflow/providers/teradata/transfers/azure_blob_to_teradata.py
@@ -90,16 +90,6 @@ def execute(self, context: Context) -> None:
         try:
             teradata_hook.run(sql, True)
         except Exception as ex:
-            # Handling permission issue errors
-            if "Error 3524" in str(ex):
-                self.log.error("The user does not have CREATE TABLE access in teradata")
-                raise
-            if "Error 9134" in str(ex):
-                self.log.error(
-                    "There is an issue with the transfer operation. Please validate azure and "
-                    "teradata connection details."
-                )
-                raise
-            self.log.error("Issue occurred at Teradata: %s", str(ex))
+            self.log.error(str(ex))
             raise
         self.log.info("The transfer of data from Azure Blob to Teradata was successful")
diff --git a/airflow/providers/teradata/transfers/s3_to_teradata.py b/airflow/providers/teradata/transfers/s3_to_teradata.py
@@ -95,16 +95,6 @@ def execute(self, context: Context) -> None:
         try:
             teradata_hook.run(sql, True)
         except Exception as ex:
-            # Handling permission issue errors
-            if "Error 3524" in str(ex):
-                self.log.error("The user does not have CREATE TABLE access in teradata")
-                raise
-            if "Error 9134" in str(ex):
-                self.log.error(
-                    "There is an issue with the transfer operation. Please validate s3 and "
-                    "teradata connection details."
-                )
-                raise
-            self.log.error("Issue occurred at Teradata: %s", str(ex))
+            self.log.error(str(ex))
             raise
         self.log.info("The transfer of data from S3 to Teradata was successful")
diff --git a/tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py b/tests/system/providers/teradata/example_azure_blob_to_teradata_transfer.py
@@ -56,7 +56,7 @@
     # [START azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_blob_to_teradata_csv]
     transfer_data_csv = AzureBlobStorageToTeradataOperator(
         task_id="transfer_data_blob_to_teradata_csv",
-        blob_source_key="/az/akiaxox5jikeotfww4ul.blob.core.windows.net/td-usgs/CSVDATA/",
+        blob_source_key="/az/akiaxox5jikeotfww4ul.blob.core.windows.net/td-usgs/CSVDATA/09380000/2018/06/",
         teradata_table="example_blob_teradata_csv",
         azure_conn_id="wasb_default",
         teradata_conn_id="teradata_default",
@@ -84,7 +84,7 @@
     # [START azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_blob_to_teradata_json]
     transfer_data_json = AzureBlobStorageToTeradataOperator(
         task_id="transfer_data_blob_to_teradata_json",
-        blob_source_key="/az/akiaxox5jikeotfww4ul.blob.core.windows.net/td-usgs/JSONDATA/",
+        blob_source_key="/az/akiaxox5jikeotfww4ul.blob.core.windows.net/td-usgs/JSONDATA/09380000/2018/06/",
         teradata_table="example_blob_teradata_json",
         azure_conn_id="wasb_default",
         teradata_conn_id="teradata_default",
@@ -112,7 +112,7 @@
     # [START azure_blob_to_teradata_transfer_operator_howto_guide_transfer_data_blob_to_teradata_parquet]
     transfer_data_parquet = AzureBlobStorageToTeradataOperator(
         task_id="transfer_data_blob_to_teradata_parquet",
-        blob_source_key="/az/akiaxox5jikeotfww4ul.blob.core.windows.net/td-usgs/PARQUETDATA/",
+        blob_source_key="/az/akiaxox5jikeotfww4ul.blob.core.windows.net/td-usgs/PARQUETDATA/09394500/2018/06/",
         teradata_table="example_blob_teradata_parquet",
         azure_conn_id="wasb_default",
         teradata_conn_id="teradata_default",
@@ -139,15 +139,15 @@
     # [END azure_blob_to_teradata_transfer_operator_howto_guide_drop_table_parquet]
 
     (
-        transfer_data_csv,
-        transfer_data_json,
-        transfer_data_parquet,
-        read_data_table_csv,
-        read_data_table_json,
-        read_data_table_parquet,
-        drop_table_csv,
-        drop_table_json,
-        drop_table_parquet,
+        transfer_data_csv >>
+        transfer_data_json >>
+        transfer_data_parquet >>
+        read_data_table_csv >>
+        read_data_table_json >>
+        read_data_table_parquet >>
+        drop_table_csv >>
+        drop_table_json >>
+        drop_table_parquet
     )
     # [END azure_blob_to_teradata_transfer_operator_howto_guide]
 

diff --git a/tests/system/providers/teradata/example_s3_to_teradata_transfer.py b/tests/system/providers/teradata/example_s3_to_teradata_transfer.py
@@ -57,7 +57,7 @@
     # [START s3_to_teradata_transfer_operator_howto_guide_transfer_data_s3_to_teradata_csv]
     transfer_data_csv = S3ToTeradataOperator(
         task_id="transfer_data_s3_to_teradata_csv",
-        s3_source_key="/s3/td-usgs-public.s3.amazonaws.com/CSVDATA/",
+        s3_source_key="/s3/td-usgs-public.s3.amazonaws.com/CSVDATA/09394500/2018/06/",
         teradata_table="example_s3_teradata_csv",
         aws_conn_id="aws_default",
         teradata_conn_id="teradata_default",
@@ -85,7 +85,7 @@
     # [START s3_to_teradata_transfer_operator_howto_guide_transfer_data_s3_to_teradata_json]
     transfer_data_json = S3ToTeradataOperator(
         task_id="transfer_data_s3_to_teradata_json",
-        s3_source_key="/s3/td-usgs-public.s3.amazonaws.com/JSONDATA/",
+        s3_source_key="/s3/td-usgs-public.s3.amazonaws.com/JSONDATA/09394500/2018/06/",
         teradata_table="example_s3_teradata_json",
         aws_conn_id="aws_default",
         teradata_conn_id="teradata_default",
@@ -113,7 +113,7 @@
     # [START s3_to_teradata_transfer_operator_howto_guide_transfer_data_s3_to_teradata_parquet]
     transfer_data_parquet = S3ToTeradataOperator(
         task_id="transfer_data_s3_to_teradata_parquet",
-        s3_source_key="/s3/td-usgs-public.s3.amazonaws.com/PARQUETDATA/",
+        s3_source_key="/s3/td-usgs-public.s3.amazonaws.com/PARQUETDATA/09394500/2018/06/",
         teradata_table="example_s3_teradata_parquet",
         aws_conn_id="aws_default",
         teradata_conn_id="teradata_default",
@@ -139,15 +139,15 @@
     )
     # [END s3_to_teradata_transfer_operator_howto_guide_drop_table_parquet]
     (
-        transfer_data_csv,
-        transfer_data_json,
-        transfer_data_parquet,
-        read_data_table_csv,
-        read_data_table_json,
-        read_data_table_parquet,
-        drop_table_csv,
-        drop_table_json,
-        drop_table_parquet,
+        transfer_data_csv >>
+        transfer_data_json >>
+        transfer_data_parquet >>
+        read_data_table_csv >>
+        read_data_table_json >>
+        read_data_table_parquet >>
+        drop_table_csv >>
+        drop_table_json >>
+        drop_table_parquet
     )
     # [END s3_to_teradata_transfer_operator_howto_guide]