From 0f329648fbd0279efc5f9a2aa7342c7b20ca2c0c Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Tue, 30 Jan 2024 15:50:20 -0500 Subject: [PATCH] stage_name is not relevant --- dlt/destinations/impl/databricks/databricks.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dlt/destinations/impl/databricks/databricks.py b/dlt/destinations/impl/databricks/databricks.py index 6773714d59..d1b0666f57 100644 --- a/dlt/destinations/impl/databricks/databricks.py +++ b/dlt/destinations/impl/databricks/databricks.py @@ -106,7 +106,6 @@ def __init__( table_name: str, load_id: str, client: DatabricksSqlClient, - stage_name: Optional[str] = None, staging_credentials: Optional[CredentialsConfiguration] = None, ) -> None: file_name = FileStorage.get_file_name_from_file_path(file_path) @@ -129,11 +128,8 @@ def __init__( if bucket_path: bucket_url = urlparse(bucket_path) bucket_scheme = bucket_url.scheme - # referencing an external s3/azure stage does not require explicit credentials - if bucket_scheme in ["s3", "az", "abfs", "gc", "gcs"] and stage_name: - from_clause = f"FROM ('{bucket_path}')" # referencing an staged files via a bucket URL requires explicit AWS credentials - elif ( + if ( bucket_scheme == "s3" and staging_credentials and isinstance(staging_credentials, AwsCredentialsWithoutDefaults) @@ -228,7 +224,6 @@ def start_file_load(self, table: TTableSchema, file_path: str, load_id: str) -> table["name"], load_id, self.sql_client, - stage_name=self.config.stage_name, staging_credentials=( self.config.staging_config.credentials if self.config.staging_config else None ),