Skip to content

Commit

Permalink
Prefer staging file formats when staging
Browse files Browse the repository at this point in the history
  • Loading branch information
steinitzu committed Oct 25, 2023
1 parent 1c1f564 commit 99c33c3
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 2 deletions.
2 changes: 1 addition & 1 deletion dlt/normalize/normalize.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def w_normalize_files(

def _get_load_storage(file_format: TLoaderFileFormat) -> LoadStorage:
# TODO: capabilities.supported_*_formats can be None, it should have defaults
supported_formats = list(set(destination_caps.supported_loader_file_formats or []) | set(destination_caps.supported_staging_file_formats or []))
supported_formats = destination_caps.supported_loader_file_formats or []
if file_format == "parquet":
if file_format in supported_formats:
supported_formats.append("arrow") # TODO: Hack to make load storage use the correct writer
Expand Down
5 changes: 4 additions & 1 deletion dlt/pipeline/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -954,7 +954,7 @@ def _get_destination_capabilities(self) -> DestinationCapabilitiesContext:
)
return self.destination.capabilities()

def _get_staging_capabilities(self) -> DestinationCapabilitiesContext:
def _get_staging_capabilities(self) -> Optional[DestinationCapabilitiesContext]:
return self.staging.capabilities() if self.staging is not None else None

def _validate_pipeline_name(self) -> None:
Expand Down Expand Up @@ -1016,6 +1016,9 @@ def _maybe_destination_capabilities(self, loader_file_format: TLoaderFileFormat
DestinationReference.to_name(self.destination),
DestinationReference.to_name(self.staging) if self.staging else None,
destination_caps, stage_caps, loader_file_format)
caps.supported_loader_file_formats = (
destination_caps.supported_staging_file_formats if stage_caps else None
) or destination_caps.supported_loader_file_formats
yield caps
finally:
if injected_caps:
Expand Down

0 comments on commit 99c33c3

Please sign in to comment.