diff --git a/Makefile b/Makefile index 1059cfdf0a..d9d92ec799 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ dev: has-poetry lint: ./check-package.sh - poetry run black ./ --diff --exclude=".*syntax_error.py|\.venv.*" + poetry run black ./ --diff --exclude=".*syntax_error.py|\.venv.*|_storage/.*" # poetry run isort ./ --diff poetry run mypy --config-file mypy.ini dlt tests poetry run flake8 --max-line-length=200 dlt @@ -56,7 +56,7 @@ lint: # $(MAKE) lint-security format: - poetry run black ./ --exclude=".*syntax_error.py|\.venv.*" + poetry run black ./ --exclude=".*syntax_error.py|\.venv.*|_storage/.*" # poetry run isort ./ test-and-lint-snippets: diff --git a/dlt/pipeline/pipeline.py b/dlt/pipeline/pipeline.py index d4ccdafcec..836442f5bb 100644 --- a/dlt/pipeline/pipeline.py +++ b/dlt/pipeline/pipeline.py @@ -433,7 +433,7 @@ def normalize( self, "normalize", n_ex, normalize.get_normalize_info() ) from n_ex - @with_runtime_trace() + @with_runtime_trace(send_state=True) @with_schemas_sync @with_state_sync() @with_config_section((known_sections.LOAD,)) @@ -486,7 +486,7 @@ def load( except Exception as l_ex: raise PipelineStepFailed(self, "load", l_ex, self._get_load_info(load)) from l_ex - @with_runtime_trace(send_state=True) + @with_runtime_trace() @with_config_section(("run",)) def run( self, diff --git a/dlt/pipeline/platform.py b/dlt/pipeline/platform.py index 5ebaaf662e..c8014d5ae7 100644 --- a/dlt/pipeline/platform.py +++ b/dlt/pipeline/platform.py @@ -15,7 +15,7 @@ STATE_URL_SUFFIX = "/state" -class TSchemaSyncPayload(TypedDict): +class TPipelineSyncPayload(TypedDict): pipeline_name: str destination_name: str destination_displayable_credentials: str @@ -64,7 +64,7 @@ def _sync_schemas_to_platform(trace: PipelineTrace, pipeline: SupportsPipeline) if not load_info: return - payload = TSchemaSyncPayload( + payload = TPipelineSyncPayload( pipeline_name=pipeline.pipeline_name, destination_name=load_info.destination_name, destination_displayable_credentials=load_info.destination_displayable_credentials, @@ -109,11 +109,10 @@ def on_end_trace_step( step_info: Any, send_state: bool, ) -> None: - pass + if send_state: + # also sync schemas to dlthub + _sync_schemas_to_platform(trace, pipeline) def on_end_trace(trace: PipelineTrace, pipeline: SupportsPipeline, send_state: bool) -> None: _send_trace_to_platform(trace, pipeline) - if send_state: - # also sync schemas to dlthub - _sync_schemas_to_platform(trace, pipeline)