From 90ff9907e39ac9013371ee575bfd9819d1d17686 Mon Sep 17 00:00:00 2001 From: David Scharf Date: Mon, 16 Oct 2023 19:17:46 +0200 Subject: [PATCH] fixes bug in drop command (#693) --- dlt/pipeline/helpers.py | 4 +++- tests/load/pipeline/test_drop.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/dlt/pipeline/helpers.py b/dlt/pipeline/helpers.py index 670b2a7887..ef4fe70664 100644 --- a/dlt/pipeline/helpers.py +++ b/dlt/pipeline/helpers.py @@ -143,10 +143,12 @@ def _create_modified_state(self) -> Dict[str, Any]: return state # type: ignore[return-value] source_states = _sources_state(state).items() for source_name, source_state in source_states: - if self.drop_state: + # drop table states + if self.drop_state and self.resource_pattern: for key in _get_matching_resources(self.resource_pattern, source_state): self.info['resource_states'].append(key) reset_resource_state(key, source_state) + # drop additional state paths resolved_paths = resolve_paths(self.state_paths_to_drop, source_state) if self.state_paths_to_drop and not resolved_paths: self.info['warnings'].append(f"State paths {self.state_paths_to_drop} did not select any paths in source {source_name}") diff --git a/tests/load/pipeline/test_drop.py b/tests/load/pipeline/test_drop.py index b611097dc9..a2714674be 100644 --- a/tests/load/pipeline/test_drop.py +++ b/tests/load/pipeline/test_drop.py @@ -122,6 +122,28 @@ def test_drop_command_resources_and_state(destination_config: DestinationTestCon assert_destination_state_loaded(pipeline) +@pytest.mark.parametrize("destination_config", destinations_configs(default_sql_configs=True), ids=lambda x: x.name) +def test_drop_command_only_state(destination_config: DestinationTestConfiguration) -> None: + """Test the drop command with resource and state path options and + verify correct data is deleted from destination and locally""" + source = droppable_source() + pipeline = destination_config.setup_pipeline('drop_test_' + uniq_id(), full_refresh=True) + pipeline.run(source) + + attached = _attach(pipeline) + helpers.drop(attached, state_paths='data_from_d.*.bar') + + attached = _attach(pipeline) + + assert_dropped_resources(attached, []) + + # Verify extra json paths are removed from state + sources_state = pipeline.state['sources'] + assert sources_state['droppable']['data_from_d'] == {'foo1': {}, 'foo2': {}} + + assert_destination_state_loaded(pipeline) + + @pytest.mark.parametrize("destination_config", destinations_configs(default_sql_configs=True), ids=lambda x: x.name) def test_drop_destination_tables_fails(destination_config: DestinationTestConfiguration) -> None: """Fail on drop tables. Command runs again."""