From 5e8c233e59e8340b3451f4d53841a7dd92bb088b Mon Sep 17 00:00:00 2001 From: Dave Date: Mon, 15 Apr 2024 17:37:28 +0200 Subject: [PATCH] remove dev code --- fs_testing_pipe.py | 39 --------------------------------------- tests/load/utils.py | 2 -- tests/utils.py | 2 -- 3 files changed, 43 deletions(-) delete mode 100644 fs_testing_pipe.py diff --git a/fs_testing_pipe.py b/fs_testing_pipe.py deleted file mode 100644 index 7f95f1f389..0000000000 --- a/fs_testing_pipe.py +++ /dev/null @@ -1,39 +0,0 @@ -import dlt -import os -import shutil -import random - -if __name__ == "__main__": - - # shutil.rmtree("./my_files", ignore_errors=True) - os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] = "file://my_files" - os.environ["DATA_WRITER__DISABLE_COMPRESSION"] = "TRUE" - - # resource with incremental for testing restoring of pipeline state - @dlt.resource(name="my_table") - def my_resouce(id=dlt.sources.incremental("id")): - yield from [ - {"id": 1}, - {"id": 2}, - {"id": 3}, - {"id": 4}, - {"id": 5} - ] - - pipeline_name = f"dave" - - pipe = dlt.pipeline(pipeline_name=pipeline_name, destination="filesystem") - pipe.run(my_resouce(), table_name="my_table") #, loader_file_format="parquet") - - # resource with incremental for testing restoring of pipeline state - @dlt.resource(name="my_table") - def updated_resouce(id=dlt.sources.incremental("id")): - yield from [ - {"id": 1}, - {"id": 2}, - {"id": 3}, - {"id": 4}, - {"id": 5}, - {"id": 6} - ] - pipe.run(updated_resouce(), table_name="my_table") #, loader_file_format="parquet") diff --git a/tests/load/utils.py b/tests/load/utils.py index 487fd588be..110c2b433d 100644 --- a/tests/load/utils.py +++ b/tests/load/utils.py @@ -64,8 +64,6 @@ "r2", ] -ALL_FILESYSTEM_DRIVERS = ["memory", "file"] - # Filter out buckets not in all filesystem drivers WITH_GDRIVE_BUCKETS = [GCS_BUCKET, AWS_BUCKET, FILE_BUCKET, MEMORY_BUCKET, AZ_BUCKET, GDRIVE_BUCKET] WITH_GDRIVE_BUCKETS = [ diff --git a/tests/utils.py b/tests/utils.py index 6defd08f5a..1ccb7fc5e4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -62,8 +62,6 @@ # filter out active destinations for current tests ACTIVE_DESTINATIONS = set(dlt.config.get("ACTIVE_DESTINATIONS", list) or IMPLEMENTED_DESTINATIONS) -ACTIVE_DESTINATIONS = {"filesystem", "duckdb"} - ACTIVE_SQL_DESTINATIONS = SQL_DESTINATIONS.intersection(ACTIVE_DESTINATIONS) ACTIVE_NON_SQL_DESTINATIONS = NON_SQL_DESTINATIONS.intersection(ACTIVE_DESTINATIONS)