Merge pull request #7 from PySport/feature/refactor-resource

Feature/refactor resource
PySport · Sep 17, 2024 · f40e3df · f40e3df
2 parents cf26d7b + 0f993f2
commit f40e3df
Show file tree

Hide file tree

Showing 17 changed files with 424 additions and 214 deletions.
diff --git a/ingestify/__init__.py b/ingestify/__init__.py
@@ -6,6 +6,6 @@
 
 if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
-    from .source_base import Source
+    from .source_base import Source, DatasetResource
 
 __version__ = "0.0.5"
diff --git a/ingestify/application/dataset_store.py b/ingestify/application/dataset_store.py
@@ -9,13 +9,15 @@
 
 from typing import Dict, List, Optional, Union, Callable, BinaryIO
 
+from ingestify.domain.models.dataset.dataset import DatasetState
 from ingestify.domain.models.dataset.events import RevisionAdded, MetadataUpdated
 from ingestify.domain.models.dataset.file_collection import FileCollection
 from ingestify.domain.models.event import EventBus
 from ingestify.domain.models import (
     Dataset,
     DatasetCollection,
     DatasetRepository,
+    DatasetResource,
     DraftFile,
     File,
     LoadedFile,
@@ -204,12 +206,12 @@ def add_revision(
     def update_dataset(
         self,
         dataset: Dataset,
-        dataset_identifier: Identifier,
+        dataset_resource: DatasetResource,
         files: Dict[str, DraftFile],
     ):
         """The add_revision will also save the dataset."""
         metadata_changed = False
-        if dataset.update_from_identifier(dataset_identifier):
+        if dataset.update_from_resource(dataset_resource):
             self.dataset_repository.save(bucket=self.bucket, dataset=dataset)
             metadata_changed = True
 
@@ -229,6 +231,9 @@ def create_dataset(
         dataset_type: str,
         provider: str,
         dataset_identifier: Identifier,
+        name: str,
+        state: DatasetState,
+        metadata: dict,
         files: Dict[str, DraftFile],
         description: str = "Create",
     ):
@@ -237,12 +242,12 @@ def create_dataset(
         dataset = Dataset(
             bucket=self.bucket,
             dataset_id=self.dataset_repository.next_identity(),
-            name=dataset_identifier.name,
-            state=dataset_identifier.state,
+            name=name,
+            state=state,
             identifier=dataset_identifier,
             dataset_type=dataset_type,
             provider=provider,
-            metadata=dataset_identifier.metadata,
+            metadata=metadata,
             created_at=now,
             updated_at=now,
         )

diff --git a/ingestify/application/ingestion_engine.py b/ingestify/application/ingestion_engine.py
@@ -21,8 +21,8 @@ def __init__(self, store: DatasetStore):
     def add_extract_job(self, extract_job: ExtractJob):
         self.loader.add_extract_job(extract_job)
 
-    def load(self):
-        self.loader.collect_and_run()
+    def load(self, dry_run: bool = False):
+        self.loader.collect_and_run(dry_run=dry_run)
 
     def list_datasets(self, as_count: bool = False):
         """Consider moving this to DataStore"""