From 0fbcdee4a184f28cb0b2a33fe57313e0ad5673c6 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 16 Nov 2023 18:09:53 +0100 Subject: [PATCH] remove name from source decorator --- dlt/extract/decorators.py | 16 ++++------------ .../docs/dlt-ecosystem/verified-sources/chess.md | 2 +- .../verified-sources/facebook_ads.md | 4 ++-- .../dlt-ecosystem/verified-sources/hubspot.md | 2 +- .../dlt-ecosystem/verified-sources/pipedrive.md | 2 +- .../dlt-ecosystem/verified-sources/salesforce.md | 2 +- .../docs/dlt-ecosystem/verified-sources/slack.md | 2 +- .../dlt-ecosystem/verified-sources/workable.md | 2 +- tests/extract/test_decorators.py | 6 +++--- tests/load/pipeline/test_drop.py | 2 +- tests/load/pipeline/test_pipelines.py | 8 ++++---- tests/pipeline/test_pipeline_state.py | 2 +- 12 files changed, 21 insertions(+), 29 deletions(-) diff --git a/dlt/extract/decorators.py b/dlt/extract/decorators.py index dbc5f2fa82..6ac11c04f0 100644 --- a/dlt/extract/decorators.py +++ b/dlt/extract/decorators.py @@ -48,7 +48,6 @@ def __init__(self, schema: Schema = None) -> None: def source( func: Callable[TSourceFunParams, Any], /, - name: str = None, section: str = None, max_table_nesting: int = None, root_key: bool = False, @@ -62,7 +61,6 @@ def source( def source( func: None = ..., /, - name: str = None, section: str = None, max_table_nesting: int = None, root_key: bool = False, @@ -75,7 +73,6 @@ def source( def source( func: Optional[AnyFun] = None, /, - name: str = None, section: str = None, max_table_nesting: int = None, root_key: bool = False, @@ -123,28 +120,23 @@ def source( `DltSource` instance """ - if name and schema: - raise ArgumentsOverloadException("'name' has no effect when `schema` argument is present", source.__name__) def decorator(f: Callable[TSourceFunParams, Any]) -> Callable[TSourceFunParams, TDltSourceImpl]: - nonlocal schema, name + nonlocal schema if not callable(f) or isinstance(f, DltResource): - raise SourceNotAFunction(name or "", f, type(f)) + raise SourceNotAFunction("", f, type(f)) if inspect.isclass(f): - raise SourceIsAClassTypeError(name or "", f) + raise SourceIsAClassTypeError("", f) # source name is passed directly or taken from decorated function name - effective_name = name or get_callable_name(f) + effective_name = get_callable_name(f) if not schema: # load the schema from file with name_schema.yaml/json from the same directory, the callable resides OR create new default schema schema = _maybe_load_schema_for_callable(f, effective_name) or Schema(effective_name) - if name and name != schema.name: - raise ExplicitSourceNameInvalid(name, schema.name) - # the name of the source must be identical to the name of the schema name = schema.name diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/chess.md b/docs/website/docs/dlt-ecosystem/verified-sources/chess.md index e528f57d87..8dee2676ed 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/chess.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/chess.md @@ -100,7 +100,7 @@ This is a `dlt.source` function for the Chess.com API named "chess", which retur DltResource objects. That we'll discuss in subsequent sections as resources. ```python -dlt.source(name="chess") +dlt.source() def source( players: List[str], start_month: str = None, end_month: str = None ) -> Sequence[DltResource]: diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/facebook_ads.md b/docs/website/docs/dlt-ecosystem/verified-sources/facebook_ads.md index db3c6e0b81..a5dee5961e 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/facebook_ads.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/facebook_ads.md @@ -194,7 +194,7 @@ This function returns a list of resources to load campaigns, ad sets, ads, creat data from Facebook Marketing API. ```python -@dlt.source(name="facebook_ads") +@dlt.source() def facebook_ads_source( account_id: str = dlt.config.value, access_token: str = dlt.secrets.value, @@ -257,7 +257,7 @@ The default fields are defined in This function returns a list of resources to load facebook_insights. ```python -@dlt.source(name="facebook_ads") +@dlt.source() def facebook_insights_source( account_id: str = dlt.config.value, access_token: str = dlt.secrets.value, diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/hubspot.md b/docs/website/docs/dlt-ecosystem/verified-sources/hubspot.md index c305b5b842..45973930f9 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/hubspot.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/hubspot.md @@ -150,7 +150,7 @@ This function returns a list of resources to load companies, contacts, deals, ti web analytics events data into the destination. ```python -@dlt.source(name="hubspot") +@dlt.source() def hubspot( api_key: str = dlt.secrets.value, include_history: bool = False ) -> Sequence[DltResource]: diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/pipedrive.md b/docs/website/docs/dlt-ecosystem/verified-sources/pipedrive.md index 689f8f7808..f0bc625147 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/pipedrive.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/pipedrive.md @@ -140,7 +140,7 @@ This function returns a list of resources including activities, deals, custom_fi other resources data from Pipedrive API. ```python -@dlt.source(name="pipedrive") +@dlt.source() def pipedrive_source( pipedrive_api_key: str = dlt.secrets.value, since_timestamp: Optional[Union[pendulum.DateTime, str]] = dlt.config.value, diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/salesforce.md b/docs/website/docs/dlt-ecosystem/verified-sources/salesforce.md index c819c8120b..26c43ec7ca 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/salesforce.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/salesforce.md @@ -140,7 +140,7 @@ This function returns a list of resources to load users, user_role, opportunity, opportunity_line_item, account etc. data from Salesforce API. ```python -@dlt.source(name="salesforce") +@dlt.source() def salesforce_source( user_name: str = dlt.secrets.value, password: str = dlt.secrets.value, diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/slack.md b/docs/website/docs/dlt-ecosystem/verified-sources/slack.md index fd25d7818b..5f21892485 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/slack.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/slack.md @@ -140,7 +140,7 @@ For more information, read the [General Usage: Credentials.](../../general-usage It retrieves data from Slack's API and fetches the Slack data such as channels, messages for selected channels, users, logs. ```python -@dlt.source(name="slack", max_table_nesting=2) +@dlt.source(max_table_nesting=2) def slack_source( page_size: int = MAX_PAGE_SIZE, access_token: str = dlt.secrets.value, diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/workable.md b/docs/website/docs/dlt-ecosystem/verified-sources/workable.md index 14530e081e..8eada40d69 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/workable.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/workable.md @@ -166,7 +166,7 @@ endpoints allow incremental 'merge' mode loading. This source returns a sequence of dltResources that correspond to the endpoints. ```python -@dlt.source(name="workable") +@dlt.source() def workable_source( access_token: str = dlt.secrets.value, subdomain: str = dlt.config.value, diff --git a/tests/extract/test_decorators.py b/tests/extract/test_decorators.py index 05e3a2fbf3..d4f9db9a9c 100644 --- a/tests/extract/test_decorators.py +++ b/tests/extract/test_decorators.py @@ -169,12 +169,12 @@ def camelCase(): # explicit name with pytest.raises(InvalidSchemaName) as py_ex: - s = dlt.source(camelCase, name="source!")() + s = dlt.source(camelCase)() assert py_ex.value.name == "source!" # explicit name and schema mismatch with pytest.raises(ArgumentsOverloadException) as py_ex2: - s = dlt.source(camelCase, name="source_ovr", schema=Schema("compat"))() + s = dlt.source(camelCase, schema=Schema("compat"))() # overload exception applies to dlt.source assert py_ex2.value.func_name == "source" @@ -782,7 +782,7 @@ def __call__(self, more: int = 1): # CAN'T decorate classes themselves with pytest.raises(SourceIsAClassTypeError): - @dlt.source(name="planB") + @dlt.source() class _SourceB: def __init__(self, elems: int) -> None: self.elems = elems diff --git a/tests/load/pipeline/test_drop.py b/tests/load/pipeline/test_drop.py index a2714674be..7a764a339c 100644 --- a/tests/load/pipeline/test_drop.py +++ b/tests/load/pipeline/test_drop.py @@ -19,7 +19,7 @@ def _attach(pipeline: Pipeline) -> Pipeline: return dlt.attach(pipeline.pipeline_name, pipeline.pipelines_dir) -@dlt.source(section='droppable', name='droppable') +@dlt.source(section='droppable') def droppable_source() -> List[DltResource]: @dlt.resource def droppable_a(a: dlt.sources.incremental[int]=dlt.sources.incremental('a', 0)) -> Iterator[Dict[str, Any]]: diff --git a/tests/load/pipeline/test_pipelines.py b/tests/load/pipeline/test_pipelines.py index 99071a7ac6..f4e64a90fe 100644 --- a/tests/load/pipeline/test_pipelines.py +++ b/tests/load/pipeline/test_pipelines.py @@ -181,7 +181,7 @@ def test_run_full_refresh(destination_config: DestinationTestConfiguration) -> N def d(): yield data - @dlt.source(name="nested") + @dlt.source() def _data(): return dlt.resource(d(), name="lists", write_disposition="replace") @@ -217,7 +217,7 @@ def test_evolve_schema(destination_config: DestinationTestConfiguration) -> None }] } - @dlt.source(name="parallel") + @dlt.source() def source(top_elements: int): @dlt.defer @@ -325,7 +325,7 @@ def test_source_max_nesting(destination_config: DestinationTestConfiguration) -> } } - @dlt.source(name="complex", max_table_nesting=0) + @dlt.source(max_table_nesting=0) def complex_data(): return dlt.resource([ { @@ -705,7 +705,7 @@ def simple_nested_pipeline(destination_config: DestinationTestConfiguration, dat def d(): yield data - @dlt.source(name="nested") + @dlt.source() def _data(): return dlt.resource(d(), name="lists", write_disposition="append") diff --git a/tests/pipeline/test_pipeline_state.py b/tests/pipeline/test_pipeline_state.py index 14b881eedc..3fd99d83ab 100644 --- a/tests/pipeline/test_pipeline_state.py +++ b/tests/pipeline/test_pipeline_state.py @@ -68,7 +68,7 @@ def test_managed_state() -> None: # attach to different source that will get separate state - @dlt.source(name="separate_state", section="different_section") + @dlt.source(section="different_section") def some_source(): assert "last_value" not in dlt.current.source_state() return some_data