Skip to content

Commit

Permalink
remove name from source decorator
Browse files Browse the repository at this point in the history
  • Loading branch information
sh-rp committed Nov 16, 2023
1 parent 185068c commit 0fbcdee
Show file tree
Hide file tree
Showing 12 changed files with 21 additions and 29 deletions.
16 changes: 4 additions & 12 deletions dlt/extract/decorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ def __init__(self, schema: Schema = None) -> None:
def source(
func: Callable[TSourceFunParams, Any],
/,
name: str = None,
section: str = None,
max_table_nesting: int = None,
root_key: bool = False,
Expand All @@ -62,7 +61,6 @@ def source(
def source(
func: None = ...,
/,
name: str = None,
section: str = None,
max_table_nesting: int = None,
root_key: bool = False,
Expand All @@ -75,7 +73,6 @@ def source(
def source(
func: Optional[AnyFun] = None,
/,
name: str = None,
section: str = None,
max_table_nesting: int = None,
root_key: bool = False,
Expand Down Expand Up @@ -123,28 +120,23 @@ def source(
`DltSource` instance
"""

if name and schema:
raise ArgumentsOverloadException("'name' has no effect when `schema` argument is present", source.__name__)

def decorator(f: Callable[TSourceFunParams, Any]) -> Callable[TSourceFunParams, TDltSourceImpl]:
nonlocal schema, name
nonlocal schema

if not callable(f) or isinstance(f, DltResource):
raise SourceNotAFunction(name or "<no name>", f, type(f))
raise SourceNotAFunction("<no name>", f, type(f))

if inspect.isclass(f):
raise SourceIsAClassTypeError(name or "<no name>", f)
raise SourceIsAClassTypeError("<no name>", f)

# source name is passed directly or taken from decorated function name
effective_name = name or get_callable_name(f)
effective_name = get_callable_name(f)

if not schema:
# load the schema from file with name_schema.yaml/json from the same directory, the callable resides OR create new default schema
schema = _maybe_load_schema_for_callable(f, effective_name) or Schema(effective_name)

if name and name != schema.name:
raise ExplicitSourceNameInvalid(name, schema.name)

# the name of the source must be identical to the name of the schema
name = schema.name

Expand Down
2 changes: 1 addition & 1 deletion docs/website/docs/dlt-ecosystem/verified-sources/chess.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ This is a `dlt.source` function for the Chess.com API named "chess", which retur
DltResource objects. That we'll discuss in subsequent sections as resources.

```python
dlt.source(name="chess")
dlt.source()
def source(
players: List[str], start_month: str = None, end_month: str = None
) -> Sequence[DltResource]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ This function returns a list of resources to load campaigns, ad sets, ads, creat
data from Facebook Marketing API.

```python
@dlt.source(name="facebook_ads")
@dlt.source()
def facebook_ads_source(
account_id: str = dlt.config.value,
access_token: str = dlt.secrets.value,
Expand Down Expand Up @@ -257,7 +257,7 @@ The default fields are defined in
This function returns a list of resources to load facebook_insights.

```python
@dlt.source(name="facebook_ads")
@dlt.source()
def facebook_insights_source(
account_id: str = dlt.config.value,
access_token: str = dlt.secrets.value,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ This function returns a list of resources to load companies, contacts, deals, ti
web analytics events data into the destination.

```python
@dlt.source(name="hubspot")
@dlt.source()
def hubspot(
api_key: str = dlt.secrets.value, include_history: bool = False
) -> Sequence[DltResource]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ This function returns a list of resources including activities, deals, custom_fi
other resources data from Pipedrive API.

```python
@dlt.source(name="pipedrive")
@dlt.source()
def pipedrive_source(
pipedrive_api_key: str = dlt.secrets.value,
since_timestamp: Optional[Union[pendulum.DateTime, str]] = dlt.config.value,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ This function returns a list of resources to load users, user_role, opportunity,
opportunity_line_item, account etc. data from Salesforce API.

```python
@dlt.source(name="salesforce")
@dlt.source()
def salesforce_source(
user_name: str = dlt.secrets.value,
password: str = dlt.secrets.value,
Expand Down
2 changes: 1 addition & 1 deletion docs/website/docs/dlt-ecosystem/verified-sources/slack.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ For more information, read the [General Usage: Credentials.](../../general-usage
It retrieves data from Slack's API and fetches the Slack data such as channels, messages for selected channels, users, logs.

```python
@dlt.source(name="slack", max_table_nesting=2)
@dlt.source(max_table_nesting=2)
def slack_source(
page_size: int = MAX_PAGE_SIZE,
access_token: str = dlt.secrets.value,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ endpoints allow incremental 'merge' mode loading.
This source returns a sequence of dltResources that correspond to the endpoints.

```python
@dlt.source(name="workable")
@dlt.source()
def workable_source(
access_token: str = dlt.secrets.value,
subdomain: str = dlt.config.value,
Expand Down
6 changes: 3 additions & 3 deletions tests/extract/test_decorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,12 +169,12 @@ def camelCase():

# explicit name
with pytest.raises(InvalidSchemaName) as py_ex:
s = dlt.source(camelCase, name="source!")()
s = dlt.source(camelCase)()
assert py_ex.value.name == "source!"

# explicit name and schema mismatch
with pytest.raises(ArgumentsOverloadException) as py_ex2:
s = dlt.source(camelCase, name="source_ovr", schema=Schema("compat"))()
s = dlt.source(camelCase, schema=Schema("compat"))()
# overload exception applies to dlt.source
assert py_ex2.value.func_name == "source"

Expand Down Expand Up @@ -782,7 +782,7 @@ def __call__(self, more: int = 1):

# CAN'T decorate classes themselves
with pytest.raises(SourceIsAClassTypeError):
@dlt.source(name="planB")
@dlt.source()
class _SourceB:
def __init__(self, elems: int) -> None:
self.elems = elems
Expand Down
2 changes: 1 addition & 1 deletion tests/load/pipeline/test_drop.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def _attach(pipeline: Pipeline) -> Pipeline:
return dlt.attach(pipeline.pipeline_name, pipeline.pipelines_dir)


@dlt.source(section='droppable', name='droppable')
@dlt.source(section='droppable')
def droppable_source() -> List[DltResource]:
@dlt.resource
def droppable_a(a: dlt.sources.incremental[int]=dlt.sources.incremental('a', 0)) -> Iterator[Dict[str, Any]]:
Expand Down
8 changes: 4 additions & 4 deletions tests/load/pipeline/test_pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ def test_run_full_refresh(destination_config: DestinationTestConfiguration) -> N
def d():
yield data

@dlt.source(name="nested")
@dlt.source()
def _data():
return dlt.resource(d(), name="lists", write_disposition="replace")

Expand Down Expand Up @@ -217,7 +217,7 @@ def test_evolve_schema(destination_config: DestinationTestConfiguration) -> None
}]
}

@dlt.source(name="parallel")
@dlt.source()
def source(top_elements: int):

@dlt.defer
Expand Down Expand Up @@ -325,7 +325,7 @@ def test_source_max_nesting(destination_config: DestinationTestConfiguration) ->
}
}

@dlt.source(name="complex", max_table_nesting=0)
@dlt.source(max_table_nesting=0)
def complex_data():
return dlt.resource([
{
Expand Down Expand Up @@ -705,7 +705,7 @@ def simple_nested_pipeline(destination_config: DestinationTestConfiguration, dat
def d():
yield data

@dlt.source(name="nested")
@dlt.source()
def _data():
return dlt.resource(d(), name="lists", write_disposition="append")

Expand Down
2 changes: 1 addition & 1 deletion tests/pipeline/test_pipeline_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def test_managed_state() -> None:

# attach to different source that will get separate state

@dlt.source(name="separate_state", section="different_section")
@dlt.source(section="different_section")
def some_source():
assert "last_value" not in dlt.current.source_state()
return some_data
Expand Down

0 comments on commit 0fbcdee

Please sign in to comment.