diff --git a/dlt/common/destination/reference.py b/dlt/common/destination/reference.py index b926f11dae..2ad5131e63 100644 --- a/dlt/common/destination/reference.py +++ b/dlt/common/destination/reference.py @@ -484,7 +484,10 @@ def should_truncate_table_before_load_on_staging_destination(self, table: TTable return True -TDestinationReferenceArg = Union[str, "Destination", Callable[..., "Destination"], None] +# TODO: type Destination properly +TDestinationReferenceArg = Union[ + str, "Destination[Any, Any]", Callable[..., "Destination[Any, Any]"], None +] class Destination(ABC, Generic[TDestinationConfig, TDestinationClient]): diff --git a/dlt/common/typing.py b/dlt/common/typing.py index 99c2604cdf..d18d8000e7 100644 --- a/dlt/common/typing.py +++ b/dlt/common/typing.py @@ -55,11 +55,13 @@ from typing import _TypedDict REPattern = _REPattern[str] + PathLike = os.PathLike[str] else: StrOrBytesPath = Any from typing import _TypedDictMeta as _TypedDict REPattern = _REPattern + PathLike = os.PathLike AnyType: TypeAlias = Any NoneType = type(None) @@ -92,7 +94,7 @@ TVariantBase = TypeVar("TVariantBase", covariant=True) TVariantRV = Tuple[str, Any] VARIANT_FIELD_FORMAT = "v_%s" -TFileOrPath = Union[str, os.PathLike, IO[Any]] +TFileOrPath = Union[str, PathLike, IO[Any]] TSortOrder = Literal["asc", "desc"] diff --git a/dlt/destinations/impl/dremio/pydremio.py b/dlt/destinations/impl/dremio/pydremio.py index b936278fbd..08ed2ed399 100644 --- a/dlt/destinations/impl/dremio/pydremio.py +++ b/dlt/destinations/impl/dremio/pydremio.py @@ -242,7 +242,7 @@ def __init__(self, factory: CookieMiddlewareFactory, *args: Any, **kwargs: Any): def received_headers(self, headers: Mapping[str, str]) -> None: for key in headers: if key.lower() == "set-cookie": - cookie = SimpleCookie() # type: ignore + cookie = SimpleCookie() for item in headers.get(key): cookie.load(item) diff --git a/dlt/extract/decorators.py b/dlt/extract/decorators.py index 7230c48516..fac6391e01 100644 --- a/dlt/extract/decorators.py +++ b/dlt/extract/decorators.py @@ -52,6 +52,7 @@ from dlt.extract.exceptions import ( CurrentSourceNotAvailable, DynamicNameNotStandaloneResource, + InvalidResourceDataTypeFunctionNotAGenerator, InvalidTransformerDataTypeGeneratorFunctionRequired, ResourceFunctionExpected, ResourceInnerCallableConfigWrapDisallowed, @@ -65,7 +66,7 @@ from dlt.extract.items import TTableHintTemplate from dlt.extract.source import DltSource -from dlt.extract.resource import DltResource, TUnboundDltResource +from dlt.extract.resource import DltResource, TUnboundDltResource, TDltResourceImpl @configspec @@ -103,7 +104,7 @@ def source( schema_contract: TSchemaContract = None, spec: Type[BaseConfiguration] = None, _impl_cls: Type[TDltSourceImpl] = DltSource, # type: ignore[assignment] -) -> Callable[TSourceFunParams, DltSource]: ... +) -> Callable[TSourceFunParams, TDltSourceImpl]: ... @overload @@ -264,7 +265,7 @@ async def _wrap_coro(*args: Any, **kwargs: Any) -> TDltSourceImpl: # get spec for wrapped function SPEC = get_fun_spec(conf_f) # get correct wrapper - wrapper = _wrap_coro if inspect.iscoroutinefunction(inspect.unwrap(f)) else _wrap + wrapper: AnyFun = _wrap_coro if inspect.iscoroutinefunction(inspect.unwrap(f)) else _wrap # type: ignore[assignment] # store the source information _SOURCES[_wrap.__qualname__] = SourceInfo(SPEC, wrapper, func_module) if inspect.iscoroutinefunction(inspect.unwrap(f)): @@ -296,7 +297,8 @@ def resource( selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, -) -> DltResource: ... + _impl_cls: Type[TDltResourceImpl] = DltResource, # type: ignore[assignment] +) -> TDltResourceImpl: ... @overload @@ -315,7 +317,8 @@ def resource( selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, -) -> Callable[[Callable[TResourceFunParams, Any]], DltResource]: ... + _impl_cls: Type[TDltResourceImpl] = DltResource, # type: ignore[assignment] +) -> Callable[[Callable[TResourceFunParams, Any]], TDltResourceImpl]: ... @overload @@ -334,8 +337,11 @@ def resource( selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, + _impl_cls: Type[TDltResourceImpl] = DltResource, # type: ignore[assignment] standalone: Literal[True] = True, -) -> Callable[[Callable[TResourceFunParams, Any]], Callable[TResourceFunParams, DltResource]]: ... +) -> Callable[ + [Callable[TResourceFunParams, Any]], Callable[TResourceFunParams, TDltResourceImpl] +]: ... @overload @@ -354,7 +360,8 @@ def resource( selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, -) -> DltResource: ... + _impl_cls: Type[TDltResourceImpl] = DltResource, # type: ignore[assignment] +) -> TDltResourceImpl: ... def resource( @@ -372,6 +379,7 @@ def resource( selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, + _impl_cls: Type[TDltResourceImpl] = DltResource, # type: ignore[assignment] standalone: bool = False, data_from: TUnboundDltResource = None, ) -> Any: @@ -435,17 +443,19 @@ def resource( parallelized (bool, optional): If `True`, the resource generator will be extracted in parallel with other resources. Defaults to `False`. + _impl_cls (Type[TDltResourceImpl], optional): A custom implementation of DltResource, may be also used to providing just a typing stub + Raises: ResourceNameMissing: indicates that name of the resource cannot be inferred from the `data` being passed. InvalidResourceDataType: indicates that the `data` argument cannot be converted into `dlt resource` Returns: - DltResource instance which may be loaded, iterated or combined with other resources into a pipeline. + TDltResourceImpl instance which may be loaded, iterated or combined with other resources into a pipeline. """ def make_resource( _name: str, _section: str, _data: Any, incremental: IncrementalResourceWrapper = None - ) -> DltResource: + ) -> TDltResourceImpl: table_template = make_hints( table_name, write_disposition=write_disposition or DEFAULT_WRITE_DISPOSITION, @@ -464,7 +474,7 @@ def make_resource( table_template.setdefault("x-normalizer", {}) # type: ignore[typeddict-item] table_template["x-normalizer"]["max_nesting"] = max_table_nesting # type: ignore[typeddict-item] - resource = DltResource.from_data( + resource = _impl_cls.from_data( _data, _name, _section, @@ -479,7 +489,7 @@ def make_resource( def decorator( f: Callable[TResourceFunParams, Any] - ) -> Callable[TResourceFunParams, DltResource]: + ) -> Callable[TResourceFunParams, TDltResourceImpl]: if not callable(f): if data_from: # raise more descriptive exception if we construct transformer @@ -490,7 +500,6 @@ def decorator( if not standalone and callable(name): raise DynamicNameNotStandaloneResource(get_callable_name(f)) - # resource_section = name if name and not callable(name) else get_callable_name(f) resource_name = name if name and not callable(name) else get_callable_name(f) # do not inject config values for inner functions, we assume that they are part of the source @@ -506,15 +515,18 @@ def decorator( incr_f = incremental.wrap(sig, f) if incremental else f resource_sections = (known_sections.SOURCES, source_section, resource_name) + # standalone resource will prefer existing section context when resolving config values # this lets the source to override those values and provide common section for all config values for resources present in that source # for autogenerated spec do not include defaults + # NOTE: allow full config for standalone, currently some edge cases for incremental does not work + # (removing it via apply hints or explicit call) conf_f = with_config( incr_f, spec=spec, sections=resource_sections, sections_merge_style=ConfigSectionContext.resource_merge_style, - include_defaults=spec is not None, + include_defaults=spec is not None, # or standalone, ) is_inner_resource = is_inner_callable(f) if conf_f != incr_f and is_inner_resource and not standalone: @@ -526,33 +538,52 @@ def decorator( if not is_inner_resource: _SOURCES[f.__qualname__] = SourceInfo(SPEC, f, func_module) - if standalone: - if data_from: - compat_wrapper, skip_args = wrap_compat_transformer, 1 - else: - compat_wrapper, skip_args = wrap_resource_gen, 0 - - @wraps(conf_f) - def _wrap(*args: Any, **kwargs: Any) -> DltResource: - _, mod_sig, bound_args = simulate_func_call(conf_f, skip_args, *args, **kwargs) - actual_resource_name = ( - name(bound_args.arguments) if callable(name) else resource_name - ) + if not standalone: + # we return a DltResource that is callable and returns dlt resource when called + # so it should match the signature + return make_resource(resource_name, source_section, conf_f, incremental) # type: ignore[return-value] + + # wrap the standalone resource + if data_from: + compat_wrapper, skip_args = wrap_compat_transformer, 1 + else: + compat_wrapper, skip_args = wrap_resource_gen, 0 + + @wraps(incr_f) + def _wrap(*args: Any, **kwargs: Any) -> TDltResourceImpl: + _, mod_sig, bound_args = simulate_func_call(incr_f, skip_args, *args, **kwargs) + actual_resource_name = name(bound_args.arguments) if callable(name) else resource_name + # wrap again with an actual resource name + conf_f = with_config( + incr_f, + spec=SPEC, + sections=resource_sections[:-1] + (actual_resource_name,), + sections_merge_style=ConfigSectionContext.resource_merge_style, + ) + try: r = make_resource( actual_resource_name, source_section, compat_wrapper(actual_resource_name, conf_f, sig, *args, **kwargs), incremental, ) - # consider transformer arguments bound - r._args_bound = True - # keep explicit args passed - r._set_explicit_args(conf_f, mod_sig, *args, **kwargs) - return r - - return _wrap - else: - return make_resource(resource_name, source_section, conf_f, incremental) + except InvalidResourceDataTypeFunctionNotAGenerator as gen_ex: + # we allow an edge case: resource can return another resource + try: + # actually call the function to see if it contains DltResource + data_ = conf_f(*args, **kwargs) + if not isinstance(data_, DltResource): + raise + r = data_ # type: ignore[assignment] + except Exception: + raise gen_ex from None + # consider transformer arguments bound + r._args_bound = True + # keep explicit args passed + r._set_explicit_args(conf_f, mod_sig, *args, **kwargs) + return r + + return _wrap # if data is callable or none use decorator if data is None: @@ -659,6 +690,7 @@ def transformer( spec: Type[BaseConfiguration] = None, parallelized: bool = False, standalone: bool = False, + _impl_cls: Type[TDltResourceImpl] = DltResource, # type: ignore[assignment] ) -> Any: """A form of `dlt resource` that takes input from other resources via `data_from` argument in order to enrich or transform the data. @@ -713,6 +745,8 @@ def transformer( spec (Type[BaseConfiguration], optional): A specification of configuration and secret values required by the source. standalone (bool, optional): Returns a wrapped decorated function that creates DltResource instance. Must be called before use. Cannot be part of a source. + + _impl_cls (Type[TDltResourceImpl], optional): A custom implementation of DltResource, may be also used to providing just a typing stub """ if isinstance(f, DltResource): raise ValueError( @@ -733,6 +767,7 @@ def transformer( standalone=standalone, data_from=data_from, parallelized=parallelized, + _impl_cls=_impl_cls, ) diff --git a/dlt/extract/incremental/__init__.py b/dlt/extract/incremental/__init__.py index ef7523b207..b6ecd2d3db 100644 --- a/dlt/extract/incremental/__init__.py +++ b/dlt/extract/incremental/__init__.py @@ -212,6 +212,9 @@ def merge(self, other: "Incremental[TCursorValue]") -> "Incremental[TCursorValue merged.resource_name = self.resource_name if other.resource_name: merged.resource_name = other.resource_name + # also pass if resolved + merged.__is_resolved__ = other.__is_resolved__ + merged.__exception__ = other.__exception__ return merged # type: ignore def copy(self) -> "Incremental[TCursorValue]": @@ -438,7 +441,8 @@ def can_close(self) -> bool: def __str__(self) -> str: return ( f"Incremental at {id(self)} for resource {self.resource_name} with cursor path:" - f" {self.cursor_path} initial {self.initial_value} lv_func {self.last_value_func}" + f" {self.cursor_path} initial {self.initial_value} - {self.end_value} lv_func" + f" {self.last_value_func}" ) def _get_transformer(self, items: TDataItems) -> IncrementalTransform: @@ -569,10 +573,16 @@ def _wrap(*args: Any, **kwargs: Any) -> Any: new_incremental.__orig_class__ = p.annotation # type: ignore # set the incremental only if not yet set or if it was passed explicitly + # NOTE: if new incremental is resolved, it was passed via config injection # NOTE: the _incremental may be also set by applying hints to the resource see `set_template` in `DltResource` - if (new_incremental and p.name in bound_args.arguments) or not self._incremental: + if ( + new_incremental + and p.name in bound_args.arguments + and not new_incremental.is_resolved() + ) or not self._incremental: self._incremental = new_incremental - self._incremental.resolve() + if not self._incremental.is_resolved(): + self._incremental.resolve() # in case of transformers the bind will be called before this wrapper is set: because transformer is called for a first time late in the pipe if self._resource_name: # rebind internal _incremental from wrapper that already holds diff --git a/dlt/extract/resource.py b/dlt/extract/resource.py index 7f4eb05d6f..a64d5070b8 100644 --- a/dlt/extract/resource.py +++ b/dlt/extract/resource.py @@ -12,6 +12,7 @@ Any, Optional, ) +from typing_extensions import TypeVar, Self from dlt.common.configuration.resolve import inject_section from dlt.common.configuration.specs import known_sections @@ -76,6 +77,9 @@ def with_hints( return DataItemWithMeta(HintsMeta(hints, create_table_variant), item) +TDltResourceImpl = TypeVar("TDltResourceImpl", bound="DltResource", default="DltResource") + + class DltResource(Iterable[TDataItem], DltResourceHints): """Implements dlt resource. Contains a data pipe that wraps a generating item and table schema that can be adjusted""" @@ -114,12 +118,12 @@ def from_data( selected: bool = True, data_from: Union["DltResource", Pipe] = None, incremental: IncrementalResourceWrapper = None, - ) -> "DltResource": + ) -> Self: if data is None: - raise InvalidResourceDataTypeIsNone(name, data, NoneType) # type: ignore + raise InvalidResourceDataTypeIsNone(name, data, NoneType) if isinstance(data, DltResource): - return data + return data # type: ignore[return-value] if isinstance(data, Pipe): return cls(data, hints, selected, incremental=incremental, section=section) @@ -170,7 +174,7 @@ def name(self) -> str: """Resource name inherited from the pipe""" return self._pipe.name - def with_name(self, new_name: str) -> "DltResource": + def with_name(self: TDltResourceImpl, new_name: str) -> TDltResourceImpl: """Clones the resource with a new name. Such resource keeps separate state and loads data to `new_name` table by default.""" return self._clone(new_name=new_name, with_parent=True) @@ -226,7 +230,7 @@ def max_table_nesting(self, value: int) -> None: self._hints.setdefault("x-normalizer", {}) # type: ignore[typeddict-item] self._hints["x-normalizer"]["max_nesting"] = value # type: ignore[typeddict-item] - def pipe_data_from(self, data_from: Union["DltResource", Pipe]) -> None: + def pipe_data_from(self: TDltResourceImpl, data_from: Union[TDltResourceImpl, Pipe]) -> None: """Replaces the parent in the transformer resource pipe from which the data is piped.""" if self.is_transformer: DltResource._ensure_valid_transformer_resource(self.name, self._pipe.gen) @@ -242,7 +246,7 @@ def add_pipe(self, data: Any) -> None: # TODO: (1) self resource cannot be a transformer (2) if data is resource both self must and it must be selected/unselected + cannot be tranformer raise InvalidResourceDataTypeMultiplePipes(self.name, data, type(data)) - def select_tables(self, *table_names: Iterable[str]) -> "DltResource": + def select_tables(self: TDltResourceImpl, *table_names: Iterable[str]) -> TDltResourceImpl: """For resources that dynamically dispatch data to several tables allows to select tables that will receive data, effectively filtering out other data items. Both `with_table_name` marker and data-based (function) table name hints are supported. @@ -258,8 +262,8 @@ def _filter(item: TDataItem, meta: Any = None) -> bool: return self def add_map( - self, item_map: ItemTransformFunc[TDataItem], insert_at: int = None - ) -> "DltResource": # noqa: A003 + self: TDltResourceImpl, item_map: ItemTransformFunc[TDataItem], insert_at: int = None + ) -> TDltResourceImpl: # noqa: A003 """Adds mapping function defined in `item_map` to the resource pipe at position `inserted_at` `item_map` receives single data items, `dlt` will enumerate any lists of data items automatically @@ -278,8 +282,10 @@ def add_map( return self def add_yield_map( - self, item_map: ItemTransformFunc[Iterator[TDataItem]], insert_at: int = None - ) -> "DltResource": # noqa: A003 + self: TDltResourceImpl, + item_map: ItemTransformFunc[Iterator[TDataItem]], + insert_at: int = None, + ) -> TDltResourceImpl: # noqa: A003 """Adds generating function defined in `item_map` to the resource pipe at position `inserted_at` `item_map` receives single data items, `dlt` will enumerate any lists of data items automatically. It may yield 0 or more data items and be used to @@ -299,8 +305,8 @@ def add_yield_map( return self def add_filter( - self, item_filter: ItemTransformFunc[bool], insert_at: int = None - ) -> "DltResource": # noqa: A003 + self: TDltResourceImpl, item_filter: ItemTransformFunc[bool], insert_at: int = None + ) -> TDltResourceImpl: # noqa: A003 """Adds filter defined in `item_filter` to the resource pipe at position `inserted_at` `item_filter` receives single data items, `dlt` will enumerate any lists of data items automatically @@ -317,7 +323,7 @@ def add_filter( self._pipe.insert_step(FilterItem(item_filter), insert_at) return self - def add_limit(self, max_items: int) -> "DltResource": # noqa: A003 + def add_limit(self: TDltResourceImpl, max_items: int) -> TDltResourceImpl: # noqa: A003 """Adds a limit `max_items` to the resource pipe This mutates the encapsulated generator to stop after `max_items` items are yielded. This is useful for testing and debugging. It is @@ -376,7 +382,7 @@ def _gen_wrap(gen: TPipeStep) -> TPipeStep: self._pipe.replace_gen(partial(_gen_wrap, gen)) return self - def parallelize(self) -> "DltResource": + def parallelize(self: TDltResourceImpl) -> TDltResourceImpl: """Wraps the resource to execute each item in a threadpool to allow multiple resources to extract in parallel. The resource must be a generator or generator function or a transformer function. @@ -395,8 +401,10 @@ def parallelize(self) -> "DltResource": return self def add_step( - self, item_transform: ItemTransformFunctionWithMeta[TDataItems], insert_at: int = None - ) -> "DltResource": # noqa: A003 + self: TDltResourceImpl, + item_transform: ItemTransformFunctionWithMeta[TDataItems], + insert_at: int = None, + ) -> TDltResourceImpl: # noqa: A003 if insert_at is None: self._pipe.append_step(item_transform) else: @@ -427,7 +435,7 @@ def _set_hints( if table_schema_template.get("validator") is not None: self.validator = table_schema_template["validator"] - def bind(self, *args: Any, **kwargs: Any) -> "DltResource": + def bind(self: TDltResourceImpl, *args: Any, **kwargs: Any) -> TDltResourceImpl: """Binds the parametrized resource to passed arguments. Modifies resource pipe in place. Does not evaluate generators or iterators.""" if self._args_bound: raise TypeError(f"Parametrized resource {self.name} is not callable") @@ -466,7 +474,7 @@ def state(self) -> StrAny: with inject_section(self._get_config_section_context()): return resource_state(self.name) - def __call__(self, *args: Any, **kwargs: Any) -> "DltResource": + def __call__(self: TDltResourceImpl, *args: Any, **kwargs: Any) -> TDltResourceImpl: """Binds the parametrized resources to passed arguments. Creates and returns a bound resource. Generators and iterators are not evaluated.""" if self._args_bound: raise TypeError(f"Parametrized resource {self.name} is not callable") @@ -489,7 +497,9 @@ def __or__(self, transform: Union["DltResource", AnyFun]) -> "DltResource": else: return self.add_map(transform) - def __ror__(self, data: Union[Iterable[Any], Iterator[Any]]) -> "DltResource": + def __ror__( + self: TDltResourceImpl, data: Union[Iterable[Any], Iterator[Any]] + ) -> TDltResourceImpl: """Allows to pipe data from across resources and transform functions with | operator This is the RIGHT side OR so the self may not be a resource and the LEFT must be an object that does not implement | ie. a list @@ -525,13 +535,15 @@ def _set_explicit_args( except Exception: pass - def _clone(self, new_name: str = None, with_parent: bool = False) -> "DltResource": + def _clone( + self: TDltResourceImpl, new_name: str = None, with_parent: bool = False + ) -> TDltResourceImpl: """Creates a deep copy of a current resource, optionally renaming the resource. The clone will not be part of the source""" pipe = self._pipe if self._pipe and not self._pipe.is_empty: pipe = pipe._clone(new_name=new_name, with_parent=with_parent) # incremental and parent are already in the pipe (if any) - return DltResource( + return self.__class__( pipe, deepcopy(self._hints), selected=self.selected, diff --git a/dlt/extract/source.py b/dlt/extract/source.py index 5d9799e29c..6e78f3c5ba 100644 --- a/dlt/extract/source.py +++ b/dlt/extract/source.py @@ -1,4 +1,3 @@ -import warnings import contextlib from copy import copy import makefun diff --git a/dlt/extract/utils.py b/dlt/extract/utils.py index 2024796972..55a8b0b8c4 100644 --- a/dlt/extract/utils.py +++ b/dlt/extract/utils.py @@ -251,7 +251,7 @@ def _parallel_gen() -> TDataItems: if callable(f): if inspect.isgeneratorfunction(inspect.unwrap(f)): - return wraps(f)(_gen_wrapper) # type: ignore[arg-type] + return wraps(f)(_gen_wrapper) # type: ignore[return-value] else: def _fun_wrapper(*args: Any, **kwargs: Any) -> Any: @@ -260,7 +260,7 @@ def _curry() -> Any: return _curry - return wraps(f)(_fun_wrapper) # type: ignore[arg-type] + return wraps(f)(_fun_wrapper) # type: ignore[return-value] return _gen_wrapper() # type: ignore[return-value] diff --git a/poetry.lock b/poetry.lock index 451a26cda9..dcab5e1730 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5316,38 +5316,38 @@ files = [ [[package]] name = "mypy" -version = "1.6.1" +version = "1.10.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e5012e5cc2ac628177eaac0e83d622b2dd499e28253d4107a08ecc59ede3fc2c"}, - {file = "mypy-1.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d8fbb68711905f8912e5af474ca8b78d077447d8f3918997fecbf26943ff3cbb"}, - {file = "mypy-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21a1ad938fee7d2d96ca666c77b7c494c3c5bd88dff792220e1afbebb2925b5e"}, - {file = "mypy-1.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b96ae2c1279d1065413965c607712006205a9ac541895004a1e0d4f281f2ff9f"}, - {file = "mypy-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:40b1844d2e8b232ed92e50a4bd11c48d2daa351f9deee6c194b83bf03e418b0c"}, - {file = "mypy-1.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:81af8adaa5e3099469e7623436881eff6b3b06db5ef75e6f5b6d4871263547e5"}, - {file = "mypy-1.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8c223fa57cb154c7eab5156856c231c3f5eace1e0bed9b32a24696b7ba3c3245"}, - {file = "mypy-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8032e00ce71c3ceb93eeba63963b864bf635a18f6c0c12da6c13c450eedb183"}, - {file = "mypy-1.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c46b51de523817a0045b150ed11b56f9fff55f12b9edd0f3ed35b15a2809de0"}, - {file = "mypy-1.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:19f905bcfd9e167159b3d63ecd8cb5e696151c3e59a1742e79bc3bcb540c42c7"}, - {file = "mypy-1.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:82e469518d3e9a321912955cc702d418773a2fd1e91c651280a1bda10622f02f"}, - {file = "mypy-1.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d4473c22cc296425bbbce7e9429588e76e05bc7342da359d6520b6427bf76660"}, - {file = "mypy-1.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59a0d7d24dfb26729e0a068639a6ce3500e31d6655df8557156c51c1cb874ce7"}, - {file = "mypy-1.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfd13d47b29ed3bbaafaff7d8b21e90d827631afda134836962011acb5904b71"}, - {file = "mypy-1.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:eb4f18589d196a4cbe5290b435d135dee96567e07c2b2d43b5c4621b6501531a"}, - {file = "mypy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:41697773aa0bf53ff917aa077e2cde7aa50254f28750f9b88884acea38a16169"}, - {file = "mypy-1.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7274b0c57737bd3476d2229c6389b2ec9eefeb090bbaf77777e9d6b1b5a9d143"}, - {file = "mypy-1.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbaf4662e498c8c2e352da5f5bca5ab29d378895fa2d980630656178bd607c46"}, - {file = "mypy-1.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bb8ccb4724f7d8601938571bf3f24da0da791fe2db7be3d9e79849cb64e0ae85"}, - {file = "mypy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:68351911e85145f582b5aa6cd9ad666c8958bcae897a1bfda8f4940472463c45"}, - {file = "mypy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:49ae115da099dcc0922a7a895c1eec82c1518109ea5c162ed50e3b3594c71208"}, - {file = "mypy-1.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8b27958f8c76bed8edaa63da0739d76e4e9ad4ed325c814f9b3851425582a3cd"}, - {file = "mypy-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:925cd6a3b7b55dfba252b7c4561892311c5358c6b5a601847015a1ad4eb7d332"}, - {file = "mypy-1.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8f57e6b6927a49550da3d122f0cb983d400f843a8a82e65b3b380d3d7259468f"}, - {file = "mypy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a43ef1c8ddfdb9575691720b6352761f3f53d85f1b57d7745701041053deff30"}, - {file = "mypy-1.6.1-py3-none-any.whl", hash = "sha256:4cbe68ef919c28ea561165206a2dcb68591c50f3bcf777932323bc208d949cf1"}, - {file = "mypy-1.6.1.tar.gz", hash = "sha256:4d01c00d09a0be62a4ca3f933e315455bde83f37f892ba4b08ce92f3cf44bcc1"}, + {file = "mypy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2"}, + {file = "mypy-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99"}, + {file = "mypy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2"}, + {file = "mypy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9"}, + {file = "mypy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee"}, + {file = "mypy-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de"}, + {file = "mypy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7"}, + {file = "mypy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30"}, + {file = "mypy-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e"}, + {file = "mypy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5"}, + {file = "mypy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727"}, + {file = "mypy-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4"}, + {file = "mypy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061"}, + {file = "mypy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec"}, + {file = "mypy-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821"}, + {file = "mypy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746"}, + {file = "mypy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a"}, + {file = "mypy-1.10.0-py3-none-any.whl", hash = "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee"}, + {file = "mypy-1.10.0.tar.gz", hash = "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131"}, ] [package.dependencies] @@ -5358,6 +5358,7 @@ typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] install-types = ["pip"] +mypyc = ["setuptools (>=50)"] reports = ["lxml"] [[package]] @@ -9334,4 +9335,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.13" -content-hash = "473a4e54434d3a71dbb8e5d6bb8bd901a48313b623bf1a2e138812e36517411f" +content-hash = "c206bfd3eab8f0c9349398c3c0ed251490bab96254327cb800d45807f05d2997" diff --git a/pyproject.toml b/pyproject.toml index e58f119b6b..4c31cda736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,7 +122,7 @@ sqlfluff = "^2.3.2" types-deprecated = "^1.2.9.2" pytest-console-scripts = "^1.4.1" pytest = "^7.0.0" -mypy = "^1.6.1" +mypy = "^1.10.0" flake8 = "^5.0.0" bandit = "^1.7.0" black = "^23.7.0" diff --git a/tests/destinations/test_custom_destination.py b/tests/destinations/test_custom_destination.py index c8445a94dc..6834006689 100644 --- a/tests/destinations/test_custom_destination.py +++ b/tests/destinations/test_custom_destination.py @@ -140,7 +140,7 @@ def local_sink_func(items: TDataItems, table: TTableSchema, my_val=dlt.config.va # test decorator calls = [] - p = dlt.pipeline("sink_test", destination=dlt.destination()(local_sink_func), full_refresh=True) # type: ignore + p = dlt.pipeline("sink_test", destination=dlt.destination()(local_sink_func), full_refresh=True) p.run([1, 2, 3], table_name="items") assert len(calls) == 1 # local func does not create entry in destinations diff --git a/tests/extract/test_decorators.py b/tests/extract/test_decorators.py index dca4c0be6e..5e85552d73 100644 --- a/tests/extract/test_decorators.py +++ b/tests/extract/test_decorators.py @@ -700,6 +700,33 @@ def users(mode: str): assert list(s.users("group")) == ["group"] +class TypedResource(DltResource): + def __call__( + self: "TypedResource", api_key: dlt.TSecretValue = dlt.secrets.value, limit: int = 10 + ) -> "TypedResource": + """Pass api key and limit""" + return super().__call__(api_key, limit) + + +@dlt.resource(_impl_cls=TypedResource) +def inner_r(api_key: dlt.TSecretValue = dlt.secrets.value, limit: int = 10): + yield from ["A"] * limit + + +def test_custom_resource_impl() -> None: + inn_r = inner_r(dlt.TSecretValue("key"), limit=3) + assert isinstance(inn_r, TypedResource) + assert list(inn_r) == ["A"] * 3 + + @dlt.resource(_impl_cls=TypedResource, standalone=True) + def inner_standalone(api_key: dlt.TSecretValue = dlt.secrets.value, limit: int = 10): + yield from range(1, limit + 1) + + std_r = inner_standalone(dlt.TSecretValue("key"), limit=4) + assert isinstance(std_r, TypedResource) + assert list(std_r) == [1, 2, 3, 4] + + # wrapped flag will not create the resource but just simple function wrapper that must be called before use @dlt.resource(standalone=True) def standalone_signature(init: int, secret_end: int = dlt.secrets.value): @@ -809,8 +836,8 @@ def test_standalone_resource_with_name() -> None: assert my_tx.section == "test_decorators" assert my_tx.name == "my_tx" - # still the config comes via the function name - os.environ["SOURCES__TEST_DECORATORS__STANDALONE_TX_WITH_NAME__INIT"] = "2" + # config uses the actual resource name (my_tx) + os.environ["SOURCES__TEST_DECORATORS__MY_TX__INIT"] = "2" assert list(dlt.resource([1, 2, 3], name="x") | my_tx) == [ "my_txmy_tx", "my_txmy_txmy_txmy_tx", @@ -833,6 +860,26 @@ def standalone_name_2(_name: str): standalone_name_2("_N") +def test_standalone_resource_returns() -> None: + @dlt.resource(standalone=True) + def rv_data(name: str): + return [name] * 10 + + with pytest.raises(InvalidResourceDataTypeFunctionNotAGenerator): + rv_data("returned") + + +def test_standalone_resource_returning_resource() -> None: + @dlt.resource(standalone=True) + def rv_resource(name: str): + return dlt.resource([1, 2, 3], name=name, primary_key="value") + + r = rv_resource("returned") + assert r.name == "returned" + assert r.compute_table_schema()["columns"]["value"]["primary_key"] is True + assert list(r) == [1, 2, 3] + + def test_resource_rename_credentials_separation(): os.environ["SOURCES__TEST_DECORATORS__STANDALONE_SIGNATURE__SECRET_END"] = "5" assert list(standalone_signature(1)) == [1, 2, 3, 4] diff --git a/tests/sources/helpers/rest_client/conftest.py b/tests/sources/helpers/rest_client/conftest.py index ef63c4526d..7453c63d14 100644 --- a/tests/sources/helpers/rest_client/conftest.py +++ b/tests/sources/helpers/rest_client/conftest.py @@ -1,5 +1,5 @@ import re -from typing import NamedTuple, Callable, Pattern, List, Union, TYPE_CHECKING +from typing import NamedTuple, Callable, Pattern, List, Union, TYPE_CHECKING, Dict, List, Any import base64 from urllib.parse import urlsplit, urlunsplit @@ -11,7 +11,7 @@ if TYPE_CHECKING: RequestCallback = Callable[ - [requests_mock.Request, requests_mock.Context], Union[str, dict, list] + [requests_mock.Request, requests_mock.Context], Union[str, Dict[str, Any], List[Any]] ] ResponseSerializer = Callable[[requests_mock.Request, requests_mock.Context], str] else: