diff --git a/dlt/__init__.py b/dlt/__init__.py index c3725af118..f5dde3f204 100644 --- a/dlt/__init__.py +++ b/dlt/__init__.py @@ -41,3 +41,27 @@ TCredentials = _CredentialsConfiguration "When typing source/resource function arguments it indicates that a given argument represents credentials and should be taken from dlt.secrets. Credentials may be a string, dictionary or any other type." + + +__all__ = [ + "__version__", + "config", + "secrets", + "state", + "Schema", + "source", + "resource", + "transformer", + "defer", + "pipeline", + "run", + "attach", + "Pipeline", + "dbt", + "progress", + "current", + "mark", + "TSecretValue", + "TCredentials", + "sources", +] diff --git a/dlt/common/__init__.py b/dlt/common/__init__.py index 222cb3d5d6..466fd7c546 100644 --- a/dlt/common/__init__.py +++ b/dlt/common/__init__.py @@ -1,6 +1,8 @@ -from dlt.common.arithmetics import Decimal # noqa: F401 +from dlt.common.arithmetics import Decimal from dlt.common.wei import Wei -from dlt.common.pendulum import pendulum # noqa: F401 -from dlt.common.json import json # noqa: F401, I251 -from dlt.common.runtime.signals import sleep # noqa: F401 -from dlt.common.runtime import logger # noqa: F401 \ No newline at end of file +from dlt.common.pendulum import pendulum +from dlt.common.json import json +from dlt.common.runtime.signals import sleep +from dlt.common.runtime import logger + +__all__ = ["Decimal", "Wei", "pendulum", "json", "sleep", "logger"] diff --git a/dlt/common/configuration/__init__.py b/dlt/common/configuration/__init__.py index 2f22314ff4..a5ffd3e7b8 100644 --- a/dlt/common/configuration/__init__.py +++ b/dlt/common/configuration/__init__.py @@ -1,11 +1,29 @@ -from .specs.base_configuration import configspec, is_valid_hint, is_secret_hint, resolve_type # noqa: F401 -from .specs import known_sections # noqa: F401 -from .resolve import resolve_configuration, inject_section # noqa: F401 -from .inject import with_config, last_config, get_fun_spec # noqa: F401 +from .specs.base_configuration import configspec, is_valid_hint, is_secret_hint, resolve_type +from .specs import known_sections +from .resolve import resolve_configuration, inject_section +from .inject import with_config, last_config, get_fun_spec -from .exceptions import ( # noqa: F401 +from .exceptions import ( ConfigFieldMissingException, ConfigValueCannotBeCoercedException, ConfigFileNotFoundException, ConfigurationValueError ) + + +__all__ = [ + "configspec", + "is_valid_hint", + "is_secret_hint", + "resolve_type", + "known_sections", + "resolve_configuration", + "inject_section", + "with_config", + "last_config", + "get_fun_spec", + "ConfigFieldMissingException", + "ConfigValueCannotBeCoercedException", + "ConfigFileNotFoundException", + "ConfigurationValueError", +] diff --git a/dlt/common/configuration/providers/__init__.py b/dlt/common/configuration/providers/__init__.py index f79aff223e..76268c14b1 100644 --- a/dlt/common/configuration/providers/__init__.py +++ b/dlt/common/configuration/providers/__init__.py @@ -3,4 +3,19 @@ from .dictionary import DictionaryProvider from .toml import SecretsTomlProvider, ConfigTomlProvider, TomlFileProvider, CONFIG_TOML, SECRETS_TOML, StringTomlProvider, SECRETS_TOML_KEY from .google_secrets import GoogleSecretsProvider -from .context import ContextProvider \ No newline at end of file +from .context import ContextProvider + +__all__ = [ + "ConfigProvider", + "EnvironProvider", + "DictionaryProvider", + "SecretsTomlProvider", + "ConfigTomlProvider", + "TomlFileProvider", + "CONFIG_TOML", + "SECRETS_TOML", + "StringTomlProvider", + "SECRETS_TOML_KEY", + "GoogleSecretsProvider", + "ContextProvider", +] diff --git a/dlt/common/configuration/specs/__init__.py b/dlt/common/configuration/specs/__init__.py index 629a11fb6e..2a033b6bbd 100644 --- a/dlt/common/configuration/specs/__init__.py +++ b/dlt/common/configuration/specs/__init__.py @@ -1,13 +1,26 @@ -from .run_configuration import RunConfiguration # noqa: F401 -from .base_configuration import BaseConfiguration, CredentialsConfiguration, CredentialsWithDefault, ContainerInjectableContext, extract_inner_hint, is_base_configuration_inner_hint, configspec # noqa: F401 -from .config_section_context import ConfigSectionContext # noqa: F401 +from .run_configuration import RunConfiguration +from .base_configuration import BaseConfiguration, CredentialsConfiguration, CredentialsWithDefault, ContainerInjectableContext, extract_inner_hint, is_base_configuration_inner_hint, configspec +from .config_section_context import ConfigSectionContext -from .gcp_credentials import GcpServiceAccountCredentialsWithoutDefaults, GcpServiceAccountCredentials, GcpOAuthCredentialsWithoutDefaults, GcpOAuthCredentials, GcpCredentials # noqa: F401 -from .connection_string_credentials import ConnectionStringCredentials # noqa: F401 -from .api_credentials import OAuth2Credentials # noqa: F401 -from .aws_credentials import AwsCredentials, AwsCredentialsWithoutDefaults # noqa: F401 -from .azure_credentials import AzureCredentials, AzureCredentialsWithoutDefaults # noqa: F401 +from .gcp_credentials import GcpServiceAccountCredentialsWithoutDefaults, GcpServiceAccountCredentials, GcpOAuthCredentialsWithoutDefaults, GcpOAuthCredentials, GcpCredentials +from .connection_string_credentials import ConnectionStringCredentials +from .api_credentials import OAuth2Credentials +from .aws_credentials import AwsCredentials, AwsCredentialsWithoutDefaults +from .azure_credentials import AzureCredentials, AzureCredentialsWithoutDefaults # backward compatibility for service account credentials -from .gcp_credentials import GcpServiceAccountCredentialsWithoutDefaults as GcpClientCredentials, GcpServiceAccountCredentials as GcpClientCredentialsWithDefault # noqa: F401 +from .gcp_credentials import GcpServiceAccountCredentialsWithoutDefaults as GcpClientCredentials, GcpServiceAccountCredentials as GcpClientCredentialsWithDefault + + +__all__ = [ + "RunConfiguration", + "BaseConfiguration", "CredentialsConfiguration", "CredentialsWithDefault", "ContainerInjectableContext", "extract_inner_hint", "is_base_configuration_inner_hint", "configspec", + "ConfigSectionContext", + "GcpServiceAccountCredentialsWithoutDefaults", "GcpServiceAccountCredentials", "GcpOAuthCredentialsWithoutDefaults", "GcpOAuthCredentials", "GcpCredentials", + "ConnectionStringCredentials", + "OAuth2Credentials", + "AwsCredentials", "AwsCredentialsWithoutDefaults", + "AzureCredentials", "AzureCredentialsWithoutDefaults", + "GcpClientCredentials", "GcpClientCredentialsWithDefault", +] diff --git a/dlt/common/data_types/__init__.py b/dlt/common/data_types/__init__.py index 9ad4df37fe..83e123f124 100644 --- a/dlt/common/data_types/__init__.py +++ b/dlt/common/data_types/__init__.py @@ -1,2 +1,6 @@ from dlt.common.data_types.type_helpers import coerce_value, py_type_to_sc_type -from dlt.common.data_types.typing import TDataType, DATA_TYPES \ No newline at end of file +from dlt.common.data_types.typing import TDataType, DATA_TYPES + +__all__ = [ + "coerce_value", "py_type_to_sc_type", "TDataType", "DATA_TYPES" +] diff --git a/dlt/common/data_writers/__init__.py b/dlt/common/data_writers/__init__.py index 89d4607c90..5865466b8f 100644 --- a/dlt/common/data_writers/__init__.py +++ b/dlt/common/data_writers/__init__.py @@ -1,3 +1,8 @@ from dlt.common.data_writers.writers import DataWriter, TLoaderFileFormat from dlt.common.data_writers.buffered import BufferedDataWriter -from dlt.common.data_writers.escape import escape_redshift_literal, escape_redshift_identifier, escape_bigquery_identifier \ No newline at end of file +from dlt.common.data_writers.escape import escape_redshift_literal, escape_redshift_identifier, escape_bigquery_identifier + +__all__ = [ + "DataWriter", "TLoaderFileFormat", "BufferedDataWriter", + "escape_redshift_literal", "escape_redshift_identifier", "escape_bigquery_identifier" +] diff --git a/dlt/common/destination/__init__.py b/dlt/common/destination/__init__.py index 79940cfbc0..88b5d5ef06 100644 --- a/dlt/common/destination/__init__.py +++ b/dlt/common/destination/__init__.py @@ -1,2 +1,10 @@ from dlt.common.destination.capabilities import DestinationCapabilitiesContext, TLoaderFileFormat, ALL_SUPPORTED_FILE_FORMATS from dlt.common.destination.reference import DestinationReference, TDestinationReferenceArg + +__all__ = [ + "DestinationCapabilitiesContext", + "TLoaderFileFormat", + "ALL_SUPPORTED_FILE_FORMATS", + "DestinationReference", + "TDestinationReferenceArg", +] diff --git a/dlt/common/json/__init__.py b/dlt/common/json/__init__.py index cdcc609d03..edb48643ef 100644 --- a/dlt/common/json/__init__.py +++ b/dlt/common/json/__init__.py @@ -193,3 +193,14 @@ def custom_pua_remove(obj: Any) -> Any: except ImportError: from dlt.common.json import _simplejson as _json_simple json = _json_simple # type: ignore[assignment] + + +__all__ = [ + "json", + "custom_encode", + "custom_pua_encode", + "custom_pua_decode", + "custom_pua_decode_nested", + "custom_pua_remove", + "SupportsJson" +] diff --git a/dlt/common/normalizers/__init__.py b/dlt/common/normalizers/__init__.py index cfe1f5beb3..e106419df9 100644 --- a/dlt/common/normalizers/__init__.py +++ b/dlt/common/normalizers/__init__.py @@ -1,3 +1,9 @@ from dlt.common.normalizers.configuration import NormalizersConfiguration from dlt.common.normalizers.typing import TJSONNormalizer, TNormalizersConfig -from dlt.common.normalizers.utils import explicit_normalizers, import_normalizers \ No newline at end of file +from dlt.common.normalizers.utils import explicit_normalizers, import_normalizers + +__all__ = [ + "NormalizersConfiguration", + "TJSONNormalizer", "TNormalizersConfig", + "explicit_normalizers", "import_normalizers" +] diff --git a/dlt/common/normalizers/json/__init__.py b/dlt/common/normalizers/json/__init__.py index 07ffd52e2e..e1c5c3b846 100644 --- a/dlt/common/normalizers/json/__init__.py +++ b/dlt/common/normalizers/json/__init__.py @@ -1,5 +1,5 @@ import abc -from typing import Any, Generic, Type, Iterator, Tuple, Callable, Protocol, TYPE_CHECKING, TypeVar +from typing import Any, Generic, Type, Iterator, Tuple, Protocol, TYPE_CHECKING, TypeVar from dlt.common.typing import DictStrAny, TDataItem, StrAny if TYPE_CHECKING: @@ -54,3 +54,12 @@ class SupportsDataItemNormalizer(Protocol): def wrap_in_dict(item: Any) -> DictStrAny: """Wraps `item` that is not a dictionary into dictionary that can be json normalized""" return {"value": item} + + +__all__ = [ + "TNormalizedRowIterator", + "TNormalizerConfig", + "DataItemNormalizer", + "SupportsDataItemNormalizer", + "wrap_in_dict", +] diff --git a/dlt/common/normalizers/naming/__init__.py b/dlt/common/normalizers/naming/__init__.py index 28f3ae5bd4..c8c08ddd63 100644 --- a/dlt/common/normalizers/naming/__init__.py +++ b/dlt/common/normalizers/naming/__init__.py @@ -1,2 +1,6 @@ from .naming import SupportsNamingConvention, NamingConvention +__all__ = [ + 'SupportsNamingConvention', "NamingConvention" +] + diff --git a/dlt/common/runners/__init__.py b/dlt/common/runners/__init__.py index 36eb8ba6ad..9af668ce87 100644 --- a/dlt/common/runners/__init__.py +++ b/dlt/common/runners/__init__.py @@ -1,5 +1,12 @@ -from . import pool_runner from .pool_runner import run_pool, NullExecutor from .runnable import Runnable, workermethod, TExecutor from .typing import TRunMetrics from .venv import Venv, VenvNotFound + + +__all__ = [ + "run_pool", "NullExecutor", + "Runnable", "workermethod", "TExecutor", + "TRunMetrics", + "Venv", "VenvNotFound" +] diff --git a/dlt/common/runtime/__init__.py b/dlt/common/runtime/__init__.py index 65f5c11696..8a6d78cf67 100644 --- a/dlt/common/runtime/__init__.py +++ b/dlt/common/runtime/__init__.py @@ -1 +1,3 @@ -from .init import initialize_runtime \ No newline at end of file +from .init import initialize_runtime + +__all__ = ["initialize_runtime"] diff --git a/dlt/common/schema/__init__.py b/dlt/common/schema/__init__.py index 6db3f21cef..1a3b4db223 100644 --- a/dlt/common/schema/__init__.py +++ b/dlt/common/schema/__init__.py @@ -1,4 +1,9 @@ -from dlt.common.schema.typing import TSchemaUpdate, TSchemaTables, TTableSchema, TStoredSchema, TTableSchemaColumns, TColumnHint, TColumnSchema, TColumnSchemaBase # noqa: F401 -from dlt.common.schema.typing import COLUMN_HINTS # noqa: F401 -from dlt.common.schema.schema import Schema # noqa: F401 -from dlt.common.schema.utils import verify_schema_hash # noqa: F401 +from dlt.common.schema.typing import TSchemaUpdate, TSchemaTables, TTableSchema, TStoredSchema, TTableSchemaColumns, TColumnHint, TColumnSchema, TColumnSchemaBase +from dlt.common.schema.typing import COLUMN_HINTS +from dlt.common.schema.schema import Schema +from dlt.common.schema.utils import verify_schema_hash + +__all__ = [ + "TSchemaUpdate", "TSchemaTables", "TTableSchema", "TStoredSchema", "TTableSchemaColumns", "TColumnHint", + "TColumnSchema", "TColumnSchemaBase", "COLUMN_HINTS", "Schema", "verify_schema_hash" +] diff --git a/dlt/common/storages/__init__.py b/dlt/common/storages/__init__.py index 05a4f36119..c18b8ab04d 100644 --- a/dlt/common/storages/__init__.py +++ b/dlt/common/storages/__init__.py @@ -1,9 +1,22 @@ -from .file_storage import FileStorage # noqa: F401 -from .versioned_storage import VersionedStorage # noqa: F401 -from .schema_storage import SchemaStorage # noqa: F401 -from .live_schema_storage import LiveSchemaStorage # noqa: F401 -from .normalize_storage import NormalizeStorage # noqa: F401 -from .load_storage import LoadStorage # noqa: F401 -from .data_item_storage import DataItemStorage # noqa: F401 -from .configuration import LoadStorageConfiguration, NormalizeStorageConfiguration, SchemaStorageConfiguration, TSchemaFileFormat, FilesystemConfiguration # noqa: F401 -from .fsspec_filesystem import fsspec_from_config, fsspec_filesystem # noqa: F401 \ No newline at end of file +from .file_storage import FileStorage +from .versioned_storage import VersionedStorage +from .schema_storage import SchemaStorage +from .live_schema_storage import LiveSchemaStorage +from .normalize_storage import NormalizeStorage +from .load_storage import LoadStorage +from .data_item_storage import DataItemStorage +from .configuration import LoadStorageConfiguration, NormalizeStorageConfiguration, SchemaStorageConfiguration, TSchemaFileFormat, FilesystemConfiguration +from .fsspec_filesystem import fsspec_from_config, fsspec_filesystem + + +__all__ = [ + "FileStorage", + "VersionedStorage", + "SchemaStorage", + "LiveSchemaStorage", + "NormalizeStorage", + "LoadStorage", + "DataItemStorage", + "LoadStorageConfiguration", "NormalizeStorageConfiguration", "SchemaStorageConfiguration", "TSchemaFileFormat", "FilesystemConfiguration", + "fsspec_from_config", "fsspec_filesystem", +] diff --git a/dlt/destinations/athena/__init__.py b/dlt/destinations/athena/__init__.py index d19a0ffdb7..1fd7f14d57 100644 --- a/dlt/destinations/athena/__init__.py +++ b/dlt/destinations/athena/__init__.py @@ -6,7 +6,6 @@ from dlt.common.schema.schema import Schema from dlt.common.data_writers.escape import escape_athena_identifier from dlt.common.arithmetics import DEFAULT_NUMERIC_PRECISION, DEFAULT_NUMERIC_SCALE -from dlt.common.wei import EVM_DECIMAL_PRECISION from dlt.destinations.athena.configuration import AthenaClientConfiguration from dlt.common.destination.reference import JobClientBase, DestinationClientConfiguration @@ -47,4 +46,6 @@ def client(schema: Schema, initial_config: DestinationClientConfiguration = conf def spec() -> Type[DestinationClientConfiguration]: - return AthenaClientConfiguration \ No newline at end of file + return AthenaClientConfiguration + + diff --git a/dlt/extract/incremental/__init__.py b/dlt/extract/incremental/__init__.py index 652adc19d2..6d042aa15d 100644 --- a/dlt/extract/incremental/__init__.py +++ b/dlt/extract/incremental/__init__.py @@ -1,8 +1,7 @@ import os -from typing import Generic, TypeVar, Any, Optional, Callable, List, TypedDict, get_args, get_origin, Sequence, Type, Dict +from typing import Generic, Any, Optional, get_args, get_origin, Type, Dict import inspect -from functools import wraps, partial -from datetime import datetime # noqa: I251 +from functools import wraps try: import pandas as pd @@ -12,25 +11,22 @@ import dlt from dlt.common.exceptions import MissingDependencyException from dlt.common import pendulum, logger -from dlt.common.json import json -from dlt.common.jsonpath import compile_path, find_values, JSONPath +from dlt.common.jsonpath import compile_path from dlt.common.typing import TDataItem, TDataItems, TFun, extract_inner_type, get_generic_type_argument_from_instance, is_optional_type from dlt.common.schema.typing import TColumnNames from dlt.common.configuration import configspec, ConfigurationValueError from dlt.common.configuration.specs import BaseConfiguration from dlt.common.pipeline import resource_state -from dlt.common.utils import digest128 from dlt.common.data_types.type_helpers import coerce_from_date_types, coerce_value, py_type_to_sc_type -from dlt.extract.exceptions import IncrementalUnboundError, PipeException +from dlt.extract.exceptions import IncrementalUnboundError from dlt.extract.incremental.exceptions import IncrementalCursorPathMissing, IncrementalPrimaryKeyMissing from dlt.extract.incremental.typing import IncrementalColumnState, TCursorValue, LastValueFunc from dlt.extract.pipe import Pipe -from dlt.extract.utils import resolve_column_value -from dlt.extract.typing import SupportsPipe, TTableHintTemplate, MapItem, YieldMapItem, FilterItem, ItemTransform +from dlt.extract.typing import SupportsPipe, TTableHintTemplate, ItemTransform from dlt.extract.incremental.transform import JsonIncremental, ArrowIncremental, IncrementalTransform try: - from dlt.common.libs.pyarrow import is_arrow_item, pyarrow as pa, TAnyArrowItem + from dlt.common.libs.pyarrow import is_arrow_item except MissingDependencyException: is_arrow_item = lambda item: False @@ -456,3 +452,15 @@ def __call__(self, item: TDataItems, meta: Any = None) -> Optional[TDataItems]: if self._incremental.primary_key is None: self._incremental.primary_key = self.primary_key return self._incremental(item, meta) + + +__all__ = [ + "Incremental", + "IncrementalResourceWrapper", + "IncrementalColumnState", + "IncrementalCursorPathMissing", + "IncrementalPrimaryKeyMissing", + "IncrementalUnboundError", + "LastValueFunc", + "TCursorValue", +] diff --git a/dlt/load/__init__.py b/dlt/load/__init__.py index 0a6c97ed3d..a181d672ba 100644 --- a/dlt/load/__init__.py +++ b/dlt/load/__init__.py @@ -1 +1,3 @@ from dlt.load.load import Load + +__all__ = ["Load"] diff --git a/dlt/normalize/__init__.py b/dlt/normalize/__init__.py index a40a5eaa7e..25d6a4afd3 100644 --- a/dlt/normalize/__init__.py +++ b/dlt/normalize/__init__.py @@ -1 +1,3 @@ -from .normalize import Normalize \ No newline at end of file +from .normalize import Normalize + +__all__ = ['Normalize'] diff --git a/dlt/pipeline/__init__.py b/dlt/pipeline/__init__.py index b6539c1b78..71c37c40ba 100644 --- a/dlt/pipeline/__init__.py +++ b/dlt/pipeline/__init__.py @@ -9,7 +9,6 @@ from dlt.common.configuration.inject import get_orig_args, last_config from dlt.common.destination.reference import DestinationReference, TDestinationReferenceArg from dlt.common.pipeline import LoadInfo, PipelineContext, get_dlt_pipelines_dir -from dlt.common.data_writers import TLoaderFileFormat from dlt.pipeline.configuration import PipelineConfiguration, ensure_correct_pipeline_kwargs from dlt.pipeline.pipeline import Pipeline diff --git a/dlt/sources/__init__.py b/dlt/sources/__init__.py index f080e736d3..6e418a3cb2 100644 --- a/dlt/sources/__init__.py +++ b/dlt/sources/__init__.py @@ -4,4 +4,15 @@ from dlt.common.typing import TDataItem, TDataItems from . import credentials from . import config -from . import filesystem \ No newline at end of file +from . import filesystem + +__all__ = [ + "DltSource", + "DltResource", + "TDataItem", + "TDataItems", + "incremental", + "credentials", + "config", + "filesystem", +] diff --git a/dlt/sources/credentials.py b/dlt/sources/credentials.py index 70e88beb49..5815324d56 100644 --- a/dlt/sources/credentials.py +++ b/dlt/sources/credentials.py @@ -2,4 +2,18 @@ from dlt.common.configuration.specs import ConnectionStringCredentials from dlt.common.configuration.specs import OAuth2Credentials from dlt.common.configuration.specs import CredentialsConfiguration, configspec -from dlt.common.storages.configuration import FileSystemCredentials, FilesystemConfiguration \ No newline at end of file +from dlt.common.storages.configuration import FileSystemCredentials, FilesystemConfiguration + + +__all__ = [ + "GcpServiceAccountCredentials", + "GcpOAuthCredentials", + "GcpCredentials", + "ConnectionStringCredentials", + "OAuth2Credentials", + "CredentialsConfiguration", + "configspec", + "FileSystemCredentials", + "FilesystemConfiguration", +] + diff --git a/dlt/sources/filesystem.py b/dlt/sources/filesystem.py index af874788c1..bb18a15f20 100644 --- a/dlt/sources/filesystem.py +++ b/dlt/sources/filesystem.py @@ -1 +1,3 @@ -from dlt.common.storages.fsspec_filesystem import FileItem, FileItemDict, fsspec_filesystem, glob_files \ No newline at end of file +from dlt.common.storages.fsspec_filesystem import FileItem, FileItemDict, fsspec_filesystem, glob_files + +__all__ = ["FileItem", "FileItemDict", "fsspec_filesystem", "glob_files"] diff --git a/dlt/sources/helpers/requests/__init__.py b/dlt/sources/helpers/requests/__init__.py index 1d0a37b6e5..39d286e29d 100644 --- a/dlt/sources/helpers/requests/__init__.py +++ b/dlt/sources/helpers/requests/__init__.py @@ -26,3 +26,22 @@ def init(config: RunConfiguration) -> None: """Initialize the default requests client from config""" client.update_from_config(config) + + +__all__ = [ + "client", + "get", + "post", + "put", + "patch", + "delete", + "options", + "head", + "request", + "init", + "Session", + "Request", "Response", "ConnectionError", "ConnectTimeout", "FileModeWarning", "HTTPError", "ReadTimeout", + "RequestException", "Timeout", "TooManyRedirects", "URLRequired", "ChunkedEncodingError", "RetryError" + "Client", + "RetryError" +] diff --git a/py.typed b/py.typed deleted file mode 100644 index e69de29bb2..0000000000