Nit: Remove py.typed file that makes Pyright incessant (#732)

* Add __all__ to __init__ modules

* Re-add py.typed

---------

Co-authored-by: Steinthor Palsson <[email protected]>

dlt/__init__.py

@@ -41,3 +41,27 @@
 
 TCredentials = _CredentialsConfiguration
 "When typing source/resource function arguments it indicates that a given argument represents credentials and should be taken from dlt.secrets. Credentials may be a string, dictionary or any other type."
+
+
+__all__ = [
+    "__version__",
+    "config",
+    "secrets",
+    "state",
+    "Schema",
+    "source",
+    "resource",
+    "transformer",
+    "defer",
+    "pipeline",
+    "run",
+    "attach",
+    "Pipeline",
+    "dbt",
+    "progress",
+    "current",
+    "mark",
+    "TSecretValue",
+    "TCredentials",
+    "sources",
+]

dlt/common/__init__.py

@@ -1,6 +1,8 @@
-from dlt.common.arithmetics import Decimal  # noqa: F401
+from dlt.common.arithmetics import Decimal
 from dlt.common.wei import Wei
-from dlt.common.pendulum import pendulum  # noqa: F401
-from dlt.common.json import json  # noqa: F401, I251
-from dlt.common.runtime.signals import sleep   # noqa: F401
-from dlt.common.runtime import logger  # noqa: F401
+from dlt.common.pendulum import pendulum
+from dlt.common.json import json
+from dlt.common.runtime.signals import sleep
+from dlt.common.runtime import logger
+
+__all__ = ["Decimal", "Wei", "pendulum", "json", "sleep", "logger"]

dlt/common/configuration/__init__.py

@@ -1,11 +1,29 @@
-from .specs.base_configuration import configspec, is_valid_hint, is_secret_hint, resolve_type  # noqa: F401
-from .specs import known_sections  # noqa: F401
-from .resolve import resolve_configuration, inject_section  # noqa: F401
-from .inject import with_config, last_config, get_fun_spec  # noqa: F401
+from .specs.base_configuration import configspec, is_valid_hint, is_secret_hint, resolve_type
+from .specs import known_sections
+from .resolve import resolve_configuration, inject_section
+from .inject import with_config, last_config, get_fun_spec
 
-from .exceptions import (  # noqa: F401
+from .exceptions import (
     ConfigFieldMissingException,
     ConfigValueCannotBeCoercedException,
     ConfigFileNotFoundException,
     ConfigurationValueError
 )
+
+
+__all__ = [
+    "configspec",
+    "is_valid_hint",
+    "is_secret_hint",
+    "resolve_type",
+    "known_sections",
+    "resolve_configuration",
+    "inject_section",
+    "with_config",
+    "last_config",
+    "get_fun_spec",
+    "ConfigFieldMissingException",
+    "ConfigValueCannotBeCoercedException",
+    "ConfigFileNotFoundException",
+    "ConfigurationValueError",
+]

dlt/common/configuration/providers/__init__.py

@@ -3,4 +3,19 @@
 from .dictionary import DictionaryProvider
 from .toml import SecretsTomlProvider, ConfigTomlProvider, TomlFileProvider, CONFIG_TOML, SECRETS_TOML, StringTomlProvider, SECRETS_TOML_KEY
 from .google_secrets import GoogleSecretsProvider
-from .context import ContextProvider
+from .context import ContextProvider
+
+__all__ = [
+    "ConfigProvider",
+    "EnvironProvider",
+    "DictionaryProvider",
+    "SecretsTomlProvider",
+    "ConfigTomlProvider",
+    "TomlFileProvider",
+    "CONFIG_TOML",
+    "SECRETS_TOML",
+    "StringTomlProvider",
+    "SECRETS_TOML_KEY",
+    "GoogleSecretsProvider",
+    "ContextProvider",
+]

dlt/common/configuration/specs/__init__.py

@@ -1,13 +1,26 @@
-from .run_configuration import RunConfiguration  # noqa: F401
-from .base_configuration import BaseConfiguration, CredentialsConfiguration, CredentialsWithDefault, ContainerInjectableContext, extract_inner_hint, is_base_configuration_inner_hint, configspec  # noqa: F401
-from .config_section_context import ConfigSectionContext  # noqa: F401
+from .run_configuration import RunConfiguration
+from .base_configuration import BaseConfiguration, CredentialsConfiguration, CredentialsWithDefault, ContainerInjectableContext, extract_inner_hint, is_base_configuration_inner_hint, configspec
+from .config_section_context import ConfigSectionContext
 
-from .gcp_credentials import GcpServiceAccountCredentialsWithoutDefaults, GcpServiceAccountCredentials, GcpOAuthCredentialsWithoutDefaults, GcpOAuthCredentials, GcpCredentials  # noqa: F401
-from .connection_string_credentials import ConnectionStringCredentials  # noqa: F401
-from .api_credentials import OAuth2Credentials  # noqa: F401
-from .aws_credentials import AwsCredentials, AwsCredentialsWithoutDefaults  # noqa: F401
-from .azure_credentials import AzureCredentials, AzureCredentialsWithoutDefaults  # noqa: F401
+from .gcp_credentials import GcpServiceAccountCredentialsWithoutDefaults, GcpServiceAccountCredentials, GcpOAuthCredentialsWithoutDefaults, GcpOAuthCredentials, GcpCredentials
+from .connection_string_credentials import ConnectionStringCredentials
+from .api_credentials import OAuth2Credentials
+from .aws_credentials import AwsCredentials, AwsCredentialsWithoutDefaults
+from .azure_credentials import AzureCredentials, AzureCredentialsWithoutDefaults
 
 
 # backward compatibility for service account credentials
-from .gcp_credentials import GcpServiceAccountCredentialsWithoutDefaults as GcpClientCredentials, GcpServiceAccountCredentials as GcpClientCredentialsWithDefault  # noqa: F401
+from .gcp_credentials import GcpServiceAccountCredentialsWithoutDefaults as GcpClientCredentials, GcpServiceAccountCredentials as GcpClientCredentialsWithDefault
+
+
+__all__ = [
+    "RunConfiguration",
+    "BaseConfiguration", "CredentialsConfiguration", "CredentialsWithDefault", "ContainerInjectableContext", "extract_inner_hint", "is_base_configuration_inner_hint", "configspec",
+    "ConfigSectionContext",
+    "GcpServiceAccountCredentialsWithoutDefaults", "GcpServiceAccountCredentials", "GcpOAuthCredentialsWithoutDefaults", "GcpOAuthCredentials", "GcpCredentials",
+    "ConnectionStringCredentials",
+    "OAuth2Credentials",
+    "AwsCredentials", "AwsCredentialsWithoutDefaults",
+    "AzureCredentials", "AzureCredentialsWithoutDefaults",
+    "GcpClientCredentials", "GcpClientCredentialsWithDefault",
+]

dlt/common/data_types/__init__.py

@@ -1,2 +1,6 @@
 from dlt.common.data_types.type_helpers import coerce_value, py_type_to_sc_type
-from dlt.common.data_types.typing import TDataType, DATA_TYPES
+from dlt.common.data_types.typing import TDataType, DATA_TYPES
+
+__all__ = [
+    "coerce_value", "py_type_to_sc_type", "TDataType", "DATA_TYPES"
+]

dlt/common/data_writers/__init__.py

@@ -1,3 +1,8 @@
 from dlt.common.data_writers.writers import DataWriter, TLoaderFileFormat
 from dlt.common.data_writers.buffered import BufferedDataWriter
-from dlt.common.data_writers.escape import escape_redshift_literal, escape_redshift_identifier, escape_bigquery_identifier
+from dlt.common.data_writers.escape import escape_redshift_literal, escape_redshift_identifier, escape_bigquery_identifier
+
+__all__ = [
+    "DataWriter", "TLoaderFileFormat", "BufferedDataWriter",
+    "escape_redshift_literal", "escape_redshift_identifier", "escape_bigquery_identifier"
+]

dlt/common/destination/__init__.py

@@ -1,2 +1,10 @@
 from dlt.common.destination.capabilities import DestinationCapabilitiesContext, TLoaderFileFormat, ALL_SUPPORTED_FILE_FORMATS
 from dlt.common.destination.reference import DestinationReference, TDestinationReferenceArg
+
+__all__ = [
+    "DestinationCapabilitiesContext",
+    "TLoaderFileFormat",
+    "ALL_SUPPORTED_FILE_FORMATS",
+    "DestinationReference",
+    "TDestinationReferenceArg",
+]

dlt/common/json/__init__.py

@@ -193,3 +193,14 @@ def custom_pua_remove(obj: Any) -> Any:
     except ImportError:
         from dlt.common.json import _simplejson as _json_simple
         json = _json_simple  # type: ignore[assignment]
+
+
+__all__ = [
+    "json",
+    "custom_encode",
+    "custom_pua_encode",
+    "custom_pua_decode",
+    "custom_pua_decode_nested",
+    "custom_pua_remove",
+    "SupportsJson"
+]

dlt/common/normalizers/__init__.py

@@ -1,3 +1,9 @@
 from dlt.common.normalizers.configuration import NormalizersConfiguration
 from dlt.common.normalizers.typing import TJSONNormalizer, TNormalizersConfig
-from dlt.common.normalizers.utils import explicit_normalizers, import_normalizers
+from dlt.common.normalizers.utils import explicit_normalizers, import_normalizers
+
+__all__ = [
+    "NormalizersConfiguration",
+    "TJSONNormalizer", "TNormalizersConfig",
+    "explicit_normalizers", "import_normalizers"
+]

dlt/common/normalizers/json/__init__.py

@@ -1,5 +1,5 @@
 import abc
-from typing import Any, Generic, Type, Iterator, Tuple, Callable, Protocol, TYPE_CHECKING, TypeVar
+from typing import Any, Generic, Type, Iterator, Tuple, Protocol, TYPE_CHECKING, TypeVar
 
 from dlt.common.typing import DictStrAny, TDataItem, StrAny
 if TYPE_CHECKING:
@@ -54,3 +54,12 @@ class SupportsDataItemNormalizer(Protocol):
 def wrap_in_dict(item: Any) -> DictStrAny:
     """Wraps `item` that is not a dictionary into dictionary that can be json normalized"""
     return {"value": item}
+
+
+__all__ = [
+    "TNormalizedRowIterator",
+    "TNormalizerConfig",
+    "DataItemNormalizer",
+    "SupportsDataItemNormalizer",
+    "wrap_in_dict",
+]

dlt/common/normalizers/naming/__init__.py

@@ -1,2 +1,6 @@
 from .naming import SupportsNamingConvention, NamingConvention
 
+__all__ = [
+    'SupportsNamingConvention', "NamingConvention"
+]
+

dlt/common/runners/__init__.py

@@ -1,5 +1,12 @@
-from . import pool_runner
 from .pool_runner import run_pool, NullExecutor
 from .runnable import Runnable, workermethod, TExecutor
 from .typing import TRunMetrics
 from .venv import Venv, VenvNotFound
+
+
+__all__ = [
+    "run_pool", "NullExecutor",
+    "Runnable", "workermethod", "TExecutor",
+    "TRunMetrics",
+    "Venv", "VenvNotFound"
+]

dlt/common/runtime/__init__.py

@@ -1 +1,3 @@
-from .init import initialize_runtime
+from .init import initialize_runtime
+
+__all__ = ["initialize_runtime"]

dlt/common/schema/__init__.py

@@ -1,4 +1,9 @@
-from dlt.common.schema.typing import TSchemaUpdate, TSchemaTables, TTableSchema, TStoredSchema, TTableSchemaColumns, TColumnHint, TColumnSchema, TColumnSchemaBase  # noqa: F401
-from dlt.common.schema.typing import COLUMN_HINTS  # noqa: F401
-from dlt.common.schema.schema import Schema  # noqa: F401
-from dlt.common.schema.utils import verify_schema_hash  # noqa: F401
+from dlt.common.schema.typing import TSchemaUpdate, TSchemaTables, TTableSchema, TStoredSchema, TTableSchemaColumns, TColumnHint, TColumnSchema, TColumnSchemaBase
+from dlt.common.schema.typing import COLUMN_HINTS
+from dlt.common.schema.schema import Schema
+from dlt.common.schema.utils import verify_schema_hash
+
+__all__ = [
+    "TSchemaUpdate", "TSchemaTables", "TTableSchema", "TStoredSchema", "TTableSchemaColumns", "TColumnHint",
+    "TColumnSchema", "TColumnSchemaBase", "COLUMN_HINTS", "Schema", "verify_schema_hash"
+]

dlt/common/storages/__init__.py

@@ -1,9 +1,22 @@
-from .file_storage import FileStorage  # noqa: F401
-from .versioned_storage import VersionedStorage  # noqa: F401
-from .schema_storage import SchemaStorage  # noqa: F401
-from .live_schema_storage import LiveSchemaStorage  # noqa: F401
-from .normalize_storage import NormalizeStorage  # noqa: F401
-from .load_storage import LoadStorage  # noqa: F401
-from .data_item_storage import DataItemStorage  # noqa: F401
-from .configuration import LoadStorageConfiguration, NormalizeStorageConfiguration, SchemaStorageConfiguration, TSchemaFileFormat, FilesystemConfiguration  # noqa: F401
-from .fsspec_filesystem import fsspec_from_config, fsspec_filesystem  # noqa: F401
+from .file_storage import FileStorage
+from .versioned_storage import VersionedStorage
+from .schema_storage import SchemaStorage
+from .live_schema_storage import LiveSchemaStorage
+from .normalize_storage import NormalizeStorage
+from .load_storage import LoadStorage
+from .data_item_storage import DataItemStorage
+from .configuration import LoadStorageConfiguration, NormalizeStorageConfiguration, SchemaStorageConfiguration, TSchemaFileFormat, FilesystemConfiguration
+from .fsspec_filesystem import fsspec_from_config, fsspec_filesystem
+
+
+__all__ = [
+    "FileStorage",
+    "VersionedStorage",
+    "SchemaStorage",
+    "LiveSchemaStorage",
+    "NormalizeStorage",
+    "LoadStorage",
+    "DataItemStorage",
+    "LoadStorageConfiguration", "NormalizeStorageConfiguration", "SchemaStorageConfiguration", "TSchemaFileFormat", "FilesystemConfiguration",
+    "fsspec_from_config", "fsspec_filesystem",
+]

dlt/destinations/athena/__init__.py

@@ -6,7 +6,6 @@
 from dlt.common.schema.schema import Schema
 from dlt.common.data_writers.escape import escape_athena_identifier
 from dlt.common.arithmetics import DEFAULT_NUMERIC_PRECISION, DEFAULT_NUMERIC_SCALE
-from dlt.common.wei import EVM_DECIMAL_PRECISION
 
 from dlt.destinations.athena.configuration import AthenaClientConfiguration
 from dlt.common.destination.reference import JobClientBase, DestinationClientConfiguration
@@ -47,4 +46,6 @@ def client(schema: Schema, initial_config: DestinationClientConfiguration = conf
 
 
 def spec() -> Type[DestinationClientConfiguration]:
-    return AthenaClientConfiguration
+    return AthenaClientConfiguration
+
+

dlt/extract/incremental/__init__.py

@@ -1,8 +1,7 @@
 import os
-from typing import Generic, TypeVar, Any, Optional, Callable, List, TypedDict, get_args, get_origin, Sequence, Type, Dict
+from typing import Generic, Any, Optional, get_args, get_origin, Type, Dict
 import inspect
-from functools import wraps, partial
-from datetime import datetime  # noqa: I251
+from functools import wraps
 
 try:
     import pandas as pd
@@ -12,25 +11,22 @@
 import dlt
 from dlt.common.exceptions import MissingDependencyException
 from dlt.common import pendulum, logger
-from dlt.common.json import json
-from dlt.common.jsonpath import compile_path, find_values, JSONPath
+from dlt.common.jsonpath import compile_path
 from dlt.common.typing import TDataItem, TDataItems, TFun, extract_inner_type, get_generic_type_argument_from_instance, is_optional_type
 from dlt.common.schema.typing import TColumnNames
 from dlt.common.configuration import configspec, ConfigurationValueError
 from dlt.common.configuration.specs import BaseConfiguration
 from dlt.common.pipeline import resource_state
-from dlt.common.utils import digest128
 from dlt.common.data_types.type_helpers import coerce_from_date_types, coerce_value, py_type_to_sc_type
 
-from dlt.extract.exceptions import IncrementalUnboundError, PipeException
+from dlt.extract.exceptions import IncrementalUnboundError
 from dlt.extract.incremental.exceptions import IncrementalCursorPathMissing, IncrementalPrimaryKeyMissing
 from dlt.extract.incremental.typing import IncrementalColumnState, TCursorValue, LastValueFunc
 from dlt.extract.pipe import Pipe
-from dlt.extract.utils import resolve_column_value
-from dlt.extract.typing import SupportsPipe, TTableHintTemplate, MapItem, YieldMapItem, FilterItem, ItemTransform
+from dlt.extract.typing import SupportsPipe, TTableHintTemplate, ItemTransform
 from dlt.extract.incremental.transform import JsonIncremental, ArrowIncremental, IncrementalTransform
 try:
-    from dlt.common.libs.pyarrow import is_arrow_item, pyarrow as pa, TAnyArrowItem
+    from dlt.common.libs.pyarrow import is_arrow_item
 except MissingDependencyException:
     is_arrow_item = lambda item: False
 
@@ -456,3 +452,15 @@ def __call__(self, item: TDataItems, meta: Any = None) -> Optional[TDataItems]:
         if self._incremental.primary_key is None:
             self._incremental.primary_key = self.primary_key
         return self._incremental(item, meta)
+
+
+__all__ = [
+    "Incremental",
+    "IncrementalResourceWrapper",
+    "IncrementalColumnState",
+    "IncrementalCursorPathMissing",
+    "IncrementalPrimaryKeyMissing",
+    "IncrementalUnboundError",
+    "LastValueFunc",
+    "TCursorValue",
+]

dlt/load/__init__.py

@@ -1 +1,3 @@
 from dlt.load.load import Load
+
+__all__ = ["Load"]

dlt/normalize/__init__.py

@@ -1 +1,3 @@
-from .normalize import Normalize
+from .normalize import Normalize
+
+__all__ = ['Normalize']

dlt/pipeline/__init__.py

@@ -9,7 +9,6 @@
 from dlt.common.configuration.inject import get_orig_args, last_config
 from dlt.common.destination.reference import DestinationReference, TDestinationReferenceArg
 from dlt.common.pipeline import LoadInfo, PipelineContext, get_dlt_pipelines_dir
-from dlt.common.data_writers import TLoaderFileFormat
 
 from dlt.pipeline.configuration import PipelineConfiguration, ensure_correct_pipeline_kwargs
 from dlt.pipeline.pipeline import Pipeline

dlt/sources/__init__.py

@@ -4,4 +4,15 @@
 from dlt.common.typing import TDataItem, TDataItems
 from . import credentials
 from . import config
-from . import filesystem
+from . import filesystem
+
+__all__ = [
+    "DltSource",
+    "DltResource",
+    "TDataItem",
+    "TDataItems",
+    "incremental",
+    "credentials",
+    "config",
+    "filesystem",
+]