Add __all__ in more modules and begin cleaning up unusued imports

dlt/cli/utils.py

@@ -1,11 +1,8 @@
 import ast
 import os
-import tempfile
 from typing import Callable
 
-from dlt.common import git
 from dlt.common.reflection.utils import set_ast_parents
-from dlt.common.storages import FileStorage
 from dlt.common.typing import TFun
 from dlt.common.configuration import resolve_configuration
 from dlt.common.configuration.specs import RunConfiguration
@@ -39,7 +36,7 @@ def parse_init_script(command: str, script_source: str, init_script_name: str) -
 
 def ensure_git_command(command: str) -> None:
     try:
-        import git
+        import git  # noqa: F401
     except ImportError as imp_ex:
         if "Bad git executable" not in str(imp_ex):
             raise

dlt/common/arithmetics.py

@@ -1,7 +1,7 @@
 import decimal # noqa: I251
 from contextlib import contextmanager
 from typing import Iterator
-from decimal import ROUND_HALF_UP, Decimal, Inexact, DivisionByZero, DefaultContext, InvalidOperation, localcontext, Context, Subnormal, ConversionSyntax  # noqa: I251
+from decimal import ROUND_HALF_UP, Decimal, Inexact, DivisionByZero, InvalidOperation, localcontext, Context, Subnormal
 
 
 DEFAULT_NUMERIC_PRECISION = 38

dlt/common/configuration/specs/azure_credentials.py

@@ -1,13 +1,8 @@
 from typing import Optional, Dict, Any
 
 from dlt.common import pendulum
-from dlt.common.exceptions import MissingDependencyException
 from dlt.common.typing import TSecretStrValue
 from dlt.common.configuration.specs import CredentialsConfiguration, CredentialsWithDefault, configspec
-from dlt.common.configuration.specs.exceptions import InvalidBoto3Session
-from dlt import version
-
-import fsspec
 
 
 @configspec

dlt/common/data_types/type_helpers.py

@@ -2,15 +2,15 @@
 import base64
 import datetime  # noqa: I251
 from collections.abc import Mapping as C_Mapping, Sequence as C_Sequence
-from typing import Any, Type, Literal, Union, cast
+from typing import Any, Type, Union
 from enum import Enum
 
-from dlt.common import pendulum, json, Decimal, Wei
+from dlt.common import json, Decimal, Wei
 from dlt.common.json import custom_pua_remove
 from dlt.common.json._simplejson import custom_encode as json_custom_encode
 from dlt.common.arithmetics import InvalidOperation
 from dlt.common.data_types.typing import TDataType
-from dlt.common.time import ensure_pendulum_datetime, parse_iso_like_datetime, ensure_pendulum_date, ensure_pendulum_time
+from dlt.common.time import ensure_pendulum_datetime, ensure_pendulum_date, ensure_pendulum_time
 from dlt.common.utils import map_nested_in_place, str2bool
 
 

dlt/common/destination/reference.py

@@ -1,16 +1,13 @@
-from abc import ABC, abstractmethod, abstractproperty
+from abc import ABC, abstractmethod
 from importlib import import_module
 from types import TracebackType, ModuleType
-from typing import ClassVar, Final, Optional, NamedTuple, Literal, Sequence, Iterable, Type, Protocol, Union, TYPE_CHECKING, cast, List, ContextManager, Dict, Any
-from contextlib import contextmanager
+from typing import ClassVar, Final, Optional, NamedTuple, Literal, Sequence, Iterable, Type, Protocol, Union, TYPE_CHECKING, cast, List, ContextManager
 import datetime  # noqa: 251
 from copy import deepcopy
 
 from dlt.common import logger
 from dlt.common.exceptions import IdentifierTooLongException, InvalidDestinationReference, UnknownDestinationModule
 from dlt.common.schema import Schema, TTableSchema, TSchemaTables
-from dlt.common.schema.typing import TWriteDisposition
-from dlt.common.schema.exceptions import InvalidDatasetName
 from dlt.common.schema.utils import get_write_disposition, get_table_format
 from dlt.common.configuration import configspec
 from dlt.common.configuration.specs import BaseConfiguration, CredentialsConfiguration

dlt/common/runtime/collector.py

@@ -121,7 +121,7 @@ def __init__(self, log_period: float = 1.0, logger: Union[logging.Logger, TextIO
         self.messages: Dict[str, Optional[str]] = None
         if dump_system_stats:
             try:
-                import psutil
+                import psutil  # noqa: F401
             except ImportError:
                 self._log(logging.WARNING, "psutil dependency is not installed and mem stats will not be available. add psutil to your environment or pass dump_system_stats argument as False to disable warning.")
                 dump_system_stats = False
@@ -313,7 +313,7 @@ def __init__(self, single_bar: bool = False, **enlighten_kwargs: Any) -> None:
             global enlighten
 
             import enlighten
-            from enlighten import Counter as EnlCounter, StatusBar as EnlStatusBar, Manager as EnlManager
+            from enlighten import Counter as EnlCounter, StatusBar as EnlStatusBar, Manager as EnlManager  # noqa: F401
         except ModuleNotFoundError:
             raise MissingDependencyException("EnlightenCollector", ["enlighten"], "We need enlighten to display progress bars with a space for log messages.")
         self.single_bar = single_bar

dlt/common/schema/schema.py

@@ -1,10 +1,10 @@
 import yaml
 from copy import copy, deepcopy
-from typing import ClassVar, Dict, List, Mapping, Optional, Sequence, Tuple, Any, cast
+from typing import ClassVar, Dict, List, Mapping, Optional, Sequence, Tuple, Any
 from dlt.common import json
 
 from dlt.common.utils import extend_list_deduplicated
-from dlt.common.typing import DictStrAny, StrAny, REPattern, SupportsVariant, VARIANT_FIELD_FORMAT, TDataItem
+from dlt.common.typing import DictStrAny, StrAny, REPattern, VARIANT_FIELD_FORMAT, TDataItem
 from dlt.common.normalizers import TNormalizersConfig, explicit_normalizers, import_normalizers
 from dlt.common.normalizers.naming import NamingConvention
 from dlt.common.normalizers.json import DataItemNormalizer, TNormalizedRowIterator

dlt/common/schema/utils.py

@@ -7,21 +7,20 @@
 
 from dlt.common import json
 from dlt.common.data_types import TDataType
-from dlt.common.exceptions import DictValidationException, MissingDependencyException
+from dlt.common.exceptions import DictValidationException
 from dlt.common.normalizers import explicit_normalizers
 from dlt.common.normalizers.naming import NamingConvention
 from dlt.common.normalizers.naming.snake_case import NamingConvention as SnakeCase
-from dlt.common.typing import DictStrAny, REPattern, is_dict_generic_type
-from dlt.common.validation import TCustomValidator, validate_dict, validate_dict_ignoring_xkeys
+from dlt.common.typing import DictStrAny, REPattern
+from dlt.common.validation import TCustomValidator, validate_dict_ignoring_xkeys
 from dlt.common.schema import detections
 from dlt.common.schema.typing import (COLUMN_HINTS, SCHEMA_ENGINE_VERSION, LOADS_TABLE_NAME, SIMPLE_REGEX_PREFIX, VERSION_TABLE_NAME, TColumnName, TPartialTableSchema, TSchemaTables, TSchemaUpdate,
                                       TSimpleRegex, TStoredSchema, TTableSchema, TTableSchemaColumns, TColumnSchemaBase, TColumnSchema, TColumnProp, TTableFormat,
                                       TColumnHint, TTypeDetectionFunc, TTypeDetections, TWriteDisposition)
-from dlt.common.schema.exceptions import (CannotCoerceColumnException, ParentTableNotFoundException, SchemaEngineNoUpgradePathException, SchemaException,
-                                          TablePropertiesConflictException, InvalidSchemaName, UnknownTableException)
+from dlt.common.schema.exceptions import (CannotCoerceColumnException, ParentTableNotFoundException, SchemaEngineNoUpgradePathException,
+                                          TablePropertiesConflictException, InvalidSchemaName)
 
 from dlt.common.normalizers.utils import import_normalizers
-from dlt.common.schema.typing import TAnySchemaColumns
 
 
 RE_NON_ALPHANUMERIC_UNDERSCORE = re.compile(r"[^a-zA-Z\d_]")

dlt/destinations/athena/athena.py

@@ -14,10 +14,9 @@
 
 from dlt.common import logger
 from dlt.common.utils import without_none
-from dlt.common.data_types import TDataType
-from dlt.common.schema import TColumnSchema, Schema, TSchemaTables, TTableSchema
-from dlt.common.schema.typing import TTableSchema, TColumnType, TWriteDisposition, TTableFormat
-from dlt.common.schema.utils import table_schema_has_type, get_table_format
+from dlt.common.schema import TColumnSchema, Schema, TTableSchema
+from dlt.common.schema.typing import TTableSchema, TColumnType, TTableFormat
+from dlt.common.schema.utils import table_schema_has_type
 from dlt.common.destination import DestinationCapabilitiesContext
 from dlt.common.destination.reference import LoadJob, FollowupJob
 from dlt.common.destination.reference import TLoadJobState, NewLoadJob, SupportsStagingDestination
@@ -27,7 +26,7 @@
 
 from dlt.destinations.typing import DBApi, DBTransaction
 from dlt.destinations.exceptions import DatabaseTerminalException, DatabaseTransientException, DatabaseUndefinedRelation, LoadJobTerminalException
-from dlt.destinations.athena import capabilities
+from dlt.destinations.athena import capabilities  # noqa: F401
 from dlt.destinations.sql_client import SqlClientBase, DBApiCursorImpl, raise_database_error, raise_open_connection_error
 from dlt.destinations.typing import DBApiCursor
 from dlt.destinations.job_client_impl import SqlJobClientWithStaging

dlt/destinations/dummy/dummy.py

@@ -5,15 +5,14 @@
 
 from dlt.common import pendulum
 from dlt.common.schema import Schema, TTableSchema, TSchemaTables
-from dlt.common.schema.typing import TWriteDisposition
 from dlt.common.storages import FileStorage
 from dlt.common.destination import DestinationCapabilitiesContext
 from dlt.common.destination.reference import FollowupJob, NewLoadJob, TLoadJobState, LoadJob, JobClientBase
 
 from dlt.destinations.exceptions import (LoadJobNotExistsException, LoadJobInvalidStateTransitionException,
                                             DestinationTerminalException, DestinationTransientException)
 
-from dlt.destinations.dummy import capabilities
+from dlt.destinations.dummy import capabilities  # noqa: F401
 from dlt.destinations.dummy.configuration import DummyClientConfiguration
 
 

dlt/destinations/job_client_impl.py

@@ -1,26 +1,23 @@
 import os
 from abc import abstractmethod
 import base64
-import binascii
 import contextlib
 from copy import copy
 import datetime  # noqa: 251
 from types import TracebackType
-from typing import Any, ClassVar, List, NamedTuple, Optional, Sequence, Tuple, Type, Iterable, Iterator, ContextManager, cast
+from typing import Any, ClassVar, List, Optional, Sequence, Tuple, Type, Iterable, Iterator
 import zlib
 import re
 
 from dlt.common import json, pendulum, logger
-from dlt.common.data_types import TDataType
-from dlt.common.schema.typing import COLUMN_HINTS, TColumnType, TColumnSchemaBase, TTableSchema, TWriteDisposition, TTableFormat
+from dlt.common.schema.typing import COLUMN_HINTS, TColumnType, TColumnSchemaBase, TTableSchema, TTableFormat
 from dlt.common.storages import FileStorage
 from dlt.common.schema import TColumnSchema, Schema, TTableSchemaColumns, TSchemaTables
-from dlt.common.destination.reference import StateInfo, StorageSchemaInfo,WithStateSync, DestinationClientConfiguration, DestinationClientDwhConfiguration, DestinationClientDwhWithStagingConfiguration, NewLoadJob, WithStagingDataset, TLoadJobState, LoadJob, JobClientBase, FollowupJob, CredentialsConfiguration
+from dlt.common.destination.reference import StateInfo, StorageSchemaInfo,WithStateSync, DestinationClientConfiguration, DestinationClientDwhConfiguration, NewLoadJob, WithStagingDataset, TLoadJobState, LoadJob, JobClientBase, FollowupJob, CredentialsConfiguration
 from dlt.common.utils import concat_strings_with_limit
-from dlt.destinations.exceptions import DatabaseUndefinedRelation, DestinationSchemaTampered, DestinationSchemaWillNotUpdate
+from dlt.destinations.exceptions import DatabaseUndefinedRelation, DestinationSchemaTampered
 from dlt.destinations.job_impl import EmptyLoadJobWithoutFollowup, NewReferenceJob
 from dlt.destinations.sql_jobs import SqlMergeJob, SqlStagingCopyJob
-from dlt.common.schema.typing import LOADS_TABLE_NAME, VERSION_TABLE_NAME
 
 from dlt.destinations.typing import TNativeConn
 from dlt.destinations.sql_client import SqlClientBase

dlt/destinations/job_impl.py

@@ -4,7 +4,6 @@
 from dlt.common.storages import FileStorage
 
 from dlt.common.destination.reference import NewLoadJob, FollowupJob, TLoadJobState, LoadJob
-from dlt.common.storages.load_storage import ParsedLoadJobFileName
 
 class EmptyLoadJobWithoutFollowup(LoadJob):
     def __init__(self, file_name: str, status: TLoadJobState, exception: str = None) -> None:

dlt/destinations/motherduck/sql_client.py

@@ -1,14 +1,7 @@
-import duckdb
+from typing import ClassVar
 
-from contextlib import contextmanager
-from typing import Any, AnyStr, ClassVar, Iterator, Optional, Sequence
 from dlt.common.destination import DestinationCapabilitiesContext
-
-from dlt.destinations.exceptions import DatabaseTerminalException, DatabaseTransientException, DatabaseUndefinedRelation
-from dlt.destinations.typing import DBApi, DBApiCursor, DBTransaction, DataFrame
-from dlt.destinations.sql_client import SqlClientBase, DBApiCursorImpl, raise_database_error, raise_open_connection_error
-
-from dlt.destinations.duckdb.sql_client import DuckDbSqlClient, DuckDBDBApiCursorImpl
+from dlt.destinations.duckdb.sql_client import DuckDbSqlClient
 from dlt.destinations.motherduck import capabilities
 from dlt.destinations.motherduck.configuration import MotherDuckCredentials
 

dlt/destinations/mssql/mssql.py

@@ -1,18 +1,15 @@
-from typing import ClassVar, Dict, Optional, Sequence, List, Any, Tuple
+from typing import ClassVar, Dict, Optional, Sequence, List, Any
 
-from dlt.common.wei import EVM_DECIMAL_PRECISION
 from dlt.common.destination.reference import NewLoadJob
 from dlt.common.destination import DestinationCapabilitiesContext
-from dlt.common.data_types import TDataType
 from dlt.common.schema import TColumnSchema, TColumnHint, Schema
 from dlt.common.schema.typing import TTableSchema, TColumnType, TTableFormat
-from dlt.common.utils import uniq_id
 
 from dlt.destinations.sql_jobs import SqlStagingCopyJob, SqlMergeJob, SqlJobParams
 
 from dlt.destinations.insert_job_client import InsertValuesJobClient
 
-from dlt.destinations.mssql import capabilities
+from dlt.destinations.mssql import capabilities as make_capabilities
 from dlt.destinations.mssql.sql_client import PyOdbcMsSqlClient
 from dlt.destinations.mssql.configuration import MsSqlClientConfiguration
 from dlt.destinations.sql_client import SqlClientBase
@@ -120,7 +117,7 @@ def _new_temp_table_name(cls, name_prefix: str) -> str:
 
 class MsSqlClient(InsertValuesJobClient):
 
-    capabilities: ClassVar[DestinationCapabilitiesContext] = capabilities()
+    capabilities: ClassVar[DestinationCapabilitiesContext] = make_capabilities()
 
     def __init__(self, schema: Schema, config: MsSqlClientConfiguration) -> None:
         sql_client = PyOdbcMsSqlClient(

dlt/destinations/path_utils.py

@@ -1,5 +1,5 @@
 # this can probably go some other place, but it is shared by destinations, so for now it is here
-from typing import List, Sequence, Tuple
+from typing import List, Sequence
 
 import pendulum
 import re

dlt/destinations/postgres/postgres.py

@@ -1,17 +1,15 @@
 from typing import ClassVar, Dict, Optional, Sequence, List, Any
 
-from dlt.common.wei import EVM_DECIMAL_PRECISION
 from dlt.common.destination.reference import NewLoadJob
 from dlt.common.destination import DestinationCapabilitiesContext
-from dlt.common.data_types import TDataType
 from dlt.common.schema import TColumnSchema, TColumnHint, Schema
 from dlt.common.schema.typing import TTableSchema, TColumnType, TTableFormat
 
 from dlt.destinations.sql_jobs import SqlStagingCopyJob, SqlJobParams
 
 from dlt.destinations.insert_job_client import InsertValuesJobClient
 
-from dlt.destinations.postgres import capabilities
+from dlt.destinations.postgres import capabilities  # noqa: F401
 from dlt.destinations.postgres.sql_client import Psycopg2SqlClient
 from dlt.destinations.postgres.configuration import PostgresClientConfiguration
 from dlt.destinations.sql_client import SqlClientBase

dlt/destinations/snowflake/snowflake.py

@@ -4,7 +4,6 @@
 from dlt.common.destination import DestinationCapabilitiesContext
 from dlt.common.destination.reference import FollowupJob, NewLoadJob, TLoadJobState, LoadJob, CredentialsConfiguration, SupportsStagingDestination
 from dlt.common.configuration.specs import AwsCredentialsWithoutDefaults, AzureCredentialsWithoutDefaults
-from dlt.common.data_types import TDataType
 from dlt.common.storages.file_storage import FileStorage
 from dlt.common.schema import TColumnSchema, Schema, TTableSchemaColumns
 from dlt.common.schema.typing import TTableSchema, TColumnType, TTableFormat
@@ -14,7 +13,7 @@
 from dlt.destinations.job_impl import EmptyLoadJob
 from dlt.destinations.exceptions import LoadJobTerminalException
 
-from dlt.destinations.snowflake import capabilities
+from dlt.destinations.snowflake import capabilities  # noqa: F401
 from dlt.destinations.snowflake.configuration import SnowflakeClientConfiguration
 from dlt.destinations.snowflake.sql_client import SnowflakeSqlClient
 from dlt.destinations.sql_jobs import SqlStagingCopyJob, SqlJobParams

dlt/destinations/sql_client.py

@@ -3,7 +3,7 @@
 from functools import wraps
 import inspect
 from types import TracebackType
-from typing import Any, ClassVar, ContextManager, Generic, Iterator, Optional, Sequence, Tuple, Type, AnyStr, List
+from typing import Any, ClassVar, ContextManager, Generic, Iterator, Optional, Sequence, Type, AnyStr, List
 
 from dlt.common.typing import TFun
 from dlt.common.destination import DestinationCapabilitiesContext

dlt/destinations/sql_jobs.py

@@ -1,4 +1,4 @@
-from typing import Any, Callable, List, Sequence, Tuple, cast, TypedDict, Optional
+from typing import Any, List, Sequence, Tuple, cast, TypedDict, Optional
 
 import yaml
 from dlt.common.runtime.logger import pretty_format_exception
@@ -229,4 +229,4 @@ def gen_merge_sql(cls, table_chain: Sequence[TTableSchema], sql_client: SqlClien
             sql.append(insert_sql)
             # -- DELETE FROM {staging_table_name} WHERE 1=1;
 
-        return sql
+        return sql

dlt/extract/extract.py

@@ -1,7 +1,6 @@
 import contextlib
 import os
-from typing import ClassVar, List, Set, Dict, Type, Any, Sequence, Optional
-from collections import defaultdict
+from typing import ClassVar, List, Set, Dict, Any, Optional
 
 from dlt.common.configuration.container import Container
 from dlt.common.configuration.resolve import inject_section
@@ -15,7 +14,7 @@
 from dlt.common.utils import uniq_id
 from dlt.common.typing import TDataItems, TDataItem
 from dlt.common.schema import Schema, utils, TSchemaUpdate
-from dlt.common.schema.typing import TColumnSchema, TTableSchemaColumns
+from dlt.common.schema.typing import TTableSchemaColumns
 from dlt.common.storages import NormalizeStorageConfiguration, NormalizeStorage, DataItemStorage, FileStorage
 from dlt.common.configuration.specs import known_sections
 

dlt/extract/incremental/transform.py

@@ -15,8 +15,8 @@
 from dlt.common.utils import digest128
 from dlt.common.json import json
 from dlt.common import pendulum
-from dlt.common.typing import TDataItem, TDataItems
-from dlt.common.jsonpath import TJsonPath, find_values, JSONPathFields, compile_path
+from dlt.common.typing import TDataItem
+from dlt.common.jsonpath import find_values, JSONPathFields, compile_path
 from dlt.extract.incremental.exceptions import IncrementalCursorPathMissing, IncrementalPrimaryKeyMissing
 from dlt.extract.incremental.typing import IncrementalColumnState, TCursorValue, LastValueFunc
 from dlt.extract.utils import resolve_column_value

dlt/extract/schema.py

@@ -1,16 +1,15 @@
 from copy import copy, deepcopy
-from collections.abc import Mapping as C_Mapping
-from typing import List, TypedDict, cast, Any
+from typing import TypedDict, cast, Any
 
 from dlt.common.schema.utils import DEFAULT_WRITE_DISPOSITION, merge_columns, new_column, new_table
-from dlt.common.schema.typing import TColumnNames, TColumnProp, TColumnSchema, TPartialTableSchema, TTableSchemaColumns, TWriteDisposition, TAnySchemaColumns, TTableFormat
+from dlt.common.schema.typing import TColumnNames, TColumnProp, TPartialTableSchema, TTableSchemaColumns, TWriteDisposition, TAnySchemaColumns, TTableFormat
 from dlt.common.typing import TDataItem
 from dlt.common.utils import update_dict_nested
 from dlt.common.validation import validate_dict_ignoring_xkeys
 
 from dlt.extract.incremental import Incremental
 from dlt.extract.typing import TFunHintTemplate, TTableHintTemplate, ValidateItem
-from dlt.extract.exceptions import DataItemRequiredForDynamicTableHints, InconsistentTableTemplate, TableNameMissing
+from dlt.extract.exceptions import DataItemRequiredForDynamicTableHints, InconsistentTableTemplate
 from dlt.extract.utils import ensure_table_schema_columns, ensure_table_schema_columns_hint
 from dlt.extract.validation import get_column_validator