diff --git a/dlt/cli/utils.py b/dlt/cli/utils.py index 996770b40d..0e5de19b8d 100644 --- a/dlt/cli/utils.py +++ b/dlt/cli/utils.py @@ -1,11 +1,8 @@ import ast import os -import tempfile from typing import Callable -from dlt.common import git from dlt.common.reflection.utils import set_ast_parents -from dlt.common.storages import FileStorage from dlt.common.typing import TFun from dlt.common.configuration import resolve_configuration from dlt.common.configuration.specs import RunConfiguration @@ -39,7 +36,7 @@ def parse_init_script(command: str, script_source: str, init_script_name: str) - def ensure_git_command(command: str) -> None: try: - import git + import git # noqa: F401 except ImportError as imp_ex: if "Bad git executable" not in str(imp_ex): raise diff --git a/dlt/common/arithmetics.py b/dlt/common/arithmetics.py index 5277acad4f..681b564829 100644 --- a/dlt/common/arithmetics.py +++ b/dlt/common/arithmetics.py @@ -1,7 +1,7 @@ import decimal # noqa: I251 from contextlib import contextmanager from typing import Iterator -from decimal import ROUND_HALF_UP, Decimal, Inexact, DivisionByZero, DefaultContext, InvalidOperation, localcontext, Context, Subnormal, ConversionSyntax # noqa: I251 +from decimal import ROUND_HALF_UP, Decimal, Inexact, DivisionByZero, InvalidOperation, localcontext, Context, Subnormal DEFAULT_NUMERIC_PRECISION = 38 diff --git a/dlt/common/configuration/specs/azure_credentials.py b/dlt/common/configuration/specs/azure_credentials.py index 49393a6343..3bfd30d5f1 100644 --- a/dlt/common/configuration/specs/azure_credentials.py +++ b/dlt/common/configuration/specs/azure_credentials.py @@ -1,13 +1,8 @@ from typing import Optional, Dict, Any from dlt.common import pendulum -from dlt.common.exceptions import MissingDependencyException from dlt.common.typing import TSecretStrValue from dlt.common.configuration.specs import CredentialsConfiguration, CredentialsWithDefault, configspec -from dlt.common.configuration.specs.exceptions import InvalidBoto3Session -from dlt import version - -import fsspec @configspec diff --git a/dlt/common/data_types/type_helpers.py b/dlt/common/data_types/type_helpers.py index f42f81b06f..584c024b25 100644 --- a/dlt/common/data_types/type_helpers.py +++ b/dlt/common/data_types/type_helpers.py @@ -2,15 +2,15 @@ import base64 import datetime # noqa: I251 from collections.abc import Mapping as C_Mapping, Sequence as C_Sequence -from typing import Any, Type, Literal, Union, cast +from typing import Any, Type, Union from enum import Enum -from dlt.common import pendulum, json, Decimal, Wei +from dlt.common import json, Decimal, Wei from dlt.common.json import custom_pua_remove from dlt.common.json._simplejson import custom_encode as json_custom_encode from dlt.common.arithmetics import InvalidOperation from dlt.common.data_types.typing import TDataType -from dlt.common.time import ensure_pendulum_datetime, parse_iso_like_datetime, ensure_pendulum_date, ensure_pendulum_time +from dlt.common.time import ensure_pendulum_datetime, ensure_pendulum_date, ensure_pendulum_time from dlt.common.utils import map_nested_in_place, str2bool diff --git a/dlt/common/destination/reference.py b/dlt/common/destination/reference.py index 13172b41e9..46ddca2fbc 100644 --- a/dlt/common/destination/reference.py +++ b/dlt/common/destination/reference.py @@ -1,16 +1,13 @@ -from abc import ABC, abstractmethod, abstractproperty +from abc import ABC, abstractmethod from importlib import import_module from types import TracebackType, ModuleType -from typing import ClassVar, Final, Optional, NamedTuple, Literal, Sequence, Iterable, Type, Protocol, Union, TYPE_CHECKING, cast, List, ContextManager, Dict, Any -from contextlib import contextmanager +from typing import ClassVar, Final, Optional, NamedTuple, Literal, Sequence, Iterable, Type, Protocol, Union, TYPE_CHECKING, cast, List, ContextManager import datetime # noqa: 251 from copy import deepcopy from dlt.common import logger from dlt.common.exceptions import IdentifierTooLongException, InvalidDestinationReference, UnknownDestinationModule from dlt.common.schema import Schema, TTableSchema, TSchemaTables -from dlt.common.schema.typing import TWriteDisposition -from dlt.common.schema.exceptions import InvalidDatasetName from dlt.common.schema.utils import get_write_disposition, get_table_format from dlt.common.configuration import configspec from dlt.common.configuration.specs import BaseConfiguration, CredentialsConfiguration diff --git a/dlt/common/runtime/collector.py b/dlt/common/runtime/collector.py index 5e7143241e..6321f44f03 100644 --- a/dlt/common/runtime/collector.py +++ b/dlt/common/runtime/collector.py @@ -121,7 +121,7 @@ def __init__(self, log_period: float = 1.0, logger: Union[logging.Logger, TextIO self.messages: Dict[str, Optional[str]] = None if dump_system_stats: try: - import psutil + import psutil # noqa: F401 except ImportError: self._log(logging.WARNING, "psutil dependency is not installed and mem stats will not be available. add psutil to your environment or pass dump_system_stats argument as False to disable warning.") dump_system_stats = False @@ -313,7 +313,7 @@ def __init__(self, single_bar: bool = False, **enlighten_kwargs: Any) -> None: global enlighten import enlighten - from enlighten import Counter as EnlCounter, StatusBar as EnlStatusBar, Manager as EnlManager + from enlighten import Counter as EnlCounter, StatusBar as EnlStatusBar, Manager as EnlManager # noqa: F401 except ModuleNotFoundError: raise MissingDependencyException("EnlightenCollector", ["enlighten"], "We need enlighten to display progress bars with a space for log messages.") self.single_bar = single_bar diff --git a/dlt/common/schema/schema.py b/dlt/common/schema/schema.py index 77a5ae8e8e..4c133bdc27 100644 --- a/dlt/common/schema/schema.py +++ b/dlt/common/schema/schema.py @@ -1,10 +1,10 @@ import yaml from copy import copy, deepcopy -from typing import ClassVar, Dict, List, Mapping, Optional, Sequence, Tuple, Any, cast +from typing import ClassVar, Dict, List, Mapping, Optional, Sequence, Tuple, Any from dlt.common import json from dlt.common.utils import extend_list_deduplicated -from dlt.common.typing import DictStrAny, StrAny, REPattern, SupportsVariant, VARIANT_FIELD_FORMAT, TDataItem +from dlt.common.typing import DictStrAny, StrAny, REPattern, VARIANT_FIELD_FORMAT, TDataItem from dlt.common.normalizers import TNormalizersConfig, explicit_normalizers, import_normalizers from dlt.common.normalizers.naming import NamingConvention from dlt.common.normalizers.json import DataItemNormalizer, TNormalizedRowIterator diff --git a/dlt/common/schema/utils.py b/dlt/common/schema/utils.py index f2075ce85d..44d98fd514 100644 --- a/dlt/common/schema/utils.py +++ b/dlt/common/schema/utils.py @@ -7,21 +7,20 @@ from dlt.common import json from dlt.common.data_types import TDataType -from dlt.common.exceptions import DictValidationException, MissingDependencyException +from dlt.common.exceptions import DictValidationException from dlt.common.normalizers import explicit_normalizers from dlt.common.normalizers.naming import NamingConvention from dlt.common.normalizers.naming.snake_case import NamingConvention as SnakeCase -from dlt.common.typing import DictStrAny, REPattern, is_dict_generic_type -from dlt.common.validation import TCustomValidator, validate_dict, validate_dict_ignoring_xkeys +from dlt.common.typing import DictStrAny, REPattern +from dlt.common.validation import TCustomValidator, validate_dict_ignoring_xkeys from dlt.common.schema import detections from dlt.common.schema.typing import (COLUMN_HINTS, SCHEMA_ENGINE_VERSION, LOADS_TABLE_NAME, SIMPLE_REGEX_PREFIX, VERSION_TABLE_NAME, TColumnName, TPartialTableSchema, TSchemaTables, TSchemaUpdate, TSimpleRegex, TStoredSchema, TTableSchema, TTableSchemaColumns, TColumnSchemaBase, TColumnSchema, TColumnProp, TTableFormat, TColumnHint, TTypeDetectionFunc, TTypeDetections, TWriteDisposition) -from dlt.common.schema.exceptions import (CannotCoerceColumnException, ParentTableNotFoundException, SchemaEngineNoUpgradePathException, SchemaException, - TablePropertiesConflictException, InvalidSchemaName, UnknownTableException) +from dlt.common.schema.exceptions import (CannotCoerceColumnException, ParentTableNotFoundException, SchemaEngineNoUpgradePathException, + TablePropertiesConflictException, InvalidSchemaName) from dlt.common.normalizers.utils import import_normalizers -from dlt.common.schema.typing import TAnySchemaColumns RE_NON_ALPHANUMERIC_UNDERSCORE = re.compile(r"[^a-zA-Z\d_]") diff --git a/dlt/destinations/athena/athena.py b/dlt/destinations/athena/athena.py index 44d020c127..e34cdf62d9 100644 --- a/dlt/destinations/athena/athena.py +++ b/dlt/destinations/athena/athena.py @@ -14,10 +14,9 @@ from dlt.common import logger from dlt.common.utils import without_none -from dlt.common.data_types import TDataType -from dlt.common.schema import TColumnSchema, Schema, TSchemaTables, TTableSchema -from dlt.common.schema.typing import TTableSchema, TColumnType, TWriteDisposition, TTableFormat -from dlt.common.schema.utils import table_schema_has_type, get_table_format +from dlt.common.schema import TColumnSchema, Schema, TTableSchema +from dlt.common.schema.typing import TTableSchema, TColumnType, TTableFormat +from dlt.common.schema.utils import table_schema_has_type from dlt.common.destination import DestinationCapabilitiesContext from dlt.common.destination.reference import LoadJob, FollowupJob from dlt.common.destination.reference import TLoadJobState, NewLoadJob, SupportsStagingDestination @@ -27,7 +26,7 @@ from dlt.destinations.typing import DBApi, DBTransaction from dlt.destinations.exceptions import DatabaseTerminalException, DatabaseTransientException, DatabaseUndefinedRelation, LoadJobTerminalException -from dlt.destinations.athena import capabilities +from dlt.destinations.athena import capabilities # noqa: F401 from dlt.destinations.sql_client import SqlClientBase, DBApiCursorImpl, raise_database_error, raise_open_connection_error from dlt.destinations.typing import DBApiCursor from dlt.destinations.job_client_impl import SqlJobClientWithStaging diff --git a/dlt/destinations/dummy/dummy.py b/dlt/destinations/dummy/dummy.py index c8cac05d3a..223bfcb78e 100644 --- a/dlt/destinations/dummy/dummy.py +++ b/dlt/destinations/dummy/dummy.py @@ -5,7 +5,6 @@ from dlt.common import pendulum from dlt.common.schema import Schema, TTableSchema, TSchemaTables -from dlt.common.schema.typing import TWriteDisposition from dlt.common.storages import FileStorage from dlt.common.destination import DestinationCapabilitiesContext from dlt.common.destination.reference import FollowupJob, NewLoadJob, TLoadJobState, LoadJob, JobClientBase @@ -13,7 +12,7 @@ from dlt.destinations.exceptions import (LoadJobNotExistsException, LoadJobInvalidStateTransitionException, DestinationTerminalException, DestinationTransientException) -from dlt.destinations.dummy import capabilities +from dlt.destinations.dummy import capabilities # noqa: F401 from dlt.destinations.dummy.configuration import DummyClientConfiguration diff --git a/dlt/destinations/job_client_impl.py b/dlt/destinations/job_client_impl.py index 7dabf278c2..d25a75b760 100644 --- a/dlt/destinations/job_client_impl.py +++ b/dlt/destinations/job_client_impl.py @@ -1,26 +1,23 @@ import os from abc import abstractmethod import base64 -import binascii import contextlib from copy import copy import datetime # noqa: 251 from types import TracebackType -from typing import Any, ClassVar, List, NamedTuple, Optional, Sequence, Tuple, Type, Iterable, Iterator, ContextManager, cast +from typing import Any, ClassVar, List, Optional, Sequence, Tuple, Type, Iterable, Iterator import zlib import re from dlt.common import json, pendulum, logger -from dlt.common.data_types import TDataType -from dlt.common.schema.typing import COLUMN_HINTS, TColumnType, TColumnSchemaBase, TTableSchema, TWriteDisposition, TTableFormat +from dlt.common.schema.typing import COLUMN_HINTS, TColumnType, TColumnSchemaBase, TTableSchema, TTableFormat from dlt.common.storages import FileStorage from dlt.common.schema import TColumnSchema, Schema, TTableSchemaColumns, TSchemaTables -from dlt.common.destination.reference import StateInfo, StorageSchemaInfo,WithStateSync, DestinationClientConfiguration, DestinationClientDwhConfiguration, DestinationClientDwhWithStagingConfiguration, NewLoadJob, WithStagingDataset, TLoadJobState, LoadJob, JobClientBase, FollowupJob, CredentialsConfiguration +from dlt.common.destination.reference import StateInfo, StorageSchemaInfo,WithStateSync, DestinationClientConfiguration, DestinationClientDwhConfiguration, NewLoadJob, WithStagingDataset, TLoadJobState, LoadJob, JobClientBase, FollowupJob, CredentialsConfiguration from dlt.common.utils import concat_strings_with_limit -from dlt.destinations.exceptions import DatabaseUndefinedRelation, DestinationSchemaTampered, DestinationSchemaWillNotUpdate +from dlt.destinations.exceptions import DatabaseUndefinedRelation, DestinationSchemaTampered from dlt.destinations.job_impl import EmptyLoadJobWithoutFollowup, NewReferenceJob from dlt.destinations.sql_jobs import SqlMergeJob, SqlStagingCopyJob -from dlt.common.schema.typing import LOADS_TABLE_NAME, VERSION_TABLE_NAME from dlt.destinations.typing import TNativeConn from dlt.destinations.sql_client import SqlClientBase diff --git a/dlt/destinations/job_impl.py b/dlt/destinations/job_impl.py index fb3ba48b6d..594d849a5e 100644 --- a/dlt/destinations/job_impl.py +++ b/dlt/destinations/job_impl.py @@ -4,7 +4,6 @@ from dlt.common.storages import FileStorage from dlt.common.destination.reference import NewLoadJob, FollowupJob, TLoadJobState, LoadJob -from dlt.common.storages.load_storage import ParsedLoadJobFileName class EmptyLoadJobWithoutFollowup(LoadJob): def __init__(self, file_name: str, status: TLoadJobState, exception: str = None) -> None: diff --git a/dlt/destinations/motherduck/sql_client.py b/dlt/destinations/motherduck/sql_client.py index 2fc664a2e8..970291e435 100644 --- a/dlt/destinations/motherduck/sql_client.py +++ b/dlt/destinations/motherduck/sql_client.py @@ -1,14 +1,7 @@ -import duckdb +from typing import ClassVar -from contextlib import contextmanager -from typing import Any, AnyStr, ClassVar, Iterator, Optional, Sequence from dlt.common.destination import DestinationCapabilitiesContext - -from dlt.destinations.exceptions import DatabaseTerminalException, DatabaseTransientException, DatabaseUndefinedRelation -from dlt.destinations.typing import DBApi, DBApiCursor, DBTransaction, DataFrame -from dlt.destinations.sql_client import SqlClientBase, DBApiCursorImpl, raise_database_error, raise_open_connection_error - -from dlt.destinations.duckdb.sql_client import DuckDbSqlClient, DuckDBDBApiCursorImpl +from dlt.destinations.duckdb.sql_client import DuckDbSqlClient from dlt.destinations.motherduck import capabilities from dlt.destinations.motherduck.configuration import MotherDuckCredentials diff --git a/dlt/destinations/mssql/mssql.py b/dlt/destinations/mssql/mssql.py index cd999441ff..2d30ecb47b 100644 --- a/dlt/destinations/mssql/mssql.py +++ b/dlt/destinations/mssql/mssql.py @@ -1,18 +1,15 @@ -from typing import ClassVar, Dict, Optional, Sequence, List, Any, Tuple +from typing import ClassVar, Dict, Optional, Sequence, List, Any -from dlt.common.wei import EVM_DECIMAL_PRECISION from dlt.common.destination.reference import NewLoadJob from dlt.common.destination import DestinationCapabilitiesContext -from dlt.common.data_types import TDataType from dlt.common.schema import TColumnSchema, TColumnHint, Schema from dlt.common.schema.typing import TTableSchema, TColumnType, TTableFormat -from dlt.common.utils import uniq_id from dlt.destinations.sql_jobs import SqlStagingCopyJob, SqlMergeJob, SqlJobParams from dlt.destinations.insert_job_client import InsertValuesJobClient -from dlt.destinations.mssql import capabilities +from dlt.destinations.mssql import capabilities as make_capabilities from dlt.destinations.mssql.sql_client import PyOdbcMsSqlClient from dlt.destinations.mssql.configuration import MsSqlClientConfiguration from dlt.destinations.sql_client import SqlClientBase @@ -120,7 +117,7 @@ def _new_temp_table_name(cls, name_prefix: str) -> str: class MsSqlClient(InsertValuesJobClient): - capabilities: ClassVar[DestinationCapabilitiesContext] = capabilities() + capabilities: ClassVar[DestinationCapabilitiesContext] = make_capabilities() def __init__(self, schema: Schema, config: MsSqlClientConfiguration) -> None: sql_client = PyOdbcMsSqlClient( diff --git a/dlt/destinations/path_utils.py b/dlt/destinations/path_utils.py index a6cf634452..16d820aebc 100644 --- a/dlt/destinations/path_utils.py +++ b/dlt/destinations/path_utils.py @@ -1,5 +1,5 @@ # this can probably go some other place, but it is shared by destinations, so for now it is here -from typing import List, Sequence, Tuple +from typing import List, Sequence import pendulum import re diff --git a/dlt/destinations/postgres/postgres.py b/dlt/destinations/postgres/postgres.py index 2812d1d4c4..d461bdf7be 100644 --- a/dlt/destinations/postgres/postgres.py +++ b/dlt/destinations/postgres/postgres.py @@ -1,9 +1,7 @@ from typing import ClassVar, Dict, Optional, Sequence, List, Any -from dlt.common.wei import EVM_DECIMAL_PRECISION from dlt.common.destination.reference import NewLoadJob from dlt.common.destination import DestinationCapabilitiesContext -from dlt.common.data_types import TDataType from dlt.common.schema import TColumnSchema, TColumnHint, Schema from dlt.common.schema.typing import TTableSchema, TColumnType, TTableFormat @@ -11,7 +9,7 @@ from dlt.destinations.insert_job_client import InsertValuesJobClient -from dlt.destinations.postgres import capabilities +from dlt.destinations.postgres import capabilities # noqa: F401 from dlt.destinations.postgres.sql_client import Psycopg2SqlClient from dlt.destinations.postgres.configuration import PostgresClientConfiguration from dlt.destinations.sql_client import SqlClientBase diff --git a/dlt/destinations/snowflake/snowflake.py b/dlt/destinations/snowflake/snowflake.py index f433ec7e7d..96fa4ab3f5 100644 --- a/dlt/destinations/snowflake/snowflake.py +++ b/dlt/destinations/snowflake/snowflake.py @@ -4,7 +4,6 @@ from dlt.common.destination import DestinationCapabilitiesContext from dlt.common.destination.reference import FollowupJob, NewLoadJob, TLoadJobState, LoadJob, CredentialsConfiguration, SupportsStagingDestination from dlt.common.configuration.specs import AwsCredentialsWithoutDefaults, AzureCredentialsWithoutDefaults -from dlt.common.data_types import TDataType from dlt.common.storages.file_storage import FileStorage from dlt.common.schema import TColumnSchema, Schema, TTableSchemaColumns from dlt.common.schema.typing import TTableSchema, TColumnType, TTableFormat @@ -14,7 +13,7 @@ from dlt.destinations.job_impl import EmptyLoadJob from dlt.destinations.exceptions import LoadJobTerminalException -from dlt.destinations.snowflake import capabilities +from dlt.destinations.snowflake import capabilities # noqa: F401 from dlt.destinations.snowflake.configuration import SnowflakeClientConfiguration from dlt.destinations.snowflake.sql_client import SnowflakeSqlClient from dlt.destinations.sql_jobs import SqlStagingCopyJob, SqlJobParams diff --git a/dlt/destinations/sql_client.py b/dlt/destinations/sql_client.py index 68af420085..a2181e6366 100644 --- a/dlt/destinations/sql_client.py +++ b/dlt/destinations/sql_client.py @@ -3,7 +3,7 @@ from functools import wraps import inspect from types import TracebackType -from typing import Any, ClassVar, ContextManager, Generic, Iterator, Optional, Sequence, Tuple, Type, AnyStr, List +from typing import Any, ClassVar, ContextManager, Generic, Iterator, Optional, Sequence, Type, AnyStr, List from dlt.common.typing import TFun from dlt.common.destination import DestinationCapabilitiesContext diff --git a/dlt/destinations/sql_jobs.py b/dlt/destinations/sql_jobs.py index 4e8393ed74..5501deaafb 100644 --- a/dlt/destinations/sql_jobs.py +++ b/dlt/destinations/sql_jobs.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, List, Sequence, Tuple, cast, TypedDict, Optional +from typing import Any, List, Sequence, Tuple, cast, TypedDict, Optional import yaml from dlt.common.runtime.logger import pretty_format_exception @@ -229,4 +229,4 @@ def gen_merge_sql(cls, table_chain: Sequence[TTableSchema], sql_client: SqlClien sql.append(insert_sql) # -- DELETE FROM {staging_table_name} WHERE 1=1; - return sql \ No newline at end of file + return sql diff --git a/dlt/extract/extract.py b/dlt/extract/extract.py index 3f71943579..f5f270089d 100644 --- a/dlt/extract/extract.py +++ b/dlt/extract/extract.py @@ -1,7 +1,6 @@ import contextlib import os -from typing import ClassVar, List, Set, Dict, Type, Any, Sequence, Optional -from collections import defaultdict +from typing import ClassVar, List, Set, Dict, Any, Optional from dlt.common.configuration.container import Container from dlt.common.configuration.resolve import inject_section @@ -15,7 +14,7 @@ from dlt.common.utils import uniq_id from dlt.common.typing import TDataItems, TDataItem from dlt.common.schema import Schema, utils, TSchemaUpdate -from dlt.common.schema.typing import TColumnSchema, TTableSchemaColumns +from dlt.common.schema.typing import TTableSchemaColumns from dlt.common.storages import NormalizeStorageConfiguration, NormalizeStorage, DataItemStorage, FileStorage from dlt.common.configuration.specs import known_sections diff --git a/dlt/extract/incremental/transform.py b/dlt/extract/incremental/transform.py index af45736da4..b2074c07ce 100644 --- a/dlt/extract/incremental/transform.py +++ b/dlt/extract/incremental/transform.py @@ -15,8 +15,8 @@ from dlt.common.utils import digest128 from dlt.common.json import json from dlt.common import pendulum -from dlt.common.typing import TDataItem, TDataItems -from dlt.common.jsonpath import TJsonPath, find_values, JSONPathFields, compile_path +from dlt.common.typing import TDataItem +from dlt.common.jsonpath import find_values, JSONPathFields, compile_path from dlt.extract.incremental.exceptions import IncrementalCursorPathMissing, IncrementalPrimaryKeyMissing from dlt.extract.incremental.typing import IncrementalColumnState, TCursorValue, LastValueFunc from dlt.extract.utils import resolve_column_value diff --git a/dlt/extract/schema.py b/dlt/extract/schema.py index c1dfd1f7f5..79df365449 100644 --- a/dlt/extract/schema.py +++ b/dlt/extract/schema.py @@ -1,16 +1,15 @@ from copy import copy, deepcopy -from collections.abc import Mapping as C_Mapping -from typing import List, TypedDict, cast, Any +from typing import TypedDict, cast, Any from dlt.common.schema.utils import DEFAULT_WRITE_DISPOSITION, merge_columns, new_column, new_table -from dlt.common.schema.typing import TColumnNames, TColumnProp, TColumnSchema, TPartialTableSchema, TTableSchemaColumns, TWriteDisposition, TAnySchemaColumns, TTableFormat +from dlt.common.schema.typing import TColumnNames, TColumnProp, TPartialTableSchema, TTableSchemaColumns, TWriteDisposition, TAnySchemaColumns, TTableFormat from dlt.common.typing import TDataItem from dlt.common.utils import update_dict_nested from dlt.common.validation import validate_dict_ignoring_xkeys from dlt.extract.incremental import Incremental from dlt.extract.typing import TFunHintTemplate, TTableHintTemplate, ValidateItem -from dlt.extract.exceptions import DataItemRequiredForDynamicTableHints, InconsistentTableTemplate, TableNameMissing +from dlt.extract.exceptions import DataItemRequiredForDynamicTableHints, InconsistentTableTemplate from dlt.extract.utils import ensure_table_schema_columns, ensure_table_schema_columns_hint from dlt.extract.validation import get_column_validator diff --git a/dlt/normalize/items_normalizers.py b/dlt/normalize/items_normalizers.py index b9bd5468dc..59f971300c 100644 --- a/dlt/normalize/items_normalizers.py +++ b/dlt/normalize/items_normalizers.py @@ -1,13 +1,12 @@ import os -from typing import List, Dict, Tuple, Protocol, Any -from pathlib import Path +from typing import List, Dict, Tuple, Any from abc import abstractmethod from dlt.common import json, logger from dlt.common.json import custom_pua_decode from dlt.common.runtime import signals from dlt.common.schema.typing import TTableSchemaColumns -from dlt.common.storages import NormalizeStorage, LoadStorage, NormalizeStorageConfiguration, FileStorage +from dlt.common.storages import NormalizeStorage, LoadStorage, FileStorage from dlt.common.typing import TDataItem from dlt.common.schema import TSchemaUpdate, Schema from dlt.common.utils import TRowCount, merge_row_count, increase_row_count diff --git a/dlt/normalize/normalize.py b/dlt/normalize/normalize.py index d60ea05965..85e1fc36f9 100644 --- a/dlt/normalize/normalize.py +++ b/dlt/normalize/normalize.py @@ -2,7 +2,7 @@ from typing import Callable, List, Dict, Sequence, Tuple, Set, Optional from concurrent.futures import Future, Executor -from dlt.common import pendulum, json, logger, sleep +from dlt.common import pendulum, logger, sleep from dlt.common.configuration import with_config, known_sections from dlt.common.configuration.accessors import config from dlt.common.configuration.container import Container @@ -14,7 +14,6 @@ from dlt.common.schema.utils import merge_schema_updates from dlt.common.storages.exceptions import SchemaNotFoundError from dlt.common.storages import NormalizeStorage, SchemaStorage, LoadStorage, LoadStorageConfiguration, NormalizeStorageConfiguration -from dlt.common.typing import TDataItem from dlt.common.schema import TSchemaUpdate, Schema from dlt.common.schema.exceptions import CannotCoerceColumnException from dlt.common.pipeline import NormalizeInfo diff --git a/dlt/pipeline/pipeline.py b/dlt/pipeline/pipeline.py index e6e27afec7..a685c45512 100644 --- a/dlt/pipeline/pipeline.py +++ b/dlt/pipeline/pipeline.py @@ -5,7 +5,6 @@ from functools import wraps from collections.abc import Sequence as C_Sequence from typing import Any, Callable, ClassVar, List, Iterator, Optional, Sequence, Tuple, cast, get_type_hints, ContextManager -from concurrent.futures import Executor from dlt import version from dlt.common import json, logger, pendulum @@ -19,7 +18,7 @@ MissingDependencyException, DestinationUndefinedEntity, DestinationIncompatibleLoaderFileFormatException) from dlt.common.normalizers import explicit_normalizers, import_normalizers from dlt.common.runtime import signals, initialize_runtime -from dlt.common.schema.typing import TColumnNames, TColumnSchema, TSchemaTables, TWriteDisposition, TAnySchemaColumns +from dlt.common.schema.typing import TColumnNames, TSchemaTables, TWriteDisposition, TAnySchemaColumns from dlt.common.storages.load_storage import LoadJobInfo, LoadPackageInfo from dlt.common.typing import TFun, TSecretValue, is_optional_type from dlt.common.runners import pool_runner as runner diff --git a/dlt/sources/credentials.py b/dlt/sources/credentials.py index 70e88beb49..5815324d56 100644 --- a/dlt/sources/credentials.py +++ b/dlt/sources/credentials.py @@ -2,4 +2,18 @@ from dlt.common.configuration.specs import ConnectionStringCredentials from dlt.common.configuration.specs import OAuth2Credentials from dlt.common.configuration.specs import CredentialsConfiguration, configspec -from dlt.common.storages.configuration import FileSystemCredentials, FilesystemConfiguration \ No newline at end of file +from dlt.common.storages.configuration import FileSystemCredentials, FilesystemConfiguration + + +__all__ = [ + "GcpServiceAccountCredentials", + "GcpOAuthCredentials", + "GcpCredentials", + "ConnectionStringCredentials", + "OAuth2Credentials", + "CredentialsConfiguration", + "configspec", + "FileSystemCredentials", + "FilesystemConfiguration", +] + diff --git a/dlt/sources/filesystem.py b/dlt/sources/filesystem.py index af874788c1..bb18a15f20 100644 --- a/dlt/sources/filesystem.py +++ b/dlt/sources/filesystem.py @@ -1 +1,3 @@ -from dlt.common.storages.fsspec_filesystem import FileItem, FileItemDict, fsspec_filesystem, glob_files \ No newline at end of file +from dlt.common.storages.fsspec_filesystem import FileItem, FileItemDict, fsspec_filesystem, glob_files + +__all__ = ["FileItem", "FileItemDict", "fsspec_filesystem", "glob_files"] diff --git a/dlt/sources/helpers/requests/__init__.py b/dlt/sources/helpers/requests/__init__.py index 39d286e29d..1c7a322fb2 100644 --- a/dlt/sources/helpers/requests/__init__.py +++ b/dlt/sources/helpers/requests/__init__.py @@ -41,7 +41,7 @@ def init(config: RunConfiguration) -> None: "init", "Session", "Request", "Response", "ConnectionError", "ConnectTimeout", "FileModeWarning", "HTTPError", "ReadTimeout", - "RequestException", "Timeout", "TooManyRedirects", "URLRequired", "ChunkedEncodingError", "RetryError" + "RequestException", "Timeout", "TooManyRedirects", "URLRequired", "ChunkedEncodingError", "RetryError", "Client", "RetryError" ] diff --git a/dlt/sources/helpers/requests/session.py b/dlt/sources/helpers/requests/session.py index b12d8da73f..7f1710c007 100644 --- a/dlt/sources/helpers/requests/session.py +++ b/dlt/sources/helpers/requests/session.py @@ -1,6 +1,5 @@ from requests import Session as BaseSession -from tenacity import Retrying, retry_if_exception_type -from typing import Optional, TYPE_CHECKING, Sequence, Union, Tuple, Type, TypeVar +from typing import Optional, TYPE_CHECKING, Union, Tuple, TypeVar from dlt.sources.helpers.requests.typing import TRequestTimeout from dlt.common.typing import TimedeltaSeconds diff --git a/dlt/sources/helpers/requests/typing.py b/dlt/sources/helpers/requests/typing.py index 8595e65b95..9418230bef 100644 --- a/dlt/sources/helpers/requests/typing.py +++ b/dlt/sources/helpers/requests/typing.py @@ -1,4 +1,4 @@ -from typing import Tuple, Union, Optional +from typing import Tuple, Union from dlt.common.typing import TimedeltaSeconds