From dee8e0835db38bd586228667bdd7216efbe7171b Mon Sep 17 00:00:00 2001 From: Marcin Rudolf Date: Sat, 13 Apr 2024 14:31:43 +0200 Subject: [PATCH] corrects json and pendulum imports --- dlt/cli/_dlt.py | 2 +- dlt/cli/config_toml_writer.py | 2 +- dlt/cli/pipeline_command.py | 2 +- dlt/cli/utils.py | 3 -- .../configuration/providers/google_secrets.py | 3 +- dlt/common/configuration/providers/toml.py | 2 +- .../configuration/specs/azure_credentials.py | 3 +- .../configuration/specs/gcp_credentials.py | 3 +- dlt/common/configuration/utils.py | 4 +-- dlt/common/data_types/type_helpers.py | 6 ++-- dlt/common/data_writers/writers.py | 2 +- dlt/common/json/__init__.py | 8 ++--- dlt/common/libs/pyarrow.py | 2 +- dlt/common/runners/pool_runner.py | 3 +- dlt/common/runtime/prometheus.py | 1 - dlt/common/runtime/segment.py | 2 -- dlt/common/runtime/slack.py | 3 +- dlt/common/schema/detections.py | 3 +- dlt/common/schema/schema.py | 1 - dlt/common/schema/utils.py | 2 +- dlt/common/storages/__init__.py | 2 ++ dlt/common/storages/fsspec_filesystem.py | 2 +- dlt/common/storages/fsspecs/google_drive.py | 2 +- dlt/common/storages/schema_storage.py | 3 +- dlt/common/time.py | 10 ++++--- dlt/destinations/decorators.py | 7 +++-- dlt/destinations/impl/bigquery/bigquery.py | 3 +- .../impl/databricks/sql_client.py | 2 -- dlt/destinations/impl/destination/factory.py | 5 ++-- dlt/destinations/impl/dummy/dummy.py | 2 +- dlt/destinations/impl/qdrant/qdrant_client.py | 4 ++- dlt/destinations/job_impl.py | 2 +- dlt/destinations/path_utils.py | 3 +- dlt/extract/incremental/__init__.py | 3 +- dlt/extract/incremental/transform.py | 2 +- dlt/helpers/airflow_helper.py | 2 +- dlt/helpers/dbt/dbt_utils.py | 3 +- dlt/helpers/streamlit_app/blocks/load_info.py | 2 +- .../streamlit_app/blocks/resource_state.py | 6 ++-- dlt/load/load.py | 3 +- dlt/normalize/items_normalizers.py | 3 +- dlt/pipeline/pipeline.py | 7 +++-- dlt/pipeline/platform.py | 7 +++-- dlt/pipeline/state_sync.py | 30 +++---------------- dlt/pipeline/trace.py | 2 +- dlt/pipeline/track.py | 3 +- dlt/sources/helpers/rest_client/auth.py | 8 ++--- dlt/sources/helpers/rest_client/client.py | 4 +-- dlt/version.py | 2 +- .../chess_production/chess_production.py | 1 - 50 files changed, 88 insertions(+), 104 deletions(-) diff --git a/dlt/cli/_dlt.py b/dlt/cli/_dlt.py index 2332c0286c..af4f2f66e9 100644 --- a/dlt/cli/_dlt.py +++ b/dlt/cli/_dlt.py @@ -5,7 +5,7 @@ import click from dlt.version import __version__ -from dlt.common import json +from dlt.common.json import json from dlt.common.schema import Schema from dlt.common.typing import DictStrAny from dlt.common.runners import Venv diff --git a/dlt/cli/config_toml_writer.py b/dlt/cli/config_toml_writer.py index 8cf831d725..7ff7f735eb 100644 --- a/dlt/cli/config_toml_writer.py +++ b/dlt/cli/config_toml_writer.py @@ -4,7 +4,7 @@ from tomlkit.container import Container as TOMLContainer from collections.abc import Sequence as C_Sequence -from dlt.common import pendulum +from dlt.common.pendulum import pendulum from dlt.common.configuration.specs import ( BaseConfiguration, is_base_configuration_inner_hint, diff --git a/dlt/cli/pipeline_command.py b/dlt/cli/pipeline_command.py index 0eb73ad7a8..d66d884ff2 100644 --- a/dlt/cli/pipeline_command.py +++ b/dlt/cli/pipeline_command.py @@ -3,7 +3,7 @@ import dlt from dlt.cli.exceptions import CliCommandException -from dlt.common import json +from dlt.common.json import json from dlt.common.pipeline import resource_state, get_dlt_pipelines_dir, TSourceState from dlt.common.destination.reference import TDestinationReferenceArg from dlt.common.runners import Venv diff --git a/dlt/cli/utils.py b/dlt/cli/utils.py index 5ea4471d7e..8699116628 100644 --- a/dlt/cli/utils.py +++ b/dlt/cli/utils.py @@ -1,11 +1,8 @@ import ast import os -import tempfile from typing import Callable -from dlt.common import git from dlt.common.reflection.utils import set_ast_parents -from dlt.common.storages import FileStorage from dlt.common.typing import TFun from dlt.common.configuration import resolve_configuration from dlt.common.configuration.specs import RunConfiguration diff --git a/dlt/common/configuration/providers/google_secrets.py b/dlt/common/configuration/providers/google_secrets.py index 98cbbc4553..43a284c67c 100644 --- a/dlt/common/configuration/providers/google_secrets.py +++ b/dlt/common/configuration/providers/google_secrets.py @@ -1,9 +1,8 @@ import base64 import string import re -from typing import Tuple -from dlt.common import json +from dlt.common.json import json from dlt.common.configuration.specs import GcpServiceAccountCredentials from dlt.common.exceptions import MissingDependencyException from .toml import VaultTomlProvider diff --git a/dlt/common/configuration/providers/toml.py b/dlt/common/configuration/providers/toml.py index 7c856e8c27..10e0b470de 100644 --- a/dlt/common/configuration/providers/toml.py +++ b/dlt/common/configuration/providers/toml.py @@ -6,7 +6,7 @@ from tomlkit.container import Container as TOMLContainer from typing import Any, Dict, Optional, Tuple, Type, Union -from dlt.common import pendulum +from dlt.common.pendulum import pendulum from dlt.common.configuration.paths import get_dlt_settings_dir, get_dlt_data_dir from dlt.common.configuration.utils import auto_cast from dlt.common.configuration.specs import known_sections diff --git a/dlt/common/configuration/specs/azure_credentials.py b/dlt/common/configuration/specs/azure_credentials.py index f7cac78dca..52d33ec0d3 100644 --- a/dlt/common/configuration/specs/azure_credentials.py +++ b/dlt/common/configuration/specs/azure_credentials.py @@ -1,7 +1,6 @@ from typing import Optional, Dict, Any -from dlt.common import pendulum -from dlt.common.exceptions import MissingDependencyException +from dlt.common.pendulum import pendulum from dlt.common.typing import TSecretStrValue from dlt.common.configuration.specs import ( CredentialsConfiguration, diff --git a/dlt/common/configuration/specs/gcp_credentials.py b/dlt/common/configuration/specs/gcp_credentials.py index 4d81a493a3..9927b81ebf 100644 --- a/dlt/common/configuration/specs/gcp_credentials.py +++ b/dlt/common/configuration/specs/gcp_credentials.py @@ -2,7 +2,8 @@ import sys from typing import Any, ClassVar, Final, List, Tuple, Union, Dict -from dlt.common import json, pendulum +from dlt.common.json import json +from dlt.common.pendulum import pendulum from dlt.common.configuration.specs.api_credentials import OAuth2Credentials from dlt.common.configuration.specs.exceptions import ( InvalidGoogleNativeCredentialsType, diff --git a/dlt/common/configuration/utils.py b/dlt/common/configuration/utils.py index 5a7330447b..51e6b5615a 100644 --- a/dlt/common/configuration/utils.py +++ b/dlt/common/configuration/utils.py @@ -5,7 +5,7 @@ from typing import Any, Dict, Mapping, NamedTuple, Optional, Tuple, Type, Sequence from collections.abc import Mapping as C_Mapping -from dlt.common import json +from dlt.common.json import json from dlt.common.typing import AnyType, TAny from dlt.common.data_types import coerce_value, py_type_to_sc_type from dlt.common.configuration.providers import EnvironProvider @@ -122,8 +122,6 @@ def log_traces( default_value: Any, traces: Sequence[LookupTrace], ) -> None: - from dlt.common import logger - # if logger.is_logging() and logger.log_level() == "DEBUG" and config: # logger.debug(f"Field {key} with type {hint} in {type(config).__name__} {'NOT RESOLVED' if value is None else 'RESOLVED'}") # print(f"Field {key} with type {hint} in {type(config).__name__} {'NOT RESOLVED' if value is None else 'RESOLVED'}") diff --git a/dlt/common/data_types/type_helpers.py b/dlt/common/data_types/type_helpers.py index 61a0aa1dbf..d8ab9eb118 100644 --- a/dlt/common/data_types/type_helpers.py +++ b/dlt/common/data_types/type_helpers.py @@ -3,13 +3,13 @@ import dataclasses import datetime # noqa: I251 from collections.abc import Mapping as C_Mapping, Sequence as C_Sequence -from typing import Any, Type, Literal, Union, cast +from typing import Any, Type, Union from enum import Enum -from dlt.common import pendulum, json, Decimal, Wei from dlt.common.json import custom_pua_remove, json from dlt.common.json._simplejson import custom_encode as json_custom_encode -from dlt.common.arithmetics import InvalidOperation +from dlt.common.wei import Wei +from dlt.common.arithmetics import InvalidOperation, Decimal from dlt.common.data_types.typing import TDataType from dlt.common.time import ( ensure_pendulum_datetime, diff --git a/dlt/common/data_writers/writers.py b/dlt/common/data_writers/writers.py index b952b39ed2..3d446b9478 100644 --- a/dlt/common/data_writers/writers.py +++ b/dlt/common/data_writers/writers.py @@ -17,7 +17,7 @@ TypeVar, ) -from dlt.common import json +from dlt.common.json import json from dlt.common.configuration import configspec, known_sections, with_config from dlt.common.configuration.specs import BaseConfiguration from dlt.common.data_writers.exceptions import DataWriterNotFound, InvalidDataItem diff --git a/dlt/common/json/__init__.py b/dlt/common/json/__init__.py index 371c74e54a..cf68e5d3d4 100644 --- a/dlt/common/json/__init__.py +++ b/dlt/common/json/__init__.py @@ -12,7 +12,7 @@ except ImportError: PydanticBaseModel = None # type: ignore[misc] -from dlt.common import pendulum +from dlt.common.pendulum import pendulum from dlt.common.arithmetics import Decimal from dlt.common.wei import Wei from dlt.common.utils import map_nested_in_place @@ -99,19 +99,19 @@ def _datetime_decoder(obj: str) -> datetime: # Backwards compatibility for data encoded with previous dlt version # fromisoformat does not support Z suffix (until py3.11) obj = obj[:-1] + "+00:00" - return pendulum.DateTime.fromisoformat(obj) # type: ignore[attr-defined, no-any-return] + return pendulum.DateTime.fromisoformat(obj) # define decoder for each prefix DECODERS: List[Callable[[Any], Any]] = [ Decimal, _datetime_decoder, - pendulum.Date.fromisoformat, # type: ignore[attr-defined] + pendulum.Date.fromisoformat, UUID, HexBytes, base64.b64decode, Wei, - pendulum.Time.fromisoformat, # type: ignore[attr-defined] + pendulum.Time.fromisoformat, ] # how many decoders? PUA_CHARACTER_MAX = len(DECODERS) diff --git a/dlt/common/libs/pyarrow.py b/dlt/common/libs/pyarrow.py index 3380157600..dc21e61d63 100644 --- a/dlt/common/libs/pyarrow.py +++ b/dlt/common/libs/pyarrow.py @@ -15,7 +15,7 @@ ) from dlt import version -from dlt.common import pendulum +from dlt.common.pendulum import pendulum from dlt.common.exceptions import MissingDependencyException from dlt.common.schema.typing import DLT_NAME_PREFIX, TTableSchemaColumns diff --git a/dlt/common/runners/pool_runner.py b/dlt/common/runners/pool_runner.py index 491c74cd18..c691347529 100644 --- a/dlt/common/runners/pool_runner.py +++ b/dlt/common/runners/pool_runner.py @@ -4,13 +4,14 @@ from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor, Future from typing_extensions import ParamSpec -from dlt.common import logger, sleep +from dlt.common import logger from dlt.common.configuration.container import Container from dlt.common.runtime import init from dlt.common.runners.runnable import Runnable, TExecutor from dlt.common.runners.configuration import PoolRunnerConfiguration from dlt.common.runners.typing import TRunMetrics from dlt.common.runtime import signals +from dlt.common.runtime.signals import sleep from dlt.common.exceptions import SignalReceivedException diff --git a/dlt/common/runtime/prometheus.py b/dlt/common/runtime/prometheus.py index 07c960efe7..9bc89211be 100644 --- a/dlt/common/runtime/prometheus.py +++ b/dlt/common/runtime/prometheus.py @@ -3,7 +3,6 @@ from prometheus_client.metrics import MetricWrapperBase from dlt.common.configuration.specs import RunConfiguration -from dlt.common import logger from dlt.common.runtime.exec_info import dlt_version_info from dlt.common.typing import DictStrAny, StrAny diff --git a/dlt/common/runtime/segment.py b/dlt/common/runtime/segment.py index 70b81fb4f4..ac64591072 100644 --- a/dlt/common/runtime/segment.py +++ b/dlt/common/runtime/segment.py @@ -6,13 +6,11 @@ import atexit import base64 import requests -from concurrent.futures import ThreadPoolExecutor from typing import Literal, Optional from dlt.common.configuration.paths import get_dlt_data_dir from dlt.common import logger from dlt.common.managed_thread_pool import ManagedThreadPool - from dlt.common.configuration.specs import RunConfiguration from dlt.common.runtime.exec_info import get_execution_context, TExecutionContext from dlt.common.typing import DictStrAny, StrAny diff --git a/dlt/common/runtime/slack.py b/dlt/common/runtime/slack.py index b1e090098d..75c01aac25 100644 --- a/dlt/common/runtime/slack.py +++ b/dlt/common/runtime/slack.py @@ -2,7 +2,8 @@ def send_slack_message(incoming_hook: str, message: str, is_markdown: bool = True) -> None: - from dlt.common import json, logger + from dlt.common import logger + from dlt.common.json import json """Sends a `message` to Slack `incoming_hook`, by default formatted as markdown.""" r = requests.post( diff --git a/dlt/common/schema/detections.py b/dlt/common/schema/detections.py index 30b23706af..c9e0e05be9 100644 --- a/dlt/common/schema/detections.py +++ b/dlt/common/schema/detections.py @@ -3,7 +3,8 @@ from hexbytes import HexBytes -from dlt.common import pendulum, Wei +from dlt.common.pendulum import pendulum +from dlt.common.wei import Wei from dlt.common.data_types import TDataType from dlt.common.time import parse_iso_like_datetime diff --git a/dlt/common/schema/schema.py b/dlt/common/schema/schema.py index c738f1753e..740e578ef2 100644 --- a/dlt/common/schema/schema.py +++ b/dlt/common/schema/schema.py @@ -1,6 +1,5 @@ from copy import copy, deepcopy from typing import ClassVar, Dict, List, Mapping, Optional, Sequence, Tuple, Any, cast, Literal -from dlt.common import json from dlt.common.schema.migrations import migrate_schema from dlt.common.utils import extend_list_deduplicated diff --git a/dlt/common/schema/utils.py b/dlt/common/schema/utils.py index 8fea6ebca8..09b438129e 100644 --- a/dlt/common/schema/utils.py +++ b/dlt/common/schema/utils.py @@ -5,7 +5,7 @@ from copy import deepcopy, copy from typing import Dict, List, Sequence, Tuple, Type, Any, cast, Iterable, Optional, Union -from dlt.common import json +from dlt.common.json import json from dlt.common.data_types import TDataType from dlt.common.exceptions import DictValidationException from dlt.common.normalizers.naming import NamingConvention diff --git a/dlt/common/storages/__init__.py b/dlt/common/storages/__init__.py index e5feeaba57..7bb3c0cf97 100644 --- a/dlt/common/storages/__init__.py +++ b/dlt/common/storages/__init__.py @@ -9,6 +9,7 @@ LoadPackageInfo, PackageStorage, TJobState, + create_load_id, ) from .data_item_storage import DataItemStorage from .load_storage import LoadStorage @@ -40,6 +41,7 @@ "LoadPackageInfo", "PackageStorage", "TJobState", + "create_load_id", "fsspec_from_config", "fsspec_filesystem", ] diff --git a/dlt/common/storages/fsspec_filesystem.py b/dlt/common/storages/fsspec_filesystem.py index b1cbc11bf9..3a2b483970 100644 --- a/dlt/common/storages/fsspec_filesystem.py +++ b/dlt/common/storages/fsspec_filesystem.py @@ -24,7 +24,7 @@ from fsspec.core import url_to_fs from dlt import version -from dlt.common import pendulum +from dlt.common.pendulum import pendulum from dlt.common.configuration.specs import ( GcpCredentials, AwsCredentials, diff --git a/dlt/common/storages/fsspecs/google_drive.py b/dlt/common/storages/fsspecs/google_drive.py index 3bc4b1d7d7..1be862668c 100644 --- a/dlt/common/storages/fsspecs/google_drive.py +++ b/dlt/common/storages/fsspecs/google_drive.py @@ -1,7 +1,7 @@ import posixpath from typing import Any, Dict, List, Literal, Optional, Tuple -from dlt.common import json +from dlt.common.json import json from dlt.common.configuration.specs import GcpCredentials, GcpOAuthCredentials from dlt.common.exceptions import MissingDependencyException diff --git a/dlt/common/storages/schema_storage.py b/dlt/common/storages/schema_storage.py index 23b695b839..1afed18929 100644 --- a/dlt/common/storages/schema_storage.py +++ b/dlt/common/storages/schema_storage.py @@ -1,7 +1,8 @@ import yaml from typing import Iterator, List, Mapping, Tuple, cast -from dlt.common import json, logger +from dlt.common import logger +from dlt.common.json import json from dlt.common.configuration import with_config from dlt.common.configuration.accessors import config from dlt.common.schema.utils import to_pretty_json, to_pretty_yaml diff --git a/dlt/common/time.py b/dlt/common/time.py index d3c8f9746c..161205deb8 100644 --- a/dlt/common/time.py +++ b/dlt/common/time.py @@ -208,10 +208,12 @@ def to_seconds(td: Optional[TimedeltaSeconds]) -> Optional[float]: return td -T = TypeVar("T", bound=Union[pendulum.DateTime, pendulum.Time]) +TTimeWithPrecision = TypeVar("TTimeWithPrecision", bound=Union[pendulum.DateTime, pendulum.Time]) -def reduce_pendulum_datetime_precision(value: T, microsecond_precision: int) -> T: - if microsecond_precision >= 6: +def reduce_pendulum_datetime_precision( + value: TTimeWithPrecision, precision: int +) -> TTimeWithPrecision: + if precision >= 6: return value - return value.replace(microsecond=value.microsecond // 10 ** (6 - microsecond_precision) * 10 ** (6 - microsecond_precision)) # type: ignore + return value.replace(microsecond=value.microsecond // 10 ** (6 - precision) * 10 ** (6 - precision)) # type: ignore diff --git a/dlt/destinations/decorators.py b/dlt/destinations/decorators.py index a920d336a2..8e0b5d5ee8 100644 --- a/dlt/destinations/decorators.py +++ b/dlt/destinations/decorators.py @@ -7,14 +7,15 @@ from functools import wraps from dlt.common import logger +from dlt.common.destination import TLoaderFileFormat +from dlt.common.typing import TDataItems +from dlt.common.schema import TTableSchema + from dlt.destinations.impl.destination.factory import destination as _destination from dlt.destinations.impl.destination.configuration import ( TDestinationCallableParams, CustomDestinationClientConfiguration, ) -from dlt.common.destination import TLoaderFileFormat -from dlt.common.typing import TDataItems -from dlt.common.schema import TTableSchema def destination( diff --git a/dlt/destinations/impl/bigquery/bigquery.py b/dlt/destinations/impl/bigquery/bigquery.py index 86448bd011..0ac042a056 100644 --- a/dlt/destinations/impl/bigquery/bigquery.py +++ b/dlt/destinations/impl/bigquery/bigquery.py @@ -9,7 +9,8 @@ from google.api_core import retry from google.cloud.bigquery.retry import _RETRYABLE_REASONS -from dlt.common import json, logger +from dlt.common import logger +from dlt.common.json import json from dlt.common.destination import DestinationCapabilitiesContext from dlt.common.destination.reference import ( FollowupJob, diff --git a/dlt/destinations/impl/databricks/sql_client.py b/dlt/destinations/impl/databricks/sql_client.py index 68ea863cc4..7e2487593d 100644 --- a/dlt/destinations/impl/databricks/sql_client.py +++ b/dlt/destinations/impl/databricks/sql_client.py @@ -8,8 +8,6 @@ ) from databricks.sql.exc import Error as DatabricksSqlError -from dlt.common import pendulum -from dlt.common import logger from dlt.common.destination import DestinationCapabilitiesContext from dlt.destinations.exceptions import ( DatabaseTerminalException, diff --git a/dlt/destinations/impl/destination/factory.py b/dlt/destinations/impl/destination/factory.py index 8395c66ac8..3ae6f2e876 100644 --- a/dlt/destinations/impl/destination/factory.py +++ b/dlt/destinations/impl/destination/factory.py @@ -1,14 +1,13 @@ import typing as t import inspect from importlib import import_module - from types import ModuleType -from dlt.common.typing import AnyFun +from dlt.common import logger +from dlt.common.typing import AnyFun from dlt.common.destination import Destination, DestinationCapabilitiesContext, TLoaderFileFormat from dlt.common.configuration import known_sections, with_config, get_fun_spec from dlt.common.configuration.exceptions import ConfigurationValueError -from dlt.common import logger from dlt.common.utils import get_callable_name, is_inner_callable from dlt.destinations.exceptions import DestinationTransientException diff --git a/dlt/destinations/impl/dummy/dummy.py b/dlt/destinations/impl/dummy/dummy.py index 0d91220d88..bafac210cc 100644 --- a/dlt/destinations/impl/dummy/dummy.py +++ b/dlt/destinations/impl/dummy/dummy.py @@ -14,7 +14,7 @@ List, ) -from dlt.common import pendulum +from dlt.common.pendulum import pendulum from dlt.common.schema import Schema, TTableSchema, TSchemaTables from dlt.common.storages import FileStorage from dlt.common.destination import DestinationCapabilitiesContext diff --git a/dlt/destinations/impl/qdrant/qdrant_client.py b/dlt/destinations/impl/qdrant/qdrant_client.py index febfe38ec9..5a5e5f8cfd 100644 --- a/dlt/destinations/impl/qdrant/qdrant_client.py +++ b/dlt/destinations/impl/qdrant/qdrant_client.py @@ -1,7 +1,9 @@ from types import TracebackType from typing import ClassVar, Optional, Sequence, List, Dict, Type, Iterable, Any, IO -from dlt.common import json, pendulum, logger +from dlt.common import logger +from dlt.common.json import json +from dlt.common.pendulum import pendulum from dlt.common.schema import Schema, TTableSchema, TSchemaTables from dlt.common.schema.utils import get_columns_names_with_prop from dlt.common.destination import DestinationCapabilitiesContext diff --git a/dlt/destinations/job_impl.py b/dlt/destinations/job_impl.py index 8e017fc791..218f73cc59 100644 --- a/dlt/destinations/job_impl.py +++ b/dlt/destinations/job_impl.py @@ -3,7 +3,7 @@ import tempfile # noqa: 251 from typing import Dict, Iterable, List -from dlt.common import json +from dlt.common.json import json from dlt.common.destination.reference import NewLoadJob, FollowupJob, TLoadJobState, LoadJob from dlt.common.schema import Schema, TTableSchema from dlt.common.storages import FileStorage diff --git a/dlt/destinations/path_utils.py b/dlt/destinations/path_utils.py index 047cb274e0..5b2ba9d183 100644 --- a/dlt/destinations/path_utils.py +++ b/dlt/destinations/path_utils.py @@ -1,9 +1,10 @@ # this can probably go some other place, but it is shared by destinations, so for now it is here from typing import List, Sequence, Tuple -import pendulum import re +from dlt.common.pendulum import pendulum + from dlt.destinations.exceptions import InvalidFilesystemLayout, CantExtractTablePrefix # TODO: ensure layout only has supported placeholders diff --git a/dlt/extract/incremental/__init__.py b/dlt/extract/incremental/__init__.py index e74e87d094..ef7523b207 100644 --- a/dlt/extract/incremental/__init__.py +++ b/dlt/extract/incremental/__init__.py @@ -8,8 +8,9 @@ import dlt +from dlt.common import logger from dlt.common.exceptions import MissingDependencyException -from dlt.common import pendulum, logger +from dlt.common.pendulum import pendulum from dlt.common.jsonpath import compile_path from dlt.common.typing import ( TDataItem, diff --git a/dlt/extract/incremental/transform.py b/dlt/extract/incremental/transform.py index 29b20de7b8..d117b4f1d8 100644 --- a/dlt/extract/incremental/transform.py +++ b/dlt/extract/incremental/transform.py @@ -4,7 +4,7 @@ from dlt.common.exceptions import MissingDependencyException from dlt.common.utils import digest128 from dlt.common.json import json -from dlt.common import pendulum +from dlt.common.pendulum import pendulum from dlt.common.typing import TDataItem from dlt.common.jsonpath import find_values, JSONPathFields, compile_path from dlt.extract.incremental.exceptions import ( diff --git a/dlt/helpers/airflow_helper.py b/dlt/helpers/airflow_helper.py index 5e625c0b48..89fe06349b 100644 --- a/dlt/helpers/airflow_helper.py +++ b/dlt/helpers/airflow_helper.py @@ -24,8 +24,8 @@ import dlt -from dlt.common import pendulum from dlt.common import logger +from dlt.common.pendulum import pendulum from dlt.common.runtime.telemetry import with_telemetry from dlt.common.destination import TLoaderFileFormat diff --git a/dlt/helpers/dbt/dbt_utils.py b/dlt/helpers/dbt/dbt_utils.py index b4097e4434..bf14504eaa 100644 --- a/dlt/helpers/dbt/dbt_utils.py +++ b/dlt/helpers/dbt/dbt_utils.py @@ -3,7 +3,8 @@ from typing import Any, Sequence, Optional, Union import warnings -from dlt.common import json, logger +from dlt.common import logger +from dlt.common.json import json from dlt.common.exceptions import MissingDependencyException from dlt.common.typing import StrAny diff --git a/dlt/helpers/streamlit_app/blocks/load_info.py b/dlt/helpers/streamlit_app/blocks/load_info.py index 134b5ad5a4..9482cb5afa 100644 --- a/dlt/helpers/streamlit_app/blocks/load_info.py +++ b/dlt/helpers/streamlit_app/blocks/load_info.py @@ -2,7 +2,7 @@ import humanize import streamlit as st -from dlt.common import pendulum +from dlt.common.pendulum import pendulum from dlt.helpers.streamlit_app.utils import query_data_live from dlt.helpers.streamlit_app.widgets import stat diff --git a/dlt/helpers/streamlit_app/blocks/resource_state.py b/dlt/helpers/streamlit_app/blocks/resource_state.py index 86b8effc98..dabbea4d46 100644 --- a/dlt/helpers/streamlit_app/blocks/resource_state.py +++ b/dlt/helpers/streamlit_app/blocks/resource_state.py @@ -1,10 +1,10 @@ from typing import Union - -import dlt -import pendulum import streamlit as st import yaml +import dlt +from dlt.common.pendulum import pendulum + def date_to_iso( dumper: yaml.SafeDumper, data: Union[pendulum.Date, pendulum.DateTime] diff --git a/dlt/load/load.py b/dlt/load/load.py index b1f786274e..c5790d467b 100644 --- a/dlt/load/load.py +++ b/dlt/load/load.py @@ -5,7 +5,8 @@ from concurrent.futures import Executor import os -from dlt.common import sleep, logger +from dlt.common import logger +from dlt.common.runtime.signals import sleep from dlt.common.configuration import with_config, known_sections from dlt.common.configuration.resolve import inject_section from dlt.common.configuration.accessors import config diff --git a/dlt/normalize/items_normalizers.py b/dlt/normalize/items_normalizers.py index 1e4e55effd..828ec4a96d 100644 --- a/dlt/normalize/items_normalizers.py +++ b/dlt/normalize/items_normalizers.py @@ -1,7 +1,8 @@ from typing import List, Dict, Set, Any from abc import abstractmethod -from dlt.common import json, logger +from dlt.common import logger +from dlt.common.json import json from dlt.common.data_writers import DataWriterMetrics from dlt.common.data_writers.writers import ArrowToObjectAdapter from dlt.common.json import custom_pua_decode, may_have_pua diff --git a/dlt/pipeline/pipeline.py b/dlt/pipeline/pipeline.py index 7591366018..5394785965 100644 --- a/dlt/pipeline/pipeline.py +++ b/dlt/pipeline/pipeline.py @@ -19,7 +19,9 @@ ) from dlt import version -from dlt.common import json, logger, pendulum +from dlt.common import logger +from dlt.common.json import json +from dlt.common.pendulum import pendulum from dlt.common.configuration import inject_section, known_sections from dlt.common.configuration.specs import RunConfiguration, CredentialsConfiguration from dlt.common.configuration.container import Container @@ -97,6 +99,7 @@ from dlt.common.schema import Schema from dlt.common.utils import is_interactive from dlt.common.warnings import deprecated, Dlt04DeprecationWarning +from dlt.common.versioned_state import json_encode_state, json_decode_state from dlt.extract import DltSource from dlt.extract.exceptions import SourceExhausted @@ -136,8 +139,6 @@ mark_state_extracted, migrate_pipeline_state, state_resource, - json_encode_state, - json_decode_state, default_pipeline_state, ) from dlt.pipeline.warnings import credentials_argument_deprecated diff --git a/dlt/pipeline/platform.py b/dlt/pipeline/platform.py index 0955e91b51..fe419d5146 100644 --- a/dlt/pipeline/platform.py +++ b/dlt/pipeline/platform.py @@ -1,15 +1,16 @@ """Implements SupportsTracking""" from typing import Any, cast, TypedDict, List import requests -from dlt.common.managed_thread_pool import ManagedThreadPool from urllib.parse import urljoin -from dlt.pipeline.trace import PipelineTrace, PipelineStepTrace, TPipelineStep, SupportsPipeline -from dlt.common import json from dlt.common import logger +from dlt.common.json import json from dlt.common.pipeline import LoadInfo +from dlt.common.managed_thread_pool import ManagedThreadPool from dlt.common.schema.typing import TStoredSchema +from dlt.pipeline.trace import PipelineTrace, PipelineStepTrace, TPipelineStep, SupportsPipeline + _THREAD_POOL: ManagedThreadPool = ManagedThreadPool(1) TRACE_URL_SUFFIX = "/trace" STATE_URL_SUFFIX = "/state" diff --git a/dlt/pipeline/state_sync.py b/dlt/pipeline/state_sync.py index 5366b9c46d..d38010f842 100644 --- a/dlt/pipeline/state_sync.py +++ b/dlt/pipeline/state_sync.py @@ -1,18 +1,17 @@ -import binascii from copy import copy -from typing import Tuple, cast, List -import pendulum +from typing import Tuple, cast import dlt -from dlt.common import json +from dlt.common.pendulum import pendulum from dlt.common.typing import DictStrAny from dlt.common.schema.typing import STATE_TABLE_NAME, TTableSchemaColumns from dlt.common.destination.reference import WithStateSync, Destination -from dlt.common.utils import compressed_b64decode, compressed_b64encode from dlt.common.versioned_state import ( generate_state_version_hash, bump_state_version_if_modified, default_versioned_state, + compress_state, + decompress_state, ) from dlt.common.pipeline import TPipelineState @@ -39,27 +38,6 @@ } -def json_encode_state(state: TPipelineState) -> str: - return json.typed_dumps(state) - - -def json_decode_state(state_str: str) -> DictStrAny: - return json.typed_loads(state_str) # type: ignore[no-any-return] - - -def compress_state(state: TPipelineState) -> str: - return compressed_b64encode(json.typed_dumpb(state)) - - -def decompress_state(state_str: str) -> DictStrAny: - try: - state_bytes = compressed_b64decode(state_str) - except binascii.Error: - return json.typed_loads(state_str) # type: ignore[no-any-return] - else: - return json.typed_loadb(state_bytes) # type: ignore[no-any-return] - - def generate_pipeline_state_version_hash(state: TPipelineState) -> str: return generate_state_version_hash(state, exclude_attrs=["_local"]) diff --git a/dlt/pipeline/trace.py b/dlt/pipeline/trace.py index b610d1751f..fc15654949 100644 --- a/dlt/pipeline/trace.py +++ b/dlt/pipeline/trace.py @@ -7,7 +7,7 @@ from typing import Any, List, NamedTuple, Optional, Protocol, Sequence import humanize -from dlt.common import pendulum, json +from dlt.common.pendulum import pendulum from dlt.common.configuration import is_secret_hint from dlt.common.configuration.exceptions import ContextDefaultCannotBeCreated from dlt.common.configuration.specs.config_section_context import ConfigSectionContext diff --git a/dlt/pipeline/track.py b/dlt/pipeline/track.py index 990c59050e..e6f8db36d6 100644 --- a/dlt/pipeline/track.py +++ b/dlt/pipeline/track.py @@ -3,7 +3,8 @@ from typing import Any import humanize -from dlt.common import pendulum, logger +from dlt.common import logger +from dlt.common.pendulum import pendulum from dlt.common.utils import digest128 from dlt.common.runtime.exec_info import github_info from dlt.common.runtime.segment import track as dlthub_telemetry_track diff --git a/dlt/sources/helpers/rest_client/auth.py b/dlt/sources/helpers/rest_client/auth.py index 99421e2c60..620135d410 100644 --- a/dlt/sources/helpers/rest_client/auth.py +++ b/dlt/sources/helpers/rest_client/auth.py @@ -12,19 +12,19 @@ Iterable, TYPE_CHECKING, ) -from dlt.sources.helpers import requests from requests.auth import AuthBase from requests import PreparedRequest # noqa: I251 -import pendulum - -from dlt.common.exceptions import MissingDependencyException from dlt.common import logger +from dlt.common.exceptions import MissingDependencyException from dlt.common.configuration.specs.base_configuration import configspec from dlt.common.configuration.specs import CredentialsConfiguration from dlt.common.configuration.specs.exceptions import NativeValueError +from dlt.common.pendulum import pendulum from dlt.common.typing import TSecretStrValue +from dlt.sources.helpers import requests + if TYPE_CHECKING: from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes else: diff --git a/dlt/sources/helpers/rest_client/client.py b/dlt/sources/helpers/rest_client/client.py index 027afc7cbb..e970b17a87 100644 --- a/dlt/sources/helpers/rest_client/client.py +++ b/dlt/sources/helpers/rest_client/client.py @@ -10,11 +10,11 @@ ) import copy from urllib.parse import urlparse - from requests import Session as BaseSession # noqa: I251 -from dlt.common import logger from dlt.common import jsonpath +from dlt.common import logger + from dlt.sources.helpers.requests.retry import Client from dlt.sources.helpers.requests import Response, Request diff --git a/dlt/version.py b/dlt/version.py index f8ca3cb873..aa87021bf7 100644 --- a/dlt/version.py +++ b/dlt/version.py @@ -14,7 +14,7 @@ def get_installed_requirement_string(package: str = DLT_PKG_NAME) -> str: # PEP 610 https://packaging.python.org/en/latest/specifications/direct-url/#specification direct_url = dist.read_text("direct_url.json") if direct_url is not None: - from dlt.common import json + from dlt.common.json import json # `url` contain the location of the distribution url = urlparse(json.loads(direct_url)["url"]) diff --git a/docs/examples/chess_production/chess_production.py b/docs/examples/chess_production/chess_production.py index 1ecc404507..c0f11203c8 100644 --- a/docs/examples/chess_production/chess_production.py +++ b/docs/examples/chess_production/chess_production.py @@ -35,7 +35,6 @@ from dlt.common.runtime.slack import send_slack_message - @dlt.source def chess( chess_url: str = dlt.config.value,