corrects json and pendulum imports

dlt/cli/_dlt.py

@@ -5,7 +5,7 @@
 import click
 
 from dlt.version import __version__
-from dlt.common import json
+from dlt.common.json import json
 from dlt.common.schema import Schema
 from dlt.common.typing import DictStrAny
 from dlt.common.runners import Venv

dlt/cli/config_toml_writer.py

@@ -4,7 +4,7 @@
 from tomlkit.container import Container as TOMLContainer
 from collections.abc import Sequence as C_Sequence
 
-from dlt.common import pendulum
+from dlt.common.pendulum import pendulum
 from dlt.common.configuration.specs import (
     BaseConfiguration,
     is_base_configuration_inner_hint,

dlt/cli/pipeline_command.py

@@ -3,7 +3,7 @@
 import dlt
 from dlt.cli.exceptions import CliCommandException
 
-from dlt.common import json
+from dlt.common.json import json
 from dlt.common.pipeline import resource_state, get_dlt_pipelines_dir, TSourceState
 from dlt.common.destination.reference import TDestinationReferenceArg
 from dlt.common.runners import Venv

dlt/cli/utils.py

@@ -1,11 +1,8 @@
 import ast
 import os
-import tempfile
 from typing import Callable
 
-from dlt.common import git
 from dlt.common.reflection.utils import set_ast_parents
-from dlt.common.storages import FileStorage
 from dlt.common.typing import TFun
 from dlt.common.configuration import resolve_configuration
 from dlt.common.configuration.specs import RunConfiguration

dlt/common/configuration/providers/google_secrets.py

@@ -1,9 +1,8 @@
 import base64
 import string
 import re
-from typing import Tuple
 
-from dlt.common import json
+from dlt.common.json import json
 from dlt.common.configuration.specs import GcpServiceAccountCredentials
 from dlt.common.exceptions import MissingDependencyException
 from .toml import VaultTomlProvider

dlt/common/configuration/providers/toml.py

@@ -6,7 +6,7 @@
 from tomlkit.container import Container as TOMLContainer
 from typing import Any, Dict, Optional, Tuple, Type, Union
 
-from dlt.common import pendulum
+from dlt.common.pendulum import pendulum
 from dlt.common.configuration.paths import get_dlt_settings_dir, get_dlt_data_dir
 from dlt.common.configuration.utils import auto_cast
 from dlt.common.configuration.specs import known_sections

dlt/common/configuration/specs/azure_credentials.py

@@ -1,7 +1,6 @@
 from typing import Optional, Dict, Any
 
-from dlt.common import pendulum
-from dlt.common.exceptions import MissingDependencyException
+from dlt.common.pendulum import pendulum
 from dlt.common.typing import TSecretStrValue
 from dlt.common.configuration.specs import (
     CredentialsConfiguration,

dlt/common/configuration/specs/gcp_credentials.py

@@ -2,7 +2,8 @@
 import sys
 from typing import Any, ClassVar, Final, List, Tuple, Union, Dict
 
-from dlt.common import json, pendulum
+from dlt.common.json import json
+from dlt.common.pendulum import pendulum
 from dlt.common.configuration.specs.api_credentials import OAuth2Credentials
 from dlt.common.configuration.specs.exceptions import (
     InvalidGoogleNativeCredentialsType,

dlt/common/configuration/utils.py

@@ -5,7 +5,7 @@
 from typing import Any, Dict, Mapping, NamedTuple, Optional, Tuple, Type, Sequence
 from collections.abc import Mapping as C_Mapping
 
-from dlt.common import json
+from dlt.common.json import json
 from dlt.common.typing import AnyType, TAny
 from dlt.common.data_types import coerce_value, py_type_to_sc_type
 from dlt.common.configuration.providers import EnvironProvider
@@ -122,8 +122,6 @@ def log_traces(
     default_value: Any,
     traces: Sequence[LookupTrace],
 ) -> None:
-    from dlt.common import logger
-
     # if logger.is_logging() and logger.log_level() == "DEBUG" and config:
     #     logger.debug(f"Field {key} with type {hint} in {type(config).__name__} {'NOT RESOLVED' if value is None else 'RESOLVED'}")
     # print(f"Field {key} with type {hint} in {type(config).__name__} {'NOT RESOLVED' if value is None else 'RESOLVED'}")

dlt/common/data_types/type_helpers.py

@@ -3,13 +3,13 @@
 import dataclasses
 import datetime  # noqa: I251
 from collections.abc import Mapping as C_Mapping, Sequence as C_Sequence
-from typing import Any, Type, Literal, Union, cast
+from typing import Any, Type, Union
 from enum import Enum
 
-from dlt.common import pendulum, json, Decimal, Wei
 from dlt.common.json import custom_pua_remove, json
 from dlt.common.json._simplejson import custom_encode as json_custom_encode
-from dlt.common.arithmetics import InvalidOperation
+from dlt.common.wei import Wei
+from dlt.common.arithmetics import InvalidOperation, Decimal
 from dlt.common.data_types.typing import TDataType
 from dlt.common.time import (
     ensure_pendulum_datetime,

dlt/common/data_writers/writers.py

@@ -17,7 +17,7 @@
     TypeVar,
 )
 
-from dlt.common import json
+from dlt.common.json import json
 from dlt.common.configuration import configspec, known_sections, with_config
 from dlt.common.configuration.specs import BaseConfiguration
 from dlt.common.data_writers.exceptions import DataWriterNotFound, InvalidDataItem

dlt/common/json/__init__.py

@@ -12,7 +12,7 @@
 except ImportError:
     PydanticBaseModel = None  # type: ignore[misc]
 
-from dlt.common import pendulum
+from dlt.common.pendulum import pendulum
 from dlt.common.arithmetics import Decimal
 from dlt.common.wei import Wei
 from dlt.common.utils import map_nested_in_place
@@ -99,19 +99,19 @@ def _datetime_decoder(obj: str) -> datetime:
         # Backwards compatibility for data encoded with previous dlt version
         # fromisoformat does not support Z suffix (until py3.11)
         obj = obj[:-1] + "+00:00"
-    return pendulum.DateTime.fromisoformat(obj)  # type: ignore[attr-defined, no-any-return]
+    return pendulum.DateTime.fromisoformat(obj)
 
 
 # define decoder for each prefix
 DECODERS: List[Callable[[Any], Any]] = [
     Decimal,
     _datetime_decoder,
-    pendulum.Date.fromisoformat,  # type: ignore[attr-defined]
+    pendulum.Date.fromisoformat,
     UUID,
     HexBytes,
     base64.b64decode,
     Wei,
-    pendulum.Time.fromisoformat,  # type: ignore[attr-defined]
+    pendulum.Time.fromisoformat,
 ]
 # how many decoders?
 PUA_CHARACTER_MAX = len(DECODERS)

dlt/common/libs/pyarrow.py

@@ -15,7 +15,7 @@
 )
 
 from dlt import version
-from dlt.common import pendulum
+from dlt.common.pendulum import pendulum
 from dlt.common.exceptions import MissingDependencyException
 from dlt.common.schema.typing import DLT_NAME_PREFIX, TTableSchemaColumns
 

dlt/common/runners/pool_runner.py

@@ -4,13 +4,14 @@
 from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor, Future
 from typing_extensions import ParamSpec
 
-from dlt.common import logger, sleep
+from dlt.common import logger
 from dlt.common.configuration.container import Container
 from dlt.common.runtime import init
 from dlt.common.runners.runnable import Runnable, TExecutor
 from dlt.common.runners.configuration import PoolRunnerConfiguration
 from dlt.common.runners.typing import TRunMetrics
 from dlt.common.runtime import signals
+from dlt.common.runtime.signals import sleep
 from dlt.common.exceptions import SignalReceivedException
 
 

dlt/common/runtime/prometheus.py

@@ -3,7 +3,6 @@
 from prometheus_client.metrics import MetricWrapperBase
 
 from dlt.common.configuration.specs import RunConfiguration
-from dlt.common import logger
 from dlt.common.runtime.exec_info import dlt_version_info
 from dlt.common.typing import DictStrAny, StrAny
 

dlt/common/runtime/segment.py

@@ -6,13 +6,11 @@
 import atexit
 import base64
 import requests
-from concurrent.futures import ThreadPoolExecutor
 from typing import Literal, Optional
 from dlt.common.configuration.paths import get_dlt_data_dir
 
 from dlt.common import logger
 from dlt.common.managed_thread_pool import ManagedThreadPool
-
 from dlt.common.configuration.specs import RunConfiguration
 from dlt.common.runtime.exec_info import get_execution_context, TExecutionContext
 from dlt.common.typing import DictStrAny, StrAny

dlt/common/runtime/slack.py

@@ -2,7 +2,8 @@
 
 
 def send_slack_message(incoming_hook: str, message: str, is_markdown: bool = True) -> None:
-    from dlt.common import json, logger
+    from dlt.common import logger
+    from dlt.common.json import json
 
     """Sends a `message` to  Slack `incoming_hook`, by default formatted as markdown."""
     r = requests.post(

dlt/common/schema/detections.py

@@ -3,7 +3,8 @@
 
 from hexbytes import HexBytes
 
-from dlt.common import pendulum, Wei
+from dlt.common.pendulum import pendulum
+from dlt.common.wei import Wei
 from dlt.common.data_types import TDataType
 from dlt.common.time import parse_iso_like_datetime
 

dlt/common/schema/schema.py

@@ -1,6 +1,5 @@
 from copy import copy, deepcopy
 from typing import ClassVar, Dict, List, Mapping, Optional, Sequence, Tuple, Any, cast, Literal
-from dlt.common import json
 from dlt.common.schema.migrations import migrate_schema
 
 from dlt.common.utils import extend_list_deduplicated

dlt/common/schema/utils.py

@@ -5,7 +5,7 @@
 from copy import deepcopy, copy
 from typing import Dict, List, Sequence, Tuple, Type, Any, cast, Iterable, Optional, Union
 
-from dlt.common import json
+from dlt.common.json import json
 from dlt.common.data_types import TDataType
 from dlt.common.exceptions import DictValidationException
 from dlt.common.normalizers.naming import NamingConvention

dlt/common/storages/__init__.py

@@ -9,6 +9,7 @@
     LoadPackageInfo,
     PackageStorage,
     TJobState,
+    create_load_id,
 )
 from .data_item_storage import DataItemStorage
 from .load_storage import LoadStorage
@@ -40,6 +41,7 @@
     "LoadPackageInfo",
     "PackageStorage",
     "TJobState",
+    "create_load_id",
     "fsspec_from_config",
     "fsspec_filesystem",
 ]

dlt/common/storages/fsspec_filesystem.py

@@ -24,7 +24,7 @@
 from fsspec.core import url_to_fs
 
 from dlt import version
-from dlt.common import pendulum
+from dlt.common.pendulum import pendulum
 from dlt.common.configuration.specs import (
     GcpCredentials,
     AwsCredentials,

dlt/common/storages/fsspecs/google_drive.py

@@ -1,7 +1,7 @@
 import posixpath
 from typing import Any, Dict, List, Literal, Optional, Tuple
 
-from dlt.common import json
+from dlt.common.json import json
 from dlt.common.configuration.specs import GcpCredentials, GcpOAuthCredentials
 from dlt.common.exceptions import MissingDependencyException
 

dlt/common/storages/schema_storage.py

@@ -1,7 +1,8 @@
 import yaml
 from typing import Iterator, List, Mapping, Tuple, cast
 
-from dlt.common import json, logger
+from dlt.common import logger
+from dlt.common.json import json
 from dlt.common.configuration import with_config
 from dlt.common.configuration.accessors import config
 from dlt.common.schema.utils import to_pretty_json, to_pretty_yaml

dlt/common/time.py

@@ -208,10 +208,12 @@ def to_seconds(td: Optional[TimedeltaSeconds]) -> Optional[float]:
     return td
 
 
-T = TypeVar("T", bound=Union[pendulum.DateTime, pendulum.Time])
+TTimeWithPrecision = TypeVar("TTimeWithPrecision", bound=Union[pendulum.DateTime, pendulum.Time])
 
 
-def reduce_pendulum_datetime_precision(value: T, microsecond_precision: int) -> T:
-    if microsecond_precision >= 6:
+def reduce_pendulum_datetime_precision(
+    value: TTimeWithPrecision, precision: int
+) -> TTimeWithPrecision:
+    if precision >= 6:
         return value
-    return value.replace(microsecond=value.microsecond // 10 ** (6 - microsecond_precision) * 10 ** (6 - microsecond_precision))  # type: ignore
+    return value.replace(microsecond=value.microsecond // 10 ** (6 - precision) * 10 ** (6 - precision))  # type: ignore

dlt/destinations/decorators.py

@@ -7,14 +7,15 @@
 from functools import wraps
 
 from dlt.common import logger
+from dlt.common.destination import TLoaderFileFormat
+from dlt.common.typing import TDataItems
+from dlt.common.schema import TTableSchema
+
 from dlt.destinations.impl.destination.factory import destination as _destination
 from dlt.destinations.impl.destination.configuration import (
     TDestinationCallableParams,
     CustomDestinationClientConfiguration,
 )
-from dlt.common.destination import TLoaderFileFormat
-from dlt.common.typing import TDataItems
-from dlt.common.schema import TTableSchema
 
 
 def destination(

dlt/destinations/impl/bigquery/bigquery.py

@@ -9,7 +9,8 @@
 from google.api_core import retry
 from google.cloud.bigquery.retry import _RETRYABLE_REASONS
 
-from dlt.common import json, logger
+from dlt.common import logger
+from dlt.common.json import json
 from dlt.common.destination import DestinationCapabilitiesContext
 from dlt.common.destination.reference import (
     FollowupJob,

dlt/destinations/impl/databricks/sql_client.py

@@ -8,8 +8,6 @@
 )
 from databricks.sql.exc import Error as DatabricksSqlError
 
-from dlt.common import pendulum
-from dlt.common import logger
 from dlt.common.destination import DestinationCapabilitiesContext
 from dlt.destinations.exceptions import (
     DatabaseTerminalException,

dlt/destinations/impl/destination/factory.py

@@ -1,14 +1,13 @@
 import typing as t
 import inspect
 from importlib import import_module
-
 from types import ModuleType
-from dlt.common.typing import AnyFun
 
+from dlt.common import logger
+from dlt.common.typing import AnyFun
 from dlt.common.destination import Destination, DestinationCapabilitiesContext, TLoaderFileFormat
 from dlt.common.configuration import known_sections, with_config, get_fun_spec
 from dlt.common.configuration.exceptions import ConfigurationValueError
-from dlt.common import logger
 from dlt.common.utils import get_callable_name, is_inner_callable
 
 from dlt.destinations.exceptions import DestinationTransientException

dlt/destinations/impl/dummy/dummy.py

@@ -14,7 +14,7 @@
     List,
 )
 
-from dlt.common import pendulum
+from dlt.common.pendulum import pendulum
 from dlt.common.schema import Schema, TTableSchema, TSchemaTables
 from dlt.common.storages import FileStorage
 from dlt.common.destination import DestinationCapabilitiesContext

dlt/destinations/impl/qdrant/qdrant_client.py

@@ -1,7 +1,9 @@
 from types import TracebackType
 from typing import ClassVar, Optional, Sequence, List, Dict, Type, Iterable, Any, IO
 
-from dlt.common import json, pendulum, logger
+from dlt.common import logger
+from dlt.common.json import json
+from dlt.common.pendulum import pendulum
 from dlt.common.schema import Schema, TTableSchema, TSchemaTables
 from dlt.common.schema.utils import get_columns_names_with_prop
 from dlt.common.destination import DestinationCapabilitiesContext

dlt/destinations/job_impl.py

@@ -3,7 +3,7 @@
 import tempfile  # noqa: 251
 from typing import Dict, Iterable, List
 
-from dlt.common import json
+from dlt.common.json import json
 from dlt.common.destination.reference import NewLoadJob, FollowupJob, TLoadJobState, LoadJob
 from dlt.common.schema import Schema, TTableSchema
 from dlt.common.storages import FileStorage

dlt/destinations/path_utils.py

@@ -1,9 +1,10 @@
 # this can probably go some other place, but it is shared by destinations, so for now it is here
 from typing import List, Sequence, Tuple
 
-import pendulum
 import re
 
+from dlt.common.pendulum import pendulum
+
 from dlt.destinations.exceptions import InvalidFilesystemLayout, CantExtractTablePrefix
 
 # TODO: ensure layout only has supported placeholders