Merge branch 'devel' into d#/platform_connection

# Conflicts:
#	dlt/common/pipeline.py
#	dlt/common/runtime/exec_info.py
#	dlt/common/runtime/segment.py
#	dlt/common/storages/load_storage.py
#	dlt/pipeline/__init__.py
#	dlt/pipeline/pipeline.py
#	dlt/pipeline/trace.py
#	dlt/pipeline/track.py
#	tests/common/configuration/test_configuration.py
#	tests/helpers/streamlit_tests/test_streamlit_show_resources.py

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# introduce formatting with black
+c3ddbaa6e61c44a3809e625c802cb4c7632934a3

Makefile

@@ -48,11 +48,17 @@ dev: has-poetry
 
 lint:
 	./check-package.sh
+	poetry run black ./ --diff --exclude=".*syntax_error.py|\.venv.*"
+	# poetry run isort ./ --diff
 	poetry run mypy --config-file mypy.ini dlt tests
 	poetry run flake8 --max-line-length=200 dlt
 	poetry run flake8 --max-line-length=200 tests --exclude tests/reflection/module_cases
 	# $(MAKE) lint-security
 
+format:
+	poetry run black ./ --exclude=".*syntax_error.py|\.venv.*"
+	# poetry run isort ./
+
 test-and-lint-snippets:
 	poetry run mypy --config-file mypy.ini docs/website docs/examples
 	poetry run flake8 --max-line-length=200 docs/website docs/examples

dlt/__init__.py

@@ -29,7 +29,15 @@
 
 from dlt import sources
 from dlt.extract.decorators import source, resource, transformer, defer
-from dlt.pipeline import pipeline as _pipeline, run, attach, Pipeline, dbt, current as _current, mark as _mark
+from dlt.pipeline import (
+    pipeline as _pipeline,
+    run,
+    attach,
+    Pipeline,
+    dbt,
+    current as _current,
+    mark as _mark,
+)
 from dlt.pipeline import progress
 from dlt import destinations
 

dlt/cli/config_toml_writer.py

@@ -5,7 +5,11 @@
 from collections.abc import Sequence as C_Sequence
 
 from dlt.common import pendulum
-from dlt.common.configuration.specs import BaseConfiguration, is_base_configuration_inner_hint, extract_inner_hint
+from dlt.common.configuration.specs import (
+    BaseConfiguration,
+    is_base_configuration_inner_hint,
+    extract_inner_hint,
+)
 from dlt.common.data_types import py_type_to_sc_type
 from dlt.common.typing import AnyType, is_final_type, is_optional_type
 
@@ -53,13 +57,15 @@ def write_value(
     hint: AnyType,
     overwrite_existing: bool,
     default_value: Any = None,
-    is_default_of_interest: bool = False
+    is_default_of_interest: bool = False,
 ) -> None:
     # skip if table contains the name already
     if name in toml_table and not overwrite_existing:
         return
     # do not dump final and optional fields if they are not of special interest
-    if (is_final_type(hint) or is_optional_type(hint) or default_value is not None) and not is_default_of_interest:
+    if (
+        is_final_type(hint) or is_optional_type(hint) or default_value is not None
+    ) and not is_default_of_interest:
         return
     # get the inner hint to generate cool examples
     hint = extract_inner_hint(hint)
@@ -84,10 +90,19 @@ def write_spec(toml_table: TOMLTable, config: BaseConfiguration, overwrite_exist
         default_value = getattr(config, name, None)
         # check if field is of particular interest and should be included if it has default
         is_default_of_interest = name in config.__config_gen_annotations__
-        write_value(toml_table, name, hint, overwrite_existing, default_value=default_value, is_default_of_interest=is_default_of_interest)
+        write_value(
+            toml_table,
+            name,
+            hint,
+            overwrite_existing,
+            default_value=default_value,
+            is_default_of_interest=is_default_of_interest,
+        )
 
 
-def write_values(toml: TOMLContainer, values: Iterable[WritableConfigValue], overwrite_existing: bool) -> None:
+def write_values(
+    toml: TOMLContainer, values: Iterable[WritableConfigValue], overwrite_existing: bool
+) -> None:
     for value in values:
         toml_table: TOMLTable = toml  # type: ignore
         for section in value.sections:
@@ -98,4 +113,11 @@ def write_values(toml: TOMLContainer, values: Iterable[WritableConfigValue], ove
             else:
                 toml_table = toml_table[section]  # type: ignore
 
-        write_value(toml_table, value.name, value.hint, overwrite_existing, default_value=value.default_value, is_default_of_interest=True)
+        write_value(
+            toml_table,
+            value.name,
+            value.hint,
+            overwrite_existing,
+            default_value=value.default_value,
+            is_default_of_interest=True,
+        )