migrate performance snippets

dlt-hub · Sep 11, 2023 · 180de6e · 180de6e
1 parent 9ac7ee9
commit 180de6e
Show file tree

Hide file tree

Showing 12 changed files with 316 additions and 166 deletions.
diff --git a/dlt/common/configuration/providers/toml.py b/dlt/common/configuration/providers/toml.py
@@ -89,6 +89,9 @@ class StringTomlProvider(BaseTomlProvider):
     def __init__(self, toml_string: str) -> None:
         super().__init__(StringTomlProvider.loads(toml_string))
 
+    def update(self, toml_string: str) -> None:
+        self._toml = self.loads(toml_string)
+
     def dumps(self) -> str:
         return tomlkit.dumps(self._toml)
 

diff --git a/docs/snippets/reference/.dlt/config.toml b/docs/snippets/reference/.dlt/config.toml
diff --git a/docs/snippets/reference/__init__.py b/docs/snippets/reference/__init__.py
diff --git a/docs/snippets/reference/parallel_load/.dlt/config.toml b/docs/snippets/reference/parallel_load/.dlt/config.toml
diff --git a/docs/snippets/reference/parallel_load/__init__.py b/docs/snippets/reference/parallel_load/__init__.py
diff --git a/docs/snippets/reference/parallel_load/parallel_load.py b/docs/snippets/reference/parallel_load/parallel_load.py
diff --git a/docs/snippets/reference/performance_chunking.py b/docs/snippets/reference/performance_chunking.py
diff --git a/docs/snippets/reference/performance_parallel_extract.py b/docs/snippets/reference/performance_parallel_extract.py
diff --git a/docs/snippets/reference/test_reference_snippets.py b/docs/snippets/reference/test_reference_snippets.py
diff --git a/docs/website/docs/conftest.py b/docs/website/docs/conftest.py
@@ -7,21 +7,25 @@
 from dlt.common.configuration.container import Container
 
 # patch which providers to enable
-from dlt.common.configuration.providers import ConfigProvider, EnvironProvider, SecretsTomlProvider, ConfigTomlProvider
+from dlt.common.configuration.providers import StringTomlProvider, EnvironProvider, SecretsTomlProvider, ConfigTomlProvider
 from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext, ConfigProvidersConfiguration
 
 from tests.utils import patch_home_dir, autouse_test_storage, preserve_environ, duckdb_pipeline_location, wipe_pipeline
 
+string_toml_provider = StringTomlProvider("")
 
 @pytest.fixture(autouse=True)
 def setup_tests(request):
     # always set working dir to main website folder
     dname =  os.path.dirname(request.module.__file__)
     config_dir = dname + "/.dlt"
 
+    # clear string toml provider
+    string_toml_provider.update("")
+
     # inject provider context so the original providers are restored at the end
     def _initial_providers():
-        return [EnvironProvider(), SecretsTomlProvider(project_dir=config_dir, add_global_config=False), ConfigTomlProvider(project_dir=config_dir, add_global_config=False)]
+        return [string_toml_provider, EnvironProvider(), SecretsTomlProvider(project_dir=config_dir, add_global_config=False), ConfigTomlProvider(project_dir=config_dir, add_global_config=False)]
 
     glob_ctx = ConfigProvidersContext()
     glob_ctx.providers = _initial_providers()