tests deploy command with and without secrets

dlt/cli/deploy_command_helpers.py

@@ -199,16 +199,26 @@ def _update_envs(self, trace: PipelineTrace) -> None:
                     # fmt.echo(f"{resolved_value.key} in {resolved_value.sections} moved to CONFIG")
 
     def _echo_secrets(self) -> None:
+        display_info = False
         for s_v in self.secret_envs:
             fmt.secho("Name:", fg="green")
-            fmt.echo(fmt.bold(BaseDocProvider.get_key_name(s_v.key, *s_v.sections)))
+            fmt.echo(fmt.bold(self.env_prov.get_key_name(s_v.key, *s_v.sections)))
             try:
                 fmt.secho("Secret:", fg="green")
                 fmt.echo(self._lookup_secret_value(s_v))
             except ConfigFieldMissingException:
-                fmt.secho(
-                    "Not found. See https://dlthub.com/docs/general-usage/credentials", fg="red"
-                )
+                fmt.secho("Not found in the available secrets.", fg="red")
+                display_info = True
+            fmt.echo()
+        if display_info:
+            fmt.warning(
+                "We could not read and display some secrets. Starting from 1.0 version of dlt,"
+                " those are not stored in the traces. Instead we are trying to read them from the"
+                " available configuration ie. secrets.toml file. Please run the deploy command from"
+                " the same working directory you ran your pipeline script. If you pass the"
+                " credentials in code we will not be able to display them here. See"
+                " https://dlthub.com/docs/general-usage/credentials"
+            )
             fmt.echo()
 
     def _lookup_secret_value(self, trace: LookupTrace) -> Any:

tests/cli/test_deploy_command.py

@@ -19,7 +19,7 @@
 from dlt.pipeline.exceptions import CannotRestorePipelineException
 from dlt.cli.deploy_command_helpers import get_schedule_description
 
-from tests.utils import TEST_STORAGE_ROOT, test_storage
+from tests.utils import TEST_STORAGE_ROOT, reset_providers, test_storage
 
 
 DEPLOY_PARAMS = [
@@ -134,28 +134,34 @@ def test_deploy_command(
             test_storage.delete(".dlt/secrets.toml")
             test_storage.atomic_rename(".dlt/secrets.toml.ci", ".dlt/secrets.toml")
 
-            # this time script will run
-            venv.run_script("debug_pipeline.py")
-            with echo.always_choose(False, always_choose_value=True):
-                with io.StringIO() as buf, contextlib.redirect_stdout(buf):
-                    deploy_command.deploy_command(
-                        "debug_pipeline.py",
-                        deployment_method,
-                        deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                        **deployment_args
-                    )
-                    _out = buf.getvalue()
-                print(_out)
-                # make sure our secret and config values are all present
-                assert "api_key_9x3ehash" in _out
-                assert "dlt_data" in _out
-                if "schedule" in deployment_args:
-                    assert get_schedule_description(deployment_args["schedule"])
-                secrets_format = deployment_args.get("secrets_format", "env")
-                if secrets_format == "env":
-                    assert "API_KEY" in _out
-                else:
-                    assert "api_key = " in _out
+            # reset toml providers to (1) CWD (2) non existing dir so API_KEY is not found
+            for project_dir, api_key in [
+                (None, "api_key_9x3ehash"),
+                (".", "Not found in the available secrets"),
+            ]:
+                with reset_providers(project_dir=project_dir):
+                    # this time script will run
+                    venv.run_script("debug_pipeline.py")
+                    with echo.always_choose(False, always_choose_value=True):
+                        with io.StringIO() as buf, contextlib.redirect_stdout(buf):
+                            deploy_command.deploy_command(
+                                "debug_pipeline.py",
+                                deployment_method,
+                                deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
+                                **deployment_args
+                            )
+                            _out = buf.getvalue()
+                        print(_out)
+                        # make sure our secret and config values are all present
+                        assert api_key in _out
+                        assert "dlt_data" in _out
+                        if "schedule" in deployment_args:
+                            assert get_schedule_description(deployment_args["schedule"])
+                        secrets_format = deployment_args.get("secrets_format", "env")
+                        if secrets_format == "env":
+                            assert "API_KEY" in _out
+                        else:
+                            assert "api_key = " in _out
 
             # non existing script name
             with pytest.raises(NoSuchPathError):

tests/common/configuration/utils.py

@@ -20,15 +20,11 @@
 from dlt.common.configuration import configspec
 from dlt.common.configuration.specs import BaseConfiguration, CredentialsConfiguration
 from dlt.common.configuration.container import Container
-from dlt.common.configuration.providers import (
-    ConfigProvider,
-    EnvironProvider,
-    ConfigTomlProvider,
-    SecretsTomlProvider,
-)
+from dlt.common.configuration.providers import ConfigProvider, EnvironProvider
 from dlt.common.configuration.utils import get_resolved_traces
 from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext
 from dlt.common.typing import TSecretValue, StrAny
+from tests.utils import _reset_providers
 
 
 @configspec
@@ -118,14 +114,7 @@ def env_provider() -> Iterator[ConfigProvider]:
 
 @pytest.fixture
 def toml_providers() -> Iterator[ConfigProvidersContext]:
-    pipeline_root = "./tests/common/cases/configuration/.dlt"
-    ctx = ConfigProvidersContext()
-    ctx.providers.clear()
-    ctx.add_provider(EnvironProvider())
-    ctx.add_provider(SecretsTomlProvider(project_dir=pipeline_root))
-    ctx.add_provider(ConfigTomlProvider(project_dir=pipeline_root))
-    with Container().injectable_context(ctx):
-        yield ctx
+    yield from _reset_providers("./tests/common/cases/configuration/.dlt")
 
 
 class MockProvider(ConfigProvider):

tests/utils.py

@@ -1,3 +1,4 @@
+import contextlib
 import multiprocessing
 import os
 import platform
@@ -12,7 +13,12 @@
 
 import dlt
 from dlt.common.configuration.container import Container
-from dlt.common.configuration.providers import DictionaryProvider
+from dlt.common.configuration.providers import (
+    DictionaryProvider,
+    EnvironProvider,
+    SecretsTomlProvider,
+    ConfigTomlProvider,
+)
 from dlt.common.configuration.resolve import resolve_configuration
 from dlt.common.configuration.specs import RunConfiguration
 from dlt.common.configuration.specs.config_providers_context import (
@@ -382,3 +388,19 @@ def assert_query_data(
                 # the second is load id
                 if info:
                     assert row[1] in info.loads_ids
+
+
+@contextlib.contextmanager
+def reset_providers(project_dir: str) -> Iterator[ConfigProvidersContext]:
+    """Context manager injecting standard set of providers where toml providers are initialized from `project_dir`"""
+    return _reset_providers(project_dir)
+
+
+def _reset_providers(project_dir: str) -> Iterator[ConfigProvidersContext]:
+    ctx = ConfigProvidersContext()
+    ctx.providers.clear()
+    ctx.add_provider(EnvironProvider())
+    ctx.add_provider(SecretsTomlProvider(project_dir=project_dir))
+    ctx.add_provider(ConfigTomlProvider(project_dir=project_dir))
+    with Container().injectable_context(ctx):
+        yield ctx