first proof of concept for cli plugin

Makefile

@@ -86,7 +86,7 @@ test-load-local:
 	DESTINATION__POSTGRES__CREDENTIALS=postgresql://loader:loader@localhost:5432/dlt_data DESTINATION__DUCKDB__CREDENTIALS=duckdb:///_storage/test_quack.duckdb  poetry run pytest tests -k '(postgres or duckdb)'
 
 test-common:
-	poetry run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/cli/common tests/load/test_dummy_client.py tests/libs tests/destinations
+	poetry run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/cli/common tests/load/test_dummy_client.py tests/libs tests/destinations tests/plugins
 
 reset-test-storage:
 	-rm -r _storage

dlt/__init__.py

@@ -42,6 +42,7 @@
 )
 from dlt.pipeline import progress
 from dlt import destinations
+from dlt.plugins import discover_plugins
 
 pipeline = _pipeline
 current = _current
@@ -53,6 +54,9 @@
 TCredentials = _CredentialsConfiguration
 "When typing source/resource function arguments it indicates that a given argument represents credentials and should be taken from dlt.secrets. Credentials may be a string, dictionary or any other type."
 
+# TODO: where would this go??
+# discover all plugins
+discover_plugins()
 
 __all__ = [
     "__version__",

dlt/cli/_dlt.py

@@ -563,6 +563,13 @@ def main() -> int:
 
     subparsers.add_parser("telemetry", help="Shows telemetry status")
 
+    # add subparsers for plugins
+    from dlt.plugins import _DLT_CLI_PLUGINS
+
+    for name, plugin in _DLT_CLI_PLUGINS.items():
+        sp = subparsers.add_parser(name, help=plugin["cli_help"])
+        plugin["func"](sp, None)
+
     args = parser.parse_args()
 
     if Venv.is_virtual_env() and not Venv.is_venv_activated():
@@ -573,6 +580,9 @@ def main() -> int:
             " the current virtual environment instead."
         )
 
+    if args.command in _DLT_CLI_PLUGINS.keys():
+        return _DLT_CLI_PLUGINS[args.command]["func"](None, args)
+
     if args.command == "schema":
         return schema_command_wrapper(args.file, args.format, args.remove_defaults)
     elif args.command == "pipeline":

dlt/common/runtime/init.py

@@ -18,6 +18,7 @@ def init_logging(config: RunConfiguration) -> None:
 def initialize_runtime(config: RunConfiguration) -> None:
     from dlt.common.runtime.telemetry import start_telemetry
     from dlt.sources.helpers import requests
+    from dlt.plugins import discover_plugins
 
     global _INITIALIZED, _RUN_CONFIGURATION
 

dlt/plugins/__init__.py

@@ -0,0 +1,44 @@
+from importlib.metadata import entry_points
+from importlib import import_module
+from argparse import ArgumentParser, Namespace
+
+from typing import Callable, Set, Optional, Dict, Any, TypedDict
+
+
+# cli plugin is called once to populate the parser and possible a second time with the received args
+TCliPlugin = Callable[[Optional[ArgumentParser], Optional[Namespace]], None]
+
+
+class CliPlugin(TypedDict):
+    func: TCliPlugin
+    cli_help: str
+
+
+# registered plugins
+_DLT_PLUGINS: Set[str] = set()
+_DLT_CLI_PLUGINS: Dict[str, CliPlugin] = {}
+_CURRENT_PLUGIN: Optional[str] = None
+
+
+def cli_plugin(subcommand: str, cli_help: str) -> Callable[[TCliPlugin], None]:
+    """decorator for creating a dlt cli subcommand"""
+
+    def register(cli_plugin: TCliPlugin) -> None:
+        # print("Registering cli command")
+        assert _CURRENT_PLUGIN
+        assert (
+            subcommand not in _DLT_CLI_PLUGINS
+        ), f"CLI Plugin with name {subcommand} already exists"
+        _DLT_CLI_PLUGINS[subcommand] = {"func": cli_plugin, "cli_help": cli_help}
+
+    return register
+
+
+def discover_plugins() -> None:
+    global _CURRENT_PLUGIN
+    for entry_point in entry_points().select(group="dlt.plugin"):  # type: ignore
+        # print(f"Found dlt plugin {entry_point.name}")
+        _CURRENT_PLUGIN = entry_point.name
+        import_module(entry_point.value)
+        _DLT_PLUGINS.add(entry_point.name)
+        _CURRENT_PLUGIN = None

tests/plugins/example_plugin/Makefile

@@ -1,5 +1,8 @@
 
-install-example-plugin:
-	pip uninstall example_plugin
+uninstall-example-plugin:
+	pip uninstall example_plugin -y
+
+install-example-plugin: uninstall-example-plugin
+	# this builds and installs the example plugin
 	poetry build
 	pip install dist/example_plugin-0.1.0-py3-none-any.whl

tests/plugins/example_plugin/example_plugin/__init__.py

@@ -1 +1,22 @@
-print("HELLO")
+import dlt
+import dlt.plugins
+
+from argparse import ArgumentParser, Namespace
+
+
+@dlt.plugins.cli_plugin(subcommand="generate-stuff", cli_help="Let's generate some stuff")
+def register_dlt_plugin(parser: ArgumentParser = None, namespace: Namespace = None) -> None:
+    # setup command
+    if parser:
+        parser.description = "HELLO"
+        parser.add_argument(
+            "--example-flag",
+            action="store_true",
+            help="This is how we add example flags.",
+            default=False,
+        )
+
+    # execute stuff
+    if namespace:
+        print("Doing some stuff")
+        print(namespace.example_flag)

tests/plugins/test_plugin_discovery.py

@@ -1,2 +1 @@
 
-