kfp python package as optional dependency, defer processor loading un…

…til runtimes determined, make elyra.pipeline.kfp.kfp_authentication import optional only when kfp available in build

Signed-off-by: shalberd <[email protected]>

elyra/metadata/schemasproviders.py

@@ -26,17 +26,22 @@
 try:
     from kfp_tekton import TektonClient
 except ImportError:
-    # We may not have kfp-tekton available and that's okay!
+    # We may not have kfp-tekton available and that's okay, for example when only using airflow!
     TektonClient = None
 
 from elyra.metadata.schema import SchemasProvider
 from elyra.metadata.schemaspaces import CodeSnippets
 from elyra.metadata.schemaspaces import ComponentCatalogs
 from elyra.metadata.schemaspaces import RuntimeImages
 from elyra.metadata.schemaspaces import Runtimes
-from elyra.pipeline.kfp.kfp_authentication import SupportedAuthProviders
 from elyra.util.gitutil import SupportedGitTypes
 
+try:
+    from elyra.pipeline.kfp.kfp_authentication import SupportedAuthProviders
+except ImportError:
+    # We may not have kfp available and that's okay, for example when only using airflow!
+    SupportedAuthProviders = None
+
 
 class ElyraSchemasProvider(SchemasProvider, metaclass=ABCMeta):
     """Base class used for retrieving Elyra-based schema files from its metadata/schemas directory."""
@@ -107,12 +112,16 @@ def get_schemas(self) -> List[Dict]:
             # For KFP schemas replace placeholders:
             # - properties.metadata.properties.auth_type.enum ({AUTH_PROVIDER_PLACEHOLDERS})
             # - properties.metadata.properties.auth_type.default ({DEFAULT_AUTH_PROVIDER_PLACEHOLDER})
-            auth_type_enum = SupportedAuthProviders.get_provider_names()
-            auth_type_default = SupportedAuthProviders.get_default_provider().name
+            if SupportedAuthProviders is not None:
+                auth_type_enum = SupportedAuthProviders.get_provider_names()
+                auth_type_default = SupportedAuthProviders.get_default_provider().name
 
             for schema in runtime_schemas:
                 if schema["name"] == "kfp":
-                    if schema["properties"]["metadata"]["properties"].get("auth_type") is not None:
+                    if (
+                        schema["properties"]["metadata"]["properties"].get("auth_type") is not None
+                        and SupportedAuthProviders is not None
+                    ):
                         schema["properties"]["metadata"]["properties"]["auth_type"]["enum"] = auth_type_enum
                         schema["properties"]["metadata"]["properties"]["auth_type"]["default"] = auth_type_default
 

elyra/pipeline/registry.py

@@ -57,8 +57,8 @@ def __init__(self, **kwargs):
         for processor in entrypoints.get_group_all("elyra.pipeline.processors"):
             try:
                 # instantiate an actual instance of the processor
-                processor_instance = processor.load()(root_dir=self.root_dir, parent=kwargs.get("parent"))
                 if not self.runtimes or processor.name in self.runtimes:
+                    processor_instance = processor.load()(root_dir=self.root_dir, parent=kwargs.get("parent"))
                     self._add_processor(processor_instance)
                 else:
                     self.log.info(

pyproject.toml

@@ -43,14 +43,13 @@ dependencies = [
     "rfc3986-validator>=0.1.1",
     "tornado>=6.1.0",
     "traitlets>=4.3.2",
-    "typing-extensions>=3.10,<5",  # Cap from kfp
+    "typing-extensions>=3.10",
     "urllib3>=1.26.5",
     "watchdog>=2.1.3",
     "websocket-client",
     "yaspin",
     # see: https://stackoverflow.com/questions/76175487/sudden-importerror-cannot-import-name-appengine-from-requests-packages-urlli
     "appengine-python-standard",
-    "kfp>=1.7.0,<2.0,!=1.7.2",  # We cap the SDK to <2.0 due to possible breaking changes
     "pygithub",
     "black>=22.8.0",
 ]
@@ -87,6 +86,10 @@ test = [
     "requests-unixsocket",
     "kfp-tekton"
 ]
+kfp = [
+    "kfp>=1.7.0,<2.0,!=1.7.2",  # We cap the SDK to <2.0 due to possible breaking changes
+    "typing-extensions>=3.10,<5",  # Cap from kfp
+]
 kfp-tekton = [
     "kfp-tekton>=1.5.2"  # requires kfp >= 1.8.19, which contains fix for Jupyterlab
 ]
@@ -98,9 +101,11 @@ gitlab = [
 ]
 # The following is a collection of "non-test" extra dependencies from above.
 all = [
+    "kfp>=1.7.0,<2.0,!=1.7.2",  # We cap the SDK to <2.0 due to possible breaking changes
     "kfp-tekton>=1.5.2",
     "elyra-examples-kfp-catalog",
     "python-gitlab",
+    "typing-extensions>=3.10,<5",  # Cap from kfp
 ]
 
 # Console scripts