feat: support symbolic linked pipelines to avoid repetition

kpops/cli/utils.py

@@ -2,6 +2,7 @@
 
 from collections.abc import Iterable, Iterator
 from pathlib import Path
+from glob import glob
 
 from kpops.const.file_type import PIPELINE_YAML
 
@@ -20,7 +21,11 @@ def collect_pipeline_paths(pipeline_paths: Iterable[Path]) -> Iterator[Path]:
         if pipeline_path.is_file():
             yield pipeline_path
         elif pipeline_path.is_dir():
-            yield from sorted(pipeline_path.glob(f"**/{PIPELINE_YAML}"))
+            # TODO: In python 3.13 symbolic links become supported by Path.glob
+            # docs.python.org/3.13#pathlib.Path.glob, probably it make sense to use it after ugprading,
+            # likely the code will look like:
+            # yield from sorted(pipeline_path.glob(f"**/{PIPELINE_YAML}", recurse_symlinks=True))
+            yield from sorted(Path(p).resolve() for p in glob(f"{pipeline_path}/**/{PIPELINE_YAML}", recursive=True))
         else:
             msg = f"The entered pipeline path '{pipeline_path}' should be a directory or file."
             raise ValueError(msg)

tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-1/pipeline.yaml

@@ -0,0 +1 @@
+../../pipeline-folders/pipeline-1/pipeline.yaml

tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-2

@@ -0,0 +1 @@
+../pipeline-folders/pipeline-2

tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-3/pipeline.yaml

@@ -0,0 +1,12 @@
+- type: filter
+  name: "a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name"
+  values:
+    commandLine:
+      TYPE: "nothing"
+    resources:
+      requests:
+        memory: 3G
+    replicaCount: 4
+    autoscaling:
+      minReplicas: 4
+      maxReplicas: 4

tests/pipeline/resources/pipeline-symlinked/pipeline.yaml

@@ -0,0 +1 @@
+../first-pipeline/pipeline.yaml

tests/pipeline/resources/symlinked-folder

@@ -0,0 +1 @@
+first-pipeline

tests/pipeline/test_generate.py

@@ -882,3 +882,39 @@ def test_streams_bootstrap(self, snapshot: Snapshot):
         assert result.exit_code == 0, result.stdout
 
         snapshot.assert_match(result.stdout, PIPELINE_YAML)
+
+    def test_symlinked_pipeline_as_original_pipeline(
+        self,
+    ):
+        pipeline_original = kpops.generate(
+            RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML,
+        )
+        pipeline_symlinked = kpops.generate(
+            RESOURCE_PATH / "pipeline-symlinked" / PIPELINE_YAML,
+        )
+
+        assert pipeline_original == pipeline_symlinked
+
+    def test_symlinked_folder_renders_as_original_folder_pipeline(
+        self,
+    ):
+        pipeline_original = kpops.generate(
+            RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML,
+        )
+        pipeline_symlinked = kpops.generate(
+            RESOURCE_PATH / "symlinked-folder" / PIPELINE_YAML,
+        )
+
+        assert pipeline_original == pipeline_symlinked
+
+    def test_symlinked_folder_and_pipelines_with_normal_pipeline_render_as_original(
+        self,
+    ):
+        pipeline_original = kpops.generate(
+            RESOURCE_PATH / "pipeline-folders" / PIPELINE_YAML,
+        )
+        pipeline_symlinked = kpops.generate(
+            RESOURCE_PATH / "pipeline-folders-with-symlinks" / PIPELINE_YAML,
+        )
+
+        assert pipeline_original == pipeline_symlinked