From db1c6546e4d5224d648f91a6ff2a9158bc81e942 Mon Sep 17 00:00:00 2001 From: David Constenla <1520001+daconstenla@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:09:00 +0100 Subject: [PATCH 1/4] feat: support symlinked pipelines to avoid repetition in case of similar pipelines are used for different scenarios --- kpops/cli/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kpops/cli/utils.py b/kpops/cli/utils.py index 61f5f83e0..1622ca0e4 100644 --- a/kpops/cli/utils.py +++ b/kpops/cli/utils.py @@ -2,6 +2,7 @@ from collections.abc import Iterable, Iterator from pathlib import Path +from glob import glob from kpops.const.file_type import PIPELINE_YAML @@ -20,7 +21,11 @@ def collect_pipeline_paths(pipeline_paths: Iterable[Path]) -> Iterator[Path]: if pipeline_path.is_file(): yield pipeline_path elif pipeline_path.is_dir(): - yield from sorted(pipeline_path.glob(f"**/{PIPELINE_YAML}")) + # TODO: In python 3.13 symbolic links become supported by Path.glob + # docs.python.org/3.13#pathlib.Path.glob, probably it make sense to use it after ugprading, + # likely the code will look like: + # yield from sorted(pipeline_path.glob(f"**/{PIPELINE_YAML}", recurse_symlinks=True)) + yield from sorted(Path(p).resolve() for p in glob(f"{pipeline_path}/**/{PIPELINE_YAML}", recursive=True)) else: msg = f"The entered pipeline path '{pipeline_path}' should be a directory or file." raise ValueError(msg) From 654a258e7fa60692a117aef042615da9488a99ce Mon Sep 17 00:00:00 2001 From: David Constenla <1520001+daconstenla@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:47:13 +0100 Subject: [PATCH 2/4] chore: add a symlinked pipeline to test the correct behaviour --- tests/pipeline/resources/pipeline-symlinked/pipeline.yaml | 1 + 1 file changed, 1 insertion(+) create mode 120000 tests/pipeline/resources/pipeline-symlinked/pipeline.yaml diff --git a/tests/pipeline/resources/pipeline-symlinked/pipeline.yaml b/tests/pipeline/resources/pipeline-symlinked/pipeline.yaml new file mode 120000 index 000000000..d55443da0 --- /dev/null +++ b/tests/pipeline/resources/pipeline-symlinked/pipeline.yaml @@ -0,0 +1 @@ +../first-pipeline/pipeline.yaml \ No newline at end of file From 24d0e49aefa7a366f2882cd979abfdff2ae577ea Mon Sep 17 00:00:00 2001 From: David Constenla <1520001+daconstenla@users.noreply.github.com> Date: Tue, 14 Jan 2025 22:21:13 +0100 Subject: [PATCH 3/4] chore: add a test using the symlinked pipeline --- tests/pipeline/test_generate.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 9fdcfdd58..8e126d16f 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -882,3 +882,21 @@ def test_streams_bootstrap(self, snapshot: Snapshot): assert result.exit_code == 0, result.stdout snapshot.assert_match(result.stdout, PIPELINE_YAML) + + def test_symlinked_pipeline_as_original_pipeline( + self, + ): + pipeline_original = kpops.generate( + RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML, + ) + pipeline_symlinked = kpops.generate( + RESOURCE_PATH / "pipeline-symlinked" / PIPELINE_YAML, + ) + + assert pipeline_original == pipeline_symlinked + assert len(pipeline_symlinked) == 3 + assert [component.type for component in pipeline_symlinked.components] == [ + "scheduled-producer", + "converter", + "filter", + ] From 6ec202631dd6bcff7b2a8c52badb2b85d7918432 Mon Sep 17 00:00:00 2001 From: David Constenla <1520001+daconstenla@users.noreply.github.com> Date: Tue, 14 Jan 2025 22:33:41 +0100 Subject: [PATCH 4/4] chore: include other combinations of symlinks --- .../pipeline-1/pipeline.yaml | 1 + .../pipeline-folders-with-symlinks/pipeline-2 | 1 + .../pipeline-3/pipeline.yaml | 12 ++++++++ tests/pipeline/resources/symlinked-folder | 1 + tests/pipeline/test_generate.py | 30 +++++++++++++++---- 5 files changed, 39 insertions(+), 6 deletions(-) create mode 120000 tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-1/pipeline.yaml create mode 120000 tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-2 create mode 100644 tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-3/pipeline.yaml create mode 120000 tests/pipeline/resources/symlinked-folder diff --git a/tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-1/pipeline.yaml b/tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-1/pipeline.yaml new file mode 120000 index 000000000..af9450c74 --- /dev/null +++ b/tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-1/pipeline.yaml @@ -0,0 +1 @@ +../../pipeline-folders/pipeline-1/pipeline.yaml \ No newline at end of file diff --git a/tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-2 b/tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-2 new file mode 120000 index 000000000..956fca4d9 --- /dev/null +++ b/tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-2 @@ -0,0 +1 @@ +../pipeline-folders/pipeline-2 \ No newline at end of file diff --git a/tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-3/pipeline.yaml b/tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-3/pipeline.yaml new file mode 100644 index 000000000..546c557ba --- /dev/null +++ b/tests/pipeline/resources/pipeline-folders-with-symlinks/pipeline-3/pipeline.yaml @@ -0,0 +1,12 @@ +- type: filter + name: "a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name" + values: + commandLine: + TYPE: "nothing" + resources: + requests: + memory: 3G + replicaCount: 4 + autoscaling: + minReplicas: 4 + maxReplicas: 4 diff --git a/tests/pipeline/resources/symlinked-folder b/tests/pipeline/resources/symlinked-folder new file mode 120000 index 000000000..e6c3904bc --- /dev/null +++ b/tests/pipeline/resources/symlinked-folder @@ -0,0 +1 @@ +first-pipeline \ No newline at end of file diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 8e126d16f..c9fa96d49 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -894,9 +894,27 @@ def test_symlinked_pipeline_as_original_pipeline( ) assert pipeline_original == pipeline_symlinked - assert len(pipeline_symlinked) == 3 - assert [component.type for component in pipeline_symlinked.components] == [ - "scheduled-producer", - "converter", - "filter", - ] + + def test_symlinked_folder_renders_as_original_folder_pipeline( + self, + ): + pipeline_original = kpops.generate( + RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML, + ) + pipeline_symlinked = kpops.generate( + RESOURCE_PATH / "symlinked-folder" / PIPELINE_YAML, + ) + + assert pipeline_original == pipeline_symlinked + + def test_symlinked_folder_and_pipelines_with_normal_pipeline_render_as_original( + self, + ): + pipeline_original = kpops.generate( + RESOURCE_PATH / "pipeline-folders" / PIPELINE_YAML, + ) + pipeline_symlinked = kpops.generate( + RESOURCE_PATH / "pipeline-folders-with-symlinks" / PIPELINE_YAML, + ) + + assert pipeline_original == pipeline_symlinked