diff --git a/_pants/pants_py_deploy/src/pants_py_deploy/plugin.py b/_pants/pants_py_deploy/src/pants_py_deploy/plugin.py index e4537e4..44c9cdc 100644 --- a/_pants/pants_py_deploy/src/pants_py_deploy/plugin.py +++ b/_pants/pants_py_deploy/src/pants_py_deploy/plugin.py @@ -55,7 +55,7 @@ FileEnvVars, HelmChartsExported, ) -from zero_3rdparty.file_utils import iter_paths_and_relative +from zero_3rdparty.file_utils import ensure_parents_write_text, iter_paths_and_relative from zero_3rdparty.str_utils import ensure_suffix @@ -124,6 +124,10 @@ def store_digest(exported_chart_path: Path): ) chart_digest = DigestContents(file_contents) + old_chart_digest = await Get( + DigestContents, + PathGlobs([f"{str(chart_path)}/*", f"{str(chart_path)}/templates/*"]), + ) service = request.service with TemporaryDirectory() as tmpdir: digest = await Get(DigestContents, PathGlobs([service.path])) @@ -131,6 +135,12 @@ def store_digest(exported_chart_path: Path): compose_yaml = as_compose_yaml([service], digest[0]) else: compose_yaml = as_new_compose_yaml([service]) + old_chart_path = Path(tmpdir) / "old_chart" + for digest_content in old_chart_digest: + rel_path = PurePath(digest_content.path).relative_to(chart_path) + ensure_parents_write_text( + old_chart_path / rel_path, digest_content.content.decode("utf-8") + ) docker_compose_path = Path(tmpdir) / "docker-compose.yaml" docker_compose_path.write_text(compose_yaml) export_from_compose( @@ -139,6 +149,7 @@ def store_digest(exported_chart_path: Path): chart_name=service.chart_inferred_name, image_url=service.image_url, on_exported=store_digest, + old_chart_path=old_chart_path, ) assert chart_digest return ComposeExportChart(chart_path=chart_yaml_path, files=chart_digest) diff --git a/compose_chart_export/src/compose_chart_export/chart_combiner.py b/compose_chart_export/src/compose_chart_export/chart_combiner.py new file mode 100644 index 0000000..21780f1 --- /dev/null +++ b/compose_chart_export/src/compose_chart_export/chart_combiner.py @@ -0,0 +1,96 @@ +import logging +from pathlib import Path +from typing import Any, cast + +from compose_chart_export.chart_read import read_chart_name +from model_lib import dump, parse_payload +from zero_3rdparty.dict_nested import ( + iter_nested_key_values, + read_nested_or_none, + update, +) +from zero_3rdparty.file_utils import iter_paths_and_relative + +logger = logging.getLogger(__name__) + + +def combine_values_yaml(old: Path, new: Path) -> str: + parsed_old: dict = cast(dict, parse_payload(old)) + parsed_new: dict = cast(dict, parse_payload(new)) + chart_name = read_chart_name(new.parent) + old_value: Any + for nested_key, old_value in iter_nested_key_values(parsed_old): + new_value = read_nested_or_none(parsed_new, nested_key) + if isinstance(old_value, dict) or new_value == old_value: + continue + logger.warning( + f"chart={chart_name} using old value for {nested_key}={old_value} instead of {new_value}" + ) + update(parsed_new, nested_key, old_value) + return dump(parsed_new, "yaml") + + +def combine(old_path: Path, new_path: Path) -> None: + """ + Warnings: + side effect of changing the new path + + Supports annotation in existing files: + # FROZEN -> Will not make any updates to the file (needs to be 1st line) + # Lines with `# noupdate` will be kept as is (only supported at the start and end) + """ + old_rel_paths: dict[str, Path] = { + rel_path: path + for path, rel_path in iter_paths_and_relative(old_path, "*", only_files=True) + } + new_rel_paths: dict[str, Path] = { + rel_path: path + for path, rel_path in iter_paths_and_relative(new_path, "*", only_files=True) + } + for rel_path, path in new_rel_paths.items(): + if existing := old_rel_paths.get(rel_path): + existing_lines = existing.read_text().splitlines() + if rel_path == "values.yaml": + new_content = combine_values_yaml(existing, path) + path.write_text(new_content) + continue + if existing_lines and existing_lines[0] == "# FROZEN": + logger.warning(f"keeping as is: {rel_path}") + path.write_text(existing.read_text()) + continue + all_lines = ensure_no_update_lines_kept( + existing_lines, path.read_text().splitlines(), rel_path + ) + path.write_text("\n".join(all_lines) + "\n") + for rel_path, existing in new_rel_paths.items() - old_rel_paths.items(): + logger.info(f"adding old file not existing in new chart: {rel_path}") + (new_path / rel_path).write_text(existing.read_text()) + + +def ensure_no_update_lines_kept( + old_lines: list[str], new_lines: list[str], rel_path: str +) -> list[str]: + """ + >>> ensure_no_update_lines_kept(["{{- if .Values.deployment.enabled }} # noupdate", "line1", "line2", "{{- end }} # noupdate", "final line # noupdate"], ["newline1", "newline2"]) + ['{{- if .Values.deployment.enabled }} # noupdate', 'newline1', 'newline2', '{{- end }} # noupdate', 'final line # noupdate'] + """ + extra_old_lines_start: list[str] = [] + extra_old_lines_end: list[str] = [] + at_start = True + for line in old_lines: + if line.endswith("# noupdate"): + if at_start: + extra_old_lines_start.append(line) + else: + extra_old_lines_end.append(line) + else: + at_start = False + if extra_old_lines_start: + logger.warning( + f"keeping {len(extra_old_lines_start)} # noupdate lines at start {rel_path}" + ) + if extra_old_lines_end: + logger.warning( + f"keeping {len(extra_old_lines_end)} # noupdate lines at end {rel_path}" + ) + return extra_old_lines_start + new_lines + extra_old_lines_end diff --git a/compose_chart_export/src/compose_chart_export/chart_file_templates.py b/compose_chart_export/src/compose_chart_export/chart_file_templates.py index 8ec49f3..bffa294 100644 --- a/compose_chart_export/src/compose_chart_export/chart_file_templates.py +++ b/compose_chart_export/src/compose_chart_export/chart_file_templates.py @@ -149,6 +149,7 @@ def chart_yaml(spec: ChartTemplateSpec) -> str: podLabels: {} podAnnotations: {} nodeSelector: {} +imagePullPolicy: IfNotPresent replicas: 1""" _RESOURCES_YAML = """\ @@ -322,7 +323,7 @@ def _add_template_spec(template: str, spec: ChartTemplateSpec): container_spec = dict( name=name, image="{{ .Values.%s.image | quote }}" % value_name, - imagePullPolicy="IfNotPresent", + imagePullPolicy="{{ .Values.imagePullPolicy | quote }}", resources="{{- toYaml .Values.resources | nindent 10 }}" if spec.use_resource_limits else {}, @@ -479,6 +480,7 @@ def service_account(spec: ChartTemplateSpec) -> str: _SECRET_OPTIONAL = """\ {{- if (eq .Values.$EXISTING_REF "") -}} +--- apiVersion: v1 kind: Secret metadata: diff --git a/compose_chart_export/src/compose_chart_export/chart_read.py b/compose_chart_export/src/compose_chart_export/chart_read.py index 9260950..9eb77f8 100644 --- a/compose_chart_export/src/compose_chart_export/chart_read.py +++ b/compose_chart_export/src/compose_chart_export/chart_read.py @@ -28,6 +28,10 @@ def read_chart_version(path: Path) -> str: return parse_payload(path / "Chart.yaml")["version"] # type: ignore +def read_chart_name(path: Path) -> str: + return parse_payload(path / "Chart.yaml")["name"] # type: ignore + + def read_app_version(path: Path) -> str: return parse_payload(path / "Chart.yaml")["appVersion"] # type: ignore diff --git a/compose_chart_export/src/compose_chart_export/compose_export.py b/compose_chart_export/src/compose_chart_export/compose_export.py index d036d83..3f04ae2 100644 --- a/compose_chart_export/src/compose_chart_export/compose_export.py +++ b/compose_chart_export/src/compose_chart_export/compose_export.py @@ -3,11 +3,12 @@ import logging from pathlib import Path from tempfile import TemporaryDirectory -from typing import Callable, Iterable, cast +from typing import Callable, Iterable, Optional, cast import semver # type: ignore from pydantic import BaseModel +from compose_chart_export.chart_combiner import combine from compose_chart_export.chart_export import export_chart from compose_chart_export.chart_file_templates import ( ChartTemplateSpec, @@ -193,13 +194,14 @@ def probe_values(healthcheck: ComposeHealthCheck) -> dict: ) -def export_from_compose( +def export_from_compose( # noqa: C901 compose_path: PathLike, chart_version: str, chart_name: str = "", image_url: str = "unset", on_exported: Callable[[Path], None] | None = None, use_chart_name_as_container_name: bool = True, + old_chart_path: Optional[Path] = None, ): chart_version = ensure_chart_version_valid(chart_version) compose_path = Path(compose_path) @@ -303,6 +305,8 @@ def export_from_compose( secret_content = secret_with_env_vars(container_name, secret_name, env_vars) (chart_path / "templates" / filename).write_text(secret_content) + if old_chart_path: + combine(old_chart_path, chart_path) if on_exported: on_exported(chart_path) diff --git a/compose_chart_export/tests/test_chart_export/charts/deployment_only/templates/deployment.yaml b/compose_chart_export/tests/test_chart_export/charts/deployment_only/templates/deployment.yaml index 5f270d6..7724ea1 100644 --- a/compose_chart_export/tests/test_chart_export/charts/deployment_only/templates/deployment.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/deployment_only/templates/deployment.yaml @@ -25,7 +25,7 @@ spec: containers: - name: deployment-only image: {{ .Values.deployment_only.image | quote }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.imagePullPolicy | quote }} resources: {} ports: [] env: diff --git a/compose_chart_export/tests/test_chart_export/charts/deployment_only/values.yaml b/compose_chart_export/tests/test_chart_export/charts/deployment_only/values.yaml index fb9f53d..b478b33 100644 --- a/compose_chart_export/tests/test_chart_export/charts/deployment_only/values.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/deployment_only/values.yaml @@ -3,6 +3,7 @@ app_kubernetes_io_instance: '' podLabels: {} podAnnotations: {} nodeSelector: {} +imagePullPolicy: IfNotPresent replicas: 1 deployment_only: env: default diff --git a/compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/templates/deployment.yaml b/compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/templates/deployment.yaml index 8e4ef6c..c3f15aa 100644 --- a/compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/templates/deployment.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/templates/deployment.yaml @@ -25,7 +25,7 @@ spec: containers: - name: deployment-only-with-service-account image: {{ .Values.deployment_only_with_service_account.image | quote }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.imagePullPolicy | quote }} resources: {} ports: [] env: diff --git a/compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/values.yaml b/compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/values.yaml index 75ba879..31a768b 100644 --- a/compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/values.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/values.yaml @@ -3,6 +3,7 @@ app_kubernetes_io_instance: '' podLabels: {} podAnnotations: {} nodeSelector: {} +imagePullPolicy: IfNotPresent replicas: 1 serviceAccount: name: docker-example diff --git a/compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/templates/deployment.yaml b/compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/templates/deployment.yaml index a0bd6c3..8ce0a5d 100644 --- a/compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/templates/deployment.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/templates/deployment.yaml @@ -25,7 +25,7 @@ spec: containers: - name: nginx-no-ports image: {{ .Values.nginx_no_ports.image | quote }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.imagePullPolicy | quote }} resources: {} ports: - containerPort: 80 diff --git a/compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/values.yaml b/compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/values.yaml index 8e8f093..85f681a 100644 --- a/compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/values.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/values.yaml @@ -3,6 +3,7 @@ app_kubernetes_io_instance: '' podLabels: {} podAnnotations: {} nodeSelector: {} +imagePullPolicy: IfNotPresent replicas: 1 nginx_no_ports: image: unset diff --git a/compose_chart_export/tests/test_chart_export/charts/service_deployment/templates/deployment.yaml b/compose_chart_export/tests/test_chart_export/charts/service_deployment/templates/deployment.yaml index e8c1827..9378326 100644 --- a/compose_chart_export/tests/test_chart_export/charts/service_deployment/templates/deployment.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/service_deployment/templates/deployment.yaml @@ -25,7 +25,7 @@ spec: containers: - name: service-deployment image: {{ .Values.service_deployment.image | quote }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.imagePullPolicy | quote }} resources: {} ports: - containerPort: 8000 diff --git a/compose_chart_export/tests/test_chart_export/charts/service_deployment/values.yaml b/compose_chart_export/tests/test_chart_export/charts/service_deployment/values.yaml index ec0a9b0..41cc7a0 100644 --- a/compose_chart_export/tests/test_chart_export/charts/service_deployment/values.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/service_deployment/values.yaml @@ -3,6 +3,7 @@ app_kubernetes_io_instance: '' podLabels: {} podAnnotations: {} nodeSelector: {} +imagePullPolicy: IfNotPresent replicas: 1 service_deployment: PORT: '8000' diff --git a/compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/templates/deployment.yaml b/compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/templates/deployment.yaml index 3aab25f..3baad50 100644 --- a/compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/templates/deployment.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/templates/deployment.yaml @@ -25,7 +25,7 @@ spec: containers: - name: service-deployment-with-healthcheck image: {{ .Values.service_deployment_with_healthcheck.image | quote }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.imagePullPolicy | quote }} resources: {} ports: - containerPort: 8000 diff --git a/compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/values.yaml b/compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/values.yaml index 920b541..a70f360 100644 --- a/compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/values.yaml +++ b/compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/values.yaml @@ -3,6 +3,7 @@ app_kubernetes_io_instance: '' podLabels: {} podAnnotations: {} nodeSelector: {} +imagePullPolicy: IfNotPresent replicas: 1 service_deployment_with_healthcheck: PORT: '8000' diff --git a/compose_chart_export/tests/test_chart_export/test_chart_combiner.py b/compose_chart_export/tests/test_chart_export/test_chart_combiner.py new file mode 100644 index 0000000..01d024b --- /dev/null +++ b/compose_chart_export/tests/test_chart_export/test_chart_combiner.py @@ -0,0 +1,133 @@ +from compose_chart_export.chart_combiner import combine +from zero_3rdparty.file_utils import ensure_parents_write_text, iter_paths_and_relative + +_frozen_file = """\ +# FROZEN +line1 +line2""" + +_no_update_file = """\ +lineextra # noupdate +line1 +line2 +lineendextra # noupdate +lineendextra2 # noupdate +""" + +_no_update_file_updated = """\ +lineextra # noupdate +newline1 +newline2 +lineendextra # noupdate +lineendextra2 # noupdate +""" + +_values_yaml_old = """\ +replicas: 3 # should be kept +docker_example: + PORT: '8000' + env: default + name: __REQUIRED__ + secret1_env_var1: KEEP_OLD # should be kept + image: docker-example-amd-docker:latest-amd + startupProbe: + exec: + command: + - sh + - -c + - curl -f http://localhost:8000/health || exit 1 + initialDelaySeconds: 0 + periodSeconds: 3 # should be kept + timeoutSeconds: 1 # should be kept + failureThreshold: 10 # should be kept +existing_secret_secret1: '' +existing_secret_secret2: '' +some_extra_value: "my-extra-value" # should be kept +""" +_values_yaml_new = """\ +replicas: 1 +serviceAccount: + name: docker-example + annotations: {} + create: true +docker_example: + PORT: '8000' + env: default + name: __REQUIRED__ + secret1_env_var1: DEFAULT1 + secret1_env_var2: DEFAULT2 # should be added + secret2_env_var3: DEFAULT3 # should be added + image: docker-example-amd-docker:latest-amd + startupProbe: + exec: + command: + - sh + - -c + - curl -f http://localhost:8000/health || exit 1 + initialDelaySeconds: 0 + periodSeconds: 30 + timeoutSeconds: 30 + failureThreshold: 3 +existing_secret_secret1: '' +existing_secret_secret2: '' +""" + +_combined_values_yaml = """\ +replicas: 3 +docker_example: + PORT: '8000' + env: default + name: __REQUIRED__ + secret1_env_var1: KEEP_OLD + secret1_env_var2: DEFAULT2 + secret2_env_var3: DEFAULT3 + image: docker-example-amd-docker:latest-amd + startupProbe: + exec: + command: + - sh + - -c + - curl -f http://localhost:8000/health || exit 1 + initialDelaySeconds: 0 + periodSeconds: 3 + timeoutSeconds: 1 + failureThreshold: 10 +existing_secret_secret1: '' +existing_secret_secret2: '' +some_extra_value: "my-extra-value" +""" + + +def test_combine(tmp_path): + old_paths = { + "frozen.yaml": _frozen_file, + "no_update.yaml": _no_update_file, + "not_in_new.yaml": "old_content_unchanged", + "templates/fully_replace.yaml": "override_me", + "values.yaml": _values_yaml_old, + } + new_paths = { + "frozen.yaml": "I WILL HAVE OLD CONTENT", + "no_update.yaml": "newline1\nnewline2\n", + "templates/fully_replace.yaml": "new_content_only", + "values.yaml": _values_yaml_new, + } + expected_content = { + "frozen.yaml": _frozen_file, + "no_update.yaml": _no_update_file_updated, + "not_in_new.yaml": "old_content_unchanged", + "templates/fully_replace.yaml": "new_content_only", + "values.yaml": _combined_values_yaml, + } + for rel_path, content in old_paths.items(): + ensure_parents_write_text(tmp_path / "old", content) + + for rel_path, content in new_paths.items(): + ensure_parents_write_text(tmp_path / "new", content) + combine(tmp_path / "old", tmp_path / "new") + + for path, rel_path in iter_paths_and_relative( + tmp_path / "new", "*", only_files=True + ): + expected = expected_content[rel_path] + assert path.read_text() == expected diff --git a/docker_example/src/docker_example/chart/templates/deployment.yaml b/docker_example/src/docker_example/chart/templates/deployment.yaml index df12e18..99d2f4a 100644 --- a/docker_example/src/docker_example/chart/templates/deployment.yaml +++ b/docker_example/src/docker_example/chart/templates/deployment.yaml @@ -1,3 +1,5 @@ +{{- if .Values.deployment.enabled }} # noupdate +--- # noupdate apiVersion: apps/v1 kind: Deployment metadata: @@ -25,7 +27,7 @@ spec: containers: - name: docker-example image: {{ .Values.docker_example.image | quote }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.imagePullPolicy | quote }} resources: {} ports: - containerPort: 8000 @@ -52,3 +54,4 @@ spec: {{- with .Values.nodeSelector }} {{- toYaml . | nindent 8 }} {{- end }} +{{- end }} # noupdate diff --git a/docker_example/src/docker_example/chart/templates/secret_secret1.yaml b/docker_example/src/docker_example/chart/templates/secret_secret1.yaml index 86e6279..151aeaa 100644 --- a/docker_example/src/docker_example/chart/templates/secret_secret1.yaml +++ b/docker_example/src/docker_example/chart/templates/secret_secret1.yaml @@ -1,4 +1,5 @@ {{- if (eq .Values.existing_secret_secret1 "") -}} +--- apiVersion: v1 kind: Secret metadata: @@ -9,4 +10,4 @@ metadata: data: secret1_env_var1: {{ .Values.docker_example.secret1_env_var1 | b64enc | quote }} secret1_env_var2: {{ .Values.docker_example.secret1_env_var2 | b64enc | quote }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/docker_example/src/docker_example/chart/templates/secret_secret2.yaml b/docker_example/src/docker_example/chart/templates/secret_secret2.yaml index 8ea4868..6196858 100644 --- a/docker_example/src/docker_example/chart/templates/secret_secret2.yaml +++ b/docker_example/src/docker_example/chart/templates/secret_secret2.yaml @@ -1,4 +1,5 @@ {{- if (eq .Values.existing_secret_secret2 "") -}} +--- apiVersion: v1 kind: Secret metadata: @@ -8,4 +9,4 @@ metadata: namespace: {{ .Release.Namespace }} data: secret2_env_var3: {{ .Values.docker_example.secret2_env_var3 | b64enc | quote }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/docker_example/src/docker_example/chart/values.yaml b/docker_example/src/docker_example/chart/values.yaml index 319bb6c..3378d82 100644 --- a/docker_example/src/docker_example/chart/values.yaml +++ b/docker_example/src/docker_example/chart/values.yaml @@ -3,6 +3,7 @@ app_kubernetes_io_instance: '' podLabels: {} podAnnotations: {} nodeSelector: {} +imagePullPolicy: IfNotPresent replicas: 1 serviceAccount: name: docker-example @@ -33,9 +34,9 @@ docker_example: - -c - curl -f http://localhost:8000/health || exit 1 initialDelaySeconds: 0 - periodSeconds: 30 + periodSeconds: 3 timeoutSeconds: 30 - failureThreshold: 3 + failureThreshold: 10 startupProbe: exec: command: @@ -48,3 +49,6 @@ docker_example: failureThreshold: 3 existing_secret_secret1: '' existing_secret_secret2: '' +some_new_value: Hello! +deployment: + enabled: true