feat(pants-py-deploy): support combining new chart with existing char…

…t and specifying pull image policy

_pants/pants_py_deploy/src/pants_py_deploy/plugin.py

@@ -55,7 +55,7 @@
     FileEnvVars,
     HelmChartsExported,
 )
-from zero_3rdparty.file_utils import iter_paths_and_relative
+from zero_3rdparty.file_utils import ensure_parents_write_text, iter_paths_and_relative
 from zero_3rdparty.str_utils import ensure_suffix
 
 
@@ -124,13 +124,23 @@ def store_digest(exported_chart_path: Path):
             )
         chart_digest = DigestContents(file_contents)
 
+    old_chart_digest = await Get(
+        DigestContents,
+        PathGlobs([f"{str(chart_path)}/*", f"{str(chart_path)}/templates/*"]),
+    )
     service = request.service
     with TemporaryDirectory() as tmpdir:
         digest = await Get(DigestContents, PathGlobs([service.path]))
         if digest:
             compose_yaml = as_compose_yaml([service], digest[0])
         else:
             compose_yaml = as_new_compose_yaml([service])
+        old_chart_path = Path(tmpdir) / "old_chart"
+        for digest_content in old_chart_digest:
+            rel_path = PurePath(digest_content.path).relative_to(chart_path)
+            ensure_parents_write_text(
+                old_chart_path / rel_path, digest_content.content.decode("utf-8")
+            )
         docker_compose_path = Path(tmpdir) / "docker-compose.yaml"
         docker_compose_path.write_text(compose_yaml)
         export_from_compose(
@@ -139,6 +149,7 @@ def store_digest(exported_chart_path: Path):
             chart_name=service.chart_inferred_name,
             image_url=service.image_url,
             on_exported=store_digest,
+            old_chart_path=old_chart_path,
         )
     assert chart_digest
     return ComposeExportChart(chart_path=chart_yaml_path, files=chart_digest)

compose_chart_export/src/compose_chart_export/chart_combiner.py

@@ -0,0 +1,96 @@
+import logging
+from pathlib import Path
+from typing import Any, cast
+
+from compose_chart_export.chart_read import read_chart_name
+from model_lib import dump, parse_payload
+from zero_3rdparty.dict_nested import (
+    iter_nested_key_values,
+    read_nested_or_none,
+    update,
+)
+from zero_3rdparty.file_utils import iter_paths_and_relative
+
+logger = logging.getLogger(__name__)
+
+
+def combine_values_yaml(old: Path, new: Path) -> str:
+    parsed_old: dict = cast(dict, parse_payload(old))
+    parsed_new: dict = cast(dict, parse_payload(new))
+    chart_name = read_chart_name(new.parent)
+    old_value: Any
+    for nested_key, old_value in iter_nested_key_values(parsed_old):
+        new_value = read_nested_or_none(parsed_new, nested_key)
+        if isinstance(old_value, dict) or new_value == old_value:
+            continue
+        logger.warning(
+            f"chart={chart_name} using old value for {nested_key}={old_value} instead of {new_value}"
+        )
+        update(parsed_new, nested_key, old_value)
+    return dump(parsed_new, "yaml")
+
+
+def combine(old_path: Path, new_path: Path) -> None:
+    """
+    Warnings:
+        side effect of changing the new path
+
+    Supports annotation in existing files:
+    # FROZEN -> Will not make any updates to the file (needs to be 1st line)
+    # Lines with `# noupdate` will be kept as is (only supported at the start and end)
+    """
+    old_rel_paths: dict[str, Path] = {
+        rel_path: path
+        for path, rel_path in iter_paths_and_relative(old_path, "*", only_files=True)
+    }
+    new_rel_paths: dict[str, Path] = {
+        rel_path: path
+        for path, rel_path in iter_paths_and_relative(new_path, "*", only_files=True)
+    }
+    for rel_path, path in new_rel_paths.items():
+        if existing := old_rel_paths.get(rel_path):
+            existing_lines = existing.read_text().splitlines()
+            if rel_path == "values.yaml":
+                new_content = combine_values_yaml(existing, path)
+                path.write_text(new_content)
+                continue
+            if existing_lines and existing_lines[0] == "# FROZEN":
+                logger.warning(f"keeping as is: {rel_path}")
+                path.write_text(existing.read_text())
+                continue
+            all_lines = ensure_no_update_lines_kept(
+                existing_lines, path.read_text().splitlines(), rel_path
+            )
+            path.write_text("\n".join(all_lines) + "\n")
+    for rel_path, existing in new_rel_paths.items() - old_rel_paths.items():
+        logger.info(f"adding old file not existing in new chart: {rel_path}")
+        (new_path / rel_path).write_text(existing.read_text())
+
+
+def ensure_no_update_lines_kept(
+    old_lines: list[str], new_lines: list[str], rel_path: str
+) -> list[str]:
+    """
+    >>> ensure_no_update_lines_kept(["{{- if .Values.deployment.enabled }} # noupdate", "line1", "line2", "{{- end }} # noupdate", "final line # noupdate"], ["newline1", "newline2"])
+    ['{{- if .Values.deployment.enabled }} # noupdate', 'newline1', 'newline2', '{{- end }} # noupdate', 'final line # noupdate']
+    """
+    extra_old_lines_start: list[str] = []
+    extra_old_lines_end: list[str] = []
+    at_start = True
+    for line in old_lines:
+        if line.endswith("# noupdate"):
+            if at_start:
+                extra_old_lines_start.append(line)
+            else:
+                extra_old_lines_end.append(line)
+        else:
+            at_start = False
+    if extra_old_lines_start:
+        logger.warning(
+            f"keeping {len(extra_old_lines_start)} # noupdate lines at start {rel_path}"
+        )
+    if extra_old_lines_end:
+        logger.warning(
+            f"keeping {len(extra_old_lines_end)} # noupdate lines at end {rel_path}"
+        )
+    return extra_old_lines_start + new_lines + extra_old_lines_end

compose_chart_export/src/compose_chart_export/chart_file_templates.py

@@ -149,6 +149,7 @@ def chart_yaml(spec: ChartTemplateSpec) -> str:
 podLabels: {}
 podAnnotations: {}
 nodeSelector: {}
+imagePullPolicy: IfNotPresent
 replicas: 1"""
 
 _RESOURCES_YAML = """\
@@ -322,7 +323,7 @@ def _add_template_spec(template: str, spec: ChartTemplateSpec):
                 container_spec = dict(
                     name=name,
                     image="{{ .Values.%s.image | quote }}" % value_name,
-                    imagePullPolicy="IfNotPresent",
+                    imagePullPolicy="{{ .Values.imagePullPolicy | quote }}",
                     resources="{{- toYaml .Values.resources | nindent 10 }}"
                     if spec.use_resource_limits
                     else {},
@@ -479,6 +480,7 @@ def service_account(spec: ChartTemplateSpec) -> str:
 
 _SECRET_OPTIONAL = """\
 {{- if (eq .Values.$EXISTING_REF "") -}}
+---
 apiVersion: v1
 kind: Secret
 metadata:

compose_chart_export/src/compose_chart_export/chart_read.py

@@ -28,6 +28,10 @@ def read_chart_version(path: Path) -> str:
     return parse_payload(path / "Chart.yaml")["version"]  # type: ignore
 
 
+def read_chart_name(path: Path) -> str:
+    return parse_payload(path / "Chart.yaml")["name"]  # type: ignore
+
+
 def read_app_version(path: Path) -> str:
     return parse_payload(path / "Chart.yaml")["appVersion"]  # type: ignore
 

compose_chart_export/src/compose_chart_export/compose_export.py

@@ -3,11 +3,12 @@
 import logging
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Callable, Iterable, cast
+from typing import Callable, Iterable, Optional, cast
 
 import semver  # type: ignore
 from pydantic import BaseModel
 
+from compose_chart_export.chart_combiner import combine
 from compose_chart_export.chart_export import export_chart
 from compose_chart_export.chart_file_templates import (
     ChartTemplateSpec,
@@ -193,13 +194,14 @@ def probe_values(healthcheck: ComposeHealthCheck) -> dict:
     )
 
 
-def export_from_compose(
+def export_from_compose(  # noqa: C901
     compose_path: PathLike,
     chart_version: str,
     chart_name: str = "",
     image_url: str = "unset",
     on_exported: Callable[[Path], None] | None = None,
     use_chart_name_as_container_name: bool = True,
+    old_chart_path: Optional[Path] = None,
 ):
     chart_version = ensure_chart_version_valid(chart_version)
     compose_path = Path(compose_path)
@@ -303,6 +305,8 @@ def export_from_compose(
             secret_content = secret_with_env_vars(container_name, secret_name, env_vars)
             (chart_path / "templates" / filename).write_text(secret_content)
 
+        if old_chart_path:
+            combine(old_chart_path, chart_path)
         if on_exported:
             on_exported(chart_path)
 

compose_chart_export/tests/test_chart_export/charts/deployment_only/templates/deployment.yaml

@@ -25,7 +25,7 @@ spec:
       containers:
       - name: deployment-only
         image: {{ .Values.deployment_only.image | quote }}
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: {{ .Values.imagePullPolicy | quote }}
         resources: {}
         ports: []
         env:

compose_chart_export/tests/test_chart_export/charts/deployment_only/values.yaml

@@ -3,6 +3,7 @@ app_kubernetes_io_instance: ''
 podLabels: {}
 podAnnotations: {}
 nodeSelector: {}
+imagePullPolicy: IfNotPresent
 replicas: 1
 deployment_only:
   env: default

compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/templates/deployment.yaml

@@ -25,7 +25,7 @@ spec:
       containers:
       - name: deployment-only-with-service-account
         image: {{ .Values.deployment_only_with_service_account.image | quote }}
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: {{ .Values.imagePullPolicy | quote }}
         resources: {}
         ports: []
         env:

compose_chart_export/tests/test_chart_export/charts/deployment_only_with_service_account/values.yaml

@@ -3,6 +3,7 @@ app_kubernetes_io_instance: ''
 podLabels: {}
 podAnnotations: {}
 nodeSelector: {}
+imagePullPolicy: IfNotPresent
 replicas: 1
 serviceAccount:
   name: docker-example

compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/templates/deployment.yaml

@@ -25,7 +25,7 @@ spec:
       containers:
       - name: nginx-no-ports
         image: {{ .Values.nginx_no_ports.image | quote }}
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: {{ .Values.imagePullPolicy | quote }}
         resources: {}
         ports:
         - containerPort: 80

compose_chart_export/tests/test_chart_export/charts/nginx_no_ports/values.yaml

@@ -3,6 +3,7 @@ app_kubernetes_io_instance: ''
 podLabels: {}
 podAnnotations: {}
 nodeSelector: {}
+imagePullPolicy: IfNotPresent
 replicas: 1
 nginx_no_ports:
   image: unset

compose_chart_export/tests/test_chart_export/charts/service_deployment/templates/deployment.yaml

@@ -25,7 +25,7 @@ spec:
       containers:
       - name: service-deployment
         image: {{ .Values.service_deployment.image | quote }}
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: {{ .Values.imagePullPolicy | quote }}
         resources: {}
         ports:
         - containerPort: 8000

compose_chart_export/tests/test_chart_export/charts/service_deployment/values.yaml

@@ -3,6 +3,7 @@ app_kubernetes_io_instance: ''
 podLabels: {}
 podAnnotations: {}
 nodeSelector: {}
+imagePullPolicy: IfNotPresent
 replicas: 1
 service_deployment:
   PORT: '8000'

compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/templates/deployment.yaml

@@ -25,7 +25,7 @@ spec:
       containers:
       - name: service-deployment-with-healthcheck
         image: {{ .Values.service_deployment_with_healthcheck.image | quote }}
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: {{ .Values.imagePullPolicy | quote }}
         resources: {}
         ports:
         - containerPort: 8000

compose_chart_export/tests/test_chart_export/charts/service_deployment_with_healthcheck/values.yaml

@@ -3,6 +3,7 @@ app_kubernetes_io_instance: ''
 podLabels: {}
 podAnnotations: {}
 nodeSelector: {}
+imagePullPolicy: IfNotPresent
 replicas: 1
 service_deployment_with_healthcheck:
   PORT: '8000'