Use --dependency-update; Add debugging; Remove junk dirs

README.md

@@ -203,7 +203,6 @@ Create and activate an isolated Python virtual environment
 
 ```bash
 python3 -m venv venv
-# This command is shell-specific, for the common use case of bash:
 source venv/bin/activate
 ```
 

deploy/scripts/package_images.py

@@ -8,6 +8,7 @@
 image names are extracted from the templates and then pulled from the repo and stored
 in ../images as compressed tarballs; zstd compression is used.
 """
+
 import argparse
 import logging
 import os
@@ -19,7 +20,7 @@
 from utils import init_logging, run_cmd
 import yaml
 
-# Define configuration and output directories'
+# Define configuration and output directories
 scripts_dir = Path(__file__).resolve().parent
 ansible_dir = scripts_dir.parent / "ansible"
 helm_dir = scripts_dir.parent / "helm"
@@ -58,107 +59,134 @@ def parse_args() -> argparse.Namespace:
     return parser.parse_args()
 
 
-def package_k3s(dest_dir: Path) -> None:
+def package_k3s(dest_dir: Path, *, debug: bool = False) -> None:
     logging.info("Packaging k3s images.")
-    run_cmd(
-        [
-            "ansible-playbook",
-            "playbook_k3s_airgapped_files.yml",
-            "--extra-vars",
-            f"package_dir={dest_dir}",
-        ],
-        cwd=str(ansible_dir),
-    )
-
-
-def package_images(image_list: List[str], tar_file: Path) -> None:
+    ansible_cmd = [
+        "ansible-playbook",
+        "playbook_k3s_airgapped_files.yml",
+        "--extra-vars",
+        f"package_dir={dest_dir}",
+    ]
+    if debug:
+        ansible_cmd.append("-vv")
+    run_cmd(ansible_cmd, cwd=str(ansible_dir), print_cmd=debug, print_output=debug)
+
+
+def package_images(image_list: List[str], tar_file: Path, *, debug: bool = False) -> None:
     container_cli_cmd = [os.getenv("CONTAINER_CLI", "docker")]
     if container_cli_cmd[0] == "nerdctl":
         container_cli_cmd.extend(["--namespace", "k8s.io"])
+
     # Pull each image
     for image in image_list:
         pull_cmd = container_cli_cmd + ["pull", image]
-        logging.debug(f"Running {pull_cmd}")
-        run_cmd(pull_cmd)
+        run_cmd(pull_cmd, print_cmd=debug, print_output=debug)
+
     # Save pulled images into a .tar archive
-    run_cmd(container_cli_cmd + ["save"] + image_list + ["-o", str(tar_file)])
+    save_cmd = container_cli_cmd + ["save"] + image_list + ["-o", str(tar_file)]
+    run_cmd(save_cmd, print_cmd=debug, print_output=debug)
+
     # Compress the tarball
-    run_cmd(["zstd", "--rm", "--force", "--quiet", str(tar_file)])
+    tar_cmd = ["zstd", "--rm", "--force", str(tar_file)]
+    if not debug:
+        tar_cmd.append("--quiet")
+    run_cmd(tar_cmd, print_cmd=debug, print_output=debug)
 
 
 def package_middleware(
-    config_file: str, *, cluster_type: str, image_dir: Path, chart_dir: Path
+    config_file: str, *, cluster_type: str, image_dir: Path, chart_dir: Path, debug: bool = False
 ) -> None:
     logging.info("Packaging middleware images.")
-    # read in cluster configuration
+
+    # Read in cluster configuration
     with open(config_file) as file:
         config: Dict[str, Any] = yaml.safe_load(file)
-    # get current repos
+
+    # Get current repos
     curr_repo_list: List[str] = []
     middleware_images: List[str] = []
+    helm_add_cmd = ["helm", "repo", "list", "-o", "yaml"]
+    if debug:
+        helm_add_cmd.append("--debug")
     helm_cmd_results = run_cmd(
-        ["helm", "repo", "list", "-o", "yaml"], print_cmd=False, check_results=False
+        helm_add_cmd, check_results=False, print_cmd=debug, print_output=debug
     )
     if helm_cmd_results.returncode == 0:
         curr_helm_repos = yaml.safe_load(helm_cmd_results.stdout)
         for repo in curr_helm_repos:
             curr_repo_list.append(repo["name"])
 
     for chart_descr in config["clusters"][cluster_type]:
-        # add the chart's repo if we don't already have it
+        logging.debug(f"Chart: ${chart_descr}")
+
+        # Add the chart's repo if we don't already have it
         repo = config[chart_descr]["repo"]
         if repo["name"] not in curr_helm_repos:
-            run_cmd(["helm", "repo", "add", repo["name"], repo["url"]])
+            helm_add_cmd = ["helm", "repo", "add", repo["name"], repo["url"]]
+            if debug:
+                helm_add_cmd.append("--debug")
+            run_cmd(helm_add_cmd, print_cmd=debug, print_output=debug)
             curr_repo_list.append(repo["name"])
 
-        # pull the middleware chart
+        # Pull the middleware chart
         chart = config[chart_descr]["chart"]
         dest_dir = chart_dir / chart["name"]
         dest_dir.mkdir(mode=0o755, parents=True, exist_ok=True)
-        helm_cmd = ["helm", "pull", chart["reference"], "--destination", str(dest_dir)]
+        helm_pull_cmd = ["helm", "pull", chart["reference"], "--destination", str(dest_dir)]
+        if debug:
+            helm_pull_cmd.append("--debug")
         if "version" in chart:
-            helm_cmd.extend(["--version", chart["version"]])
-        run_cmd(helm_cmd)
-        # render chart templates and extract images
+            helm_pull_cmd.extend(["--version", chart["version"]])
+        run_cmd(helm_pull_cmd, print_cmd=debug, print_output=debug)
+
+        # Render chart templates and extract images
         for chart_file in dest_dir.glob("*.tgz"):
-            results = run_cmd(["helm", "template", chart_file])
+            results = run_cmd(["helm", "template", chart_file], print_cmd=debug)
             for line in results.stdout.splitlines():
                 match = re.match(r'[-\s]+image:\s+"*([^"\n]*)"*', line)
-                if match:
+                if match and not match.group(1) in middleware_images:
                     logging.debug(f"    - Found image {match.group(1)}")
                     middleware_images.append(match.group(1))
+
     logging.debug(f"Middleware images: {middleware_images}")
-    package_images(middleware_images, image_dir / "middleware-airgap-images-amd64.tar")
+    package_images(
+        middleware_images, image_dir / "middleware-airgap-images-amd64.tar", debug=debug
+    )
 
 
-def package_thecombine(tag: str, image_dir: Path) -> None:
+def package_thecombine(tag: str, image_dir: Path, *, debug: bool = False) -> None:
     logging.info(f"Packaging The Combine version {tag}.")
     logging.debug("Create helm charts from templates")
     combine_charts.generate(tag)
+
     logging.debug(" - Get template for The Combine.")
-    results = run_cmd(
-        [
-            "helm",
-            "template",
-            "thecombine",
-            str(helm_dir / "thecombine"),
-            "--set",
-            "global.imageRegistry=public.ecr.aws/thecombine",
-            "--set",
-            f"global.imageTag={tag}",
-        ]
-    )
+    helm_template_cmd = [
+        "helm",
+        "template",
+        "thecombine",
+        str(helm_dir / "thecombine"),
+        "--dependency-update",
+        "--set",
+        "global.imageRegistry=public.ecr.aws/thecombine",
+        "--set",
+        f"global.imageTag={tag}",
+    ]
+    if debug:
+        helm_template_cmd.append("--debug")
+    results = run_cmd(helm_template_cmd, print_cmd=debug)
+
     combine_images: List[str] = []
     for line in results.stdout.splitlines():
         match = re.match(r'^[-\s]+image:\s+"*([^"\n]*)"*', line)
         if match:
             image = match.group(1)
-            logging.debug(f"    - Found image {image}")
             if image not in combine_images:
+                logging.debug(f"    - Found image {image}")
                 combine_images.append(image)
     logging.debug(f"Combine images: {combine_images}")
+
     # Logout of AWS to allow pulling the images
-    package_images(combine_images, image_dir / "combine-airgap-images-amd64.tar")
+    package_images(combine_images, image_dir / "combine-airgap-images-amd64.tar", debug=debug)
 
 
 def main() -> None:
@@ -179,11 +207,15 @@ def main() -> None:
     os.environ["AWS_DEFAULT_REGION"] = ""
 
     # Update helm repos
-    package_k3s(image_dir)
+    package_k3s(image_dir, debug=args.debug)
     package_middleware(
-        args.config, cluster_type="standard", image_dir=image_dir, chart_dir=chart_dir
+        args.config,
+        cluster_type="standard",
+        image_dir=image_dir,
+        chart_dir=chart_dir,
+        debug=args.debug,
     )
-    package_thecombine(args.tag, image_dir)
+    package_thecombine(args.tag, image_dir, debug=args.debug)
 
 
 if __name__ == "__main__":

deploy/scripts/setup_combine.py

@@ -204,13 +204,18 @@ def main() -> None:
             # Create the base helm install command
             chart_dir = helm_dir / chart
             helm_install_cmd = helm_cmd + [
+                "--dependency-update",
                 "--namespace",
                 chart_namespace,
                 helm_action.value,
                 chart,
                 str(chart_dir),
             ]
 
+            # Set the debug option if desired
+            if args.debug:
+                helm_install_cmd.append("--debug")
+
             # Set the dry-run option if desired
             if args.dry_run:
                 helm_install_cmd.append("--dry-run")
@@ -251,15 +256,7 @@ def main() -> None:
             for variable in target_vars:
                 helm_install_cmd.extend(["--set", variable])
 
-            # Update chart dependencies
-            # Note that this operation is performed on the local helm charts
-            # so the kubeconfig and context arguments are not passed to the
-            # helm command.
-            helm_deps_cmd = ["helm", "dependency", "update", str(chart_dir)]
-            logging.debug(helm_deps_cmd)
-            run_cmd(helm_deps_cmd, print_cmd=not args.quiet, print_output=True)
-
-            logging.debug(helm_install_cmd)
+            # Install the chart
             run_cmd(helm_install_cmd, print_cmd=not args.quiet, print_output=True)
 
 

deploy/scripts/utils.py

@@ -22,7 +22,7 @@ def run_cmd(
 ) -> subprocess.CompletedProcess[str]:
     """Run a command with subprocess and catch any CalledProcessErrors."""
     if print_cmd:
-        print(f"Running: {' '.join(cmd)}")
+        print(f"Running: {' '.join([str(arg) for arg in cmd])}")
     try:
         process_results = subprocess.run(
             cmd,

installer/make-combine-installer.sh

@@ -13,11 +13,15 @@ error () {
 # cd to the directory where the script is installed
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
+DEBUG=0
 NET_INSTALL=0
 # Parse arguments to customize installation
 while (( "$#" )) ; do
   OPT=$1
   case $OPT in
+    --debug)
+      DEBUG=1
+      ;;
     --net-install)
       NET_INSTALL=1
       ;;
@@ -48,23 +52,30 @@ if [[ $NET_INSTALL == 0 ]] ; then
   fi
   source venv/bin/activate
   # Update the environment if necessary
-  python -m pip install --upgrade pip pip-tools
+  python -m pip $((( DEBUG == 0)) && echo "-q" ) install --upgrade pip pip-tools
   python -m piptools sync requirements.txt
 
   # Package The Combine for "offline" installation
   TEMP_DIR=/tmp/images-$$
   pushd scripts
-  ./package_images.py ${COMBINE_VERSION} ${TEMP_DIR}
+  ./package_images.py ${COMBINE_VERSION} ${TEMP_DIR} $((( DEBUG == 1 )) && echo "--debug")
   INSTALLER_NAME="combine-installer.run"
   popd
-  rm -rf venv
 else
   # Package The Combine for network installation
   INSTALLER_NAME="combine-net-installer.run"
 fi
 
+# Remove unwanted folders
+for DIR in venv scripts/__pycache__ ; do
+  if [ -d $DIR ] ; then
+    (( DEBUG == 1 )) && echo "Removing ../deploy/$DIR/"
+    rm -rf $DIR
+  fi
+done
+
 cd ${SCRIPT_DIR}
-makeself --tar-quietly ../deploy ${INSTALLER_NAME} "Combine Installer" scripts/install-combine.sh ${COMBINE_VERSION}
+makeself $((( DEBUG == 0)) && echo "--tar-quietly" ) ../deploy ${INSTALLER_NAME} "Combine Installer" scripts/install-combine.sh ${COMBINE_VERSION}
 if  [[ $NET_INSTALL == 0 ]] ; then
   makeself --append ${TEMP_DIR} ${INSTALLER_NAME}
   rm -rf ${TEMP_DIR}