diff --git a/README.md b/README.md index 939240a294..0595c16089 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,6 @@ Create and activate an isolated Python virtual environment ```bash python3 -m venv venv -# This command is shell-specific, for the common use case of bash: source venv/bin/activate ``` diff --git a/deploy/scripts/package_images.py b/deploy/scripts/package_images.py index 0fa690db54..a2cbc9c0b3 100755 --- a/deploy/scripts/package_images.py +++ b/deploy/scripts/package_images.py @@ -8,6 +8,7 @@ image names are extracted from the templates and then pulled from the repo and stored in ../images as compressed tarballs; zstd compression is used. """ + import argparse import logging import os @@ -19,7 +20,7 @@ from utils import init_logging, run_cmd import yaml -# Define configuration and output directories' +# Define configuration and output directories scripts_dir = Path(__file__).resolve().parent ansible_dir = scripts_dir.parent / "ansible" helm_dir = scripts_dir.parent / "helm" @@ -58,46 +59,57 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def package_k3s(dest_dir: Path) -> None: +def package_k3s(dest_dir: Path, *, debug: bool = False) -> None: logging.info("Packaging k3s images.") - run_cmd( - [ - "ansible-playbook", - "playbook_k3s_airgapped_files.yml", - "--extra-vars", - f"package_dir={dest_dir}", - ], - cwd=str(ansible_dir), - ) - - -def package_images(image_list: List[str], tar_file: Path) -> None: + ansible_cmd = [ + "ansible-playbook", + "playbook_k3s_airgapped_files.yml", + "--extra-vars", + f"package_dir={dest_dir}", + ] + if debug: + ansible_cmd.append("-vv") + run_cmd(ansible_cmd, cwd=str(ansible_dir), print_cmd=debug, print_output=debug) + + +def package_images(image_list: List[str], tar_file: Path, *, debug: bool = False) -> None: container_cli_cmd = [os.getenv("CONTAINER_CLI", "docker")] if container_cli_cmd[0] == "nerdctl": container_cli_cmd.extend(["--namespace", "k8s.io"]) + # Pull each image for image in image_list: pull_cmd = container_cli_cmd + ["pull", image] - logging.debug(f"Running {pull_cmd}") - run_cmd(pull_cmd) + run_cmd(pull_cmd, print_cmd=debug, print_output=debug) + # Save pulled images into a .tar archive - run_cmd(container_cli_cmd + ["save"] + image_list + ["-o", str(tar_file)]) + save_cmd = container_cli_cmd + ["save"] + image_list + ["-o", str(tar_file)] + run_cmd(save_cmd, print_cmd=debug, print_output=debug) + # Compress the tarball - run_cmd(["zstd", "--rm", "--force", "--quiet", str(tar_file)]) + tar_cmd = ["zstd", "--rm", "--force", str(tar_file)] + if not debug: + tar_cmd.append("--quiet") + run_cmd(tar_cmd, print_cmd=debug, print_output=debug) def package_middleware( - config_file: str, *, cluster_type: str, image_dir: Path, chart_dir: Path + config_file: str, *, cluster_type: str, image_dir: Path, chart_dir: Path, debug: bool = False ) -> None: logging.info("Packaging middleware images.") - # read in cluster configuration + + # Read in cluster configuration with open(config_file) as file: config: Dict[str, Any] = yaml.safe_load(file) - # get current repos + + # Get current repos curr_repo_list: List[str] = [] middleware_images: List[str] = [] + helm_add_cmd = ["helm", "repo", "list", "-o", "yaml"] + if debug: + helm_add_cmd.append("--debug") helm_cmd_results = run_cmd( - ["helm", "repo", "list", "-o", "yaml"], print_cmd=False, check_results=False + helm_add_cmd, check_results=False, print_cmd=debug, print_output=debug ) if helm_cmd_results.returncode == 0: curr_helm_repos = yaml.safe_load(helm_cmd_results.stdout) @@ -105,60 +117,76 @@ def package_middleware( curr_repo_list.append(repo["name"]) for chart_descr in config["clusters"][cluster_type]: - # add the chart's repo if we don't already have it + logging.debug(f"Chart: ${chart_descr}") + + # Add the chart's repo if we don't already have it repo = config[chart_descr]["repo"] if repo["name"] not in curr_helm_repos: - run_cmd(["helm", "repo", "add", repo["name"], repo["url"]]) + helm_add_cmd = ["helm", "repo", "add", repo["name"], repo["url"]] + if debug: + helm_add_cmd.append("--debug") + run_cmd(helm_add_cmd, print_cmd=debug, print_output=debug) curr_repo_list.append(repo["name"]) - # pull the middleware chart + # Pull the middleware chart chart = config[chart_descr]["chart"] dest_dir = chart_dir / chart["name"] dest_dir.mkdir(mode=0o755, parents=True, exist_ok=True) - helm_cmd = ["helm", "pull", chart["reference"], "--destination", str(dest_dir)] + helm_pull_cmd = ["helm", "pull", chart["reference"], "--destination", str(dest_dir)] + if debug: + helm_pull_cmd.append("--debug") if "version" in chart: - helm_cmd.extend(["--version", chart["version"]]) - run_cmd(helm_cmd) - # render chart templates and extract images + helm_pull_cmd.extend(["--version", chart["version"]]) + run_cmd(helm_pull_cmd, print_cmd=debug, print_output=debug) + + # Render chart templates and extract images for chart_file in dest_dir.glob("*.tgz"): - results = run_cmd(["helm", "template", chart_file]) + results = run_cmd(["helm", "template", chart_file], print_cmd=debug) for line in results.stdout.splitlines(): match = re.match(r'[-\s]+image:\s+"*([^"\n]*)"*', line) - if match: + if match and not match.group(1) in middleware_images: logging.debug(f" - Found image {match.group(1)}") middleware_images.append(match.group(1)) + logging.debug(f"Middleware images: {middleware_images}") - package_images(middleware_images, image_dir / "middleware-airgap-images-amd64.tar") + package_images( + middleware_images, image_dir / "middleware-airgap-images-amd64.tar", debug=debug + ) -def package_thecombine(tag: str, image_dir: Path) -> None: +def package_thecombine(tag: str, image_dir: Path, *, debug: bool = False) -> None: logging.info(f"Packaging The Combine version {tag}.") logging.debug("Create helm charts from templates") combine_charts.generate(tag) + logging.debug(" - Get template for The Combine.") - results = run_cmd( - [ - "helm", - "template", - "thecombine", - str(helm_dir / "thecombine"), - "--set", - "global.imageRegistry=public.ecr.aws/thecombine", - "--set", - f"global.imageTag={tag}", - ] - ) + helm_template_cmd = [ + "helm", + "template", + "thecombine", + str(helm_dir / "thecombine"), + "--dependency-update", + "--set", + "global.imageRegistry=public.ecr.aws/thecombine", + "--set", + f"global.imageTag={tag}", + ] + if debug: + helm_template_cmd.append("--debug") + results = run_cmd(helm_template_cmd, print_cmd=debug) + combine_images: List[str] = [] for line in results.stdout.splitlines(): match = re.match(r'^[-\s]+image:\s+"*([^"\n]*)"*', line) if match: image = match.group(1) - logging.debug(f" - Found image {image}") if image not in combine_images: + logging.debug(f" - Found image {image}") combine_images.append(image) logging.debug(f"Combine images: {combine_images}") + # Logout of AWS to allow pulling the images - package_images(combine_images, image_dir / "combine-airgap-images-amd64.tar") + package_images(combine_images, image_dir / "combine-airgap-images-amd64.tar", debug=debug) def main() -> None: @@ -179,11 +207,15 @@ def main() -> None: os.environ["AWS_DEFAULT_REGION"] = "" # Update helm repos - package_k3s(image_dir) + package_k3s(image_dir, debug=args.debug) package_middleware( - args.config, cluster_type="standard", image_dir=image_dir, chart_dir=chart_dir + args.config, + cluster_type="standard", + image_dir=image_dir, + chart_dir=chart_dir, + debug=args.debug, ) - package_thecombine(args.tag, image_dir) + package_thecombine(args.tag, image_dir, debug=args.debug) if __name__ == "__main__": diff --git a/deploy/scripts/setup_combine.py b/deploy/scripts/setup_combine.py index 052103d824..e1b627fddd 100755 --- a/deploy/scripts/setup_combine.py +++ b/deploy/scripts/setup_combine.py @@ -204,6 +204,7 @@ def main() -> None: # Create the base helm install command chart_dir = helm_dir / chart helm_install_cmd = helm_cmd + [ + "--dependency-update", "--namespace", chart_namespace, helm_action.value, @@ -211,6 +212,10 @@ def main() -> None: str(chart_dir), ] + # Set the debug option if desired + if args.debug: + helm_install_cmd.append("--debug") + # Set the dry-run option if desired if args.dry_run: helm_install_cmd.append("--dry-run") @@ -251,15 +256,7 @@ def main() -> None: for variable in target_vars: helm_install_cmd.extend(["--set", variable]) - # Update chart dependencies - # Note that this operation is performed on the local helm charts - # so the kubeconfig and context arguments are not passed to the - # helm command. - helm_deps_cmd = ["helm", "dependency", "update", str(chart_dir)] - logging.debug(helm_deps_cmd) - run_cmd(helm_deps_cmd, print_cmd=not args.quiet, print_output=True) - - logging.debug(helm_install_cmd) + # Install the chart run_cmd(helm_install_cmd, print_cmd=not args.quiet, print_output=True) diff --git a/deploy/scripts/utils.py b/deploy/scripts/utils.py index 0af9cff521..d451b3aa26 100644 --- a/deploy/scripts/utils.py +++ b/deploy/scripts/utils.py @@ -22,7 +22,7 @@ def run_cmd( ) -> subprocess.CompletedProcess[str]: """Run a command with subprocess and catch any CalledProcessErrors.""" if print_cmd: - print(f"Running: {' '.join(cmd)}") + print(f"Running: {' '.join([str(arg) for arg in cmd])}") try: process_results = subprocess.run( cmd, diff --git a/installer/make-combine-installer.sh b/installer/make-combine-installer.sh index 908f4ecd4d..b7ad7085dd 100755 --- a/installer/make-combine-installer.sh +++ b/installer/make-combine-installer.sh @@ -13,11 +13,15 @@ error () { # cd to the directory where the script is installed SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +DEBUG=0 NET_INSTALL=0 # Parse arguments to customize installation while (( "$#" )) ; do OPT=$1 case $OPT in + --debug) + DEBUG=1 + ;; --net-install) NET_INSTALL=1 ;; @@ -48,23 +52,30 @@ if [[ $NET_INSTALL == 0 ]] ; then fi source venv/bin/activate # Update the environment if necessary - python -m pip install --upgrade pip pip-tools + python -m pip $((( DEBUG == 0)) && echo "-q" ) install --upgrade pip pip-tools python -m piptools sync requirements.txt # Package The Combine for "offline" installation TEMP_DIR=/tmp/images-$$ pushd scripts - ./package_images.py ${COMBINE_VERSION} ${TEMP_DIR} + ./package_images.py ${COMBINE_VERSION} ${TEMP_DIR} $((( DEBUG == 1 )) && echo "--debug") INSTALLER_NAME="combine-installer.run" popd - rm -rf venv else # Package The Combine for network installation INSTALLER_NAME="combine-net-installer.run" fi +# Remove unwanted folders +for DIR in venv scripts/__pycache__ ; do + if [ -d $DIR ] ; then + (( DEBUG == 1 )) && echo "Removing ../deploy/$DIR/" + rm -rf $DIR + fi +done + cd ${SCRIPT_DIR} -makeself --tar-quietly ../deploy ${INSTALLER_NAME} "Combine Installer" scripts/install-combine.sh ${COMBINE_VERSION} +makeself $((( DEBUG == 0)) && echo "--tar-quietly" ) ../deploy ${INSTALLER_NAME} "Combine Installer" scripts/install-combine.sh ${COMBINE_VERSION} if [[ $NET_INSTALL == 0 ]] ; then makeself --append ${TEMP_DIR} ${INSTALLER_NAME} rm -rf ${TEMP_DIR}