From 119f1b1c7855a174f8331da4ef028d4dfbd3e318 Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Tue, 7 Jan 2025 18:50:33 -0800
Subject: [PATCH] templates/common/util_functions: * increased minimum memory
 threshold for ram disk * moved apt_add_repo and friends to
 common/install_functions

templates/dask/util_functions:
* validating conda tarball before caching to gcs

templates/generate-action.pl:
* improved usage documentation a little

templates/gpu/install_functions
* using /opt/conda/miniconda3/bin/python3 instead of /usr/bin/ for
  venv pre-install
---
 templates/common/util_functions | 56 +--------------------------------
 templates/dask/util_functions   |  4 +--
 templates/generate-action.pl    | 14 ++++++++-
 templates/gpu/install_functions |  4 +--
 4 files changed, 18 insertions(+), 60 deletions(-)

diff --git a/templates/common/util_functions b/templates/common/util_functions
index 4d9f983a4..336af37f8 100644
--- a/templates/common/util_functions
+++ b/templates/common/util_functions
@@ -297,7 +297,7 @@ function is_ramdisk() {
 function mount_ramdisk(){
   local free_mem
   free_mem="$(awk '/^MemFree/ {print $2}' /proc/meminfo)"
-  if [[ ${free_mem} -lt 10500000 ]]; then return 0 ; fi
+  if [[ ${free_mem} -lt 20500000 ]]; then return 0 ; fi
 
   # Write to a ramdisk instead of churning the persistent disk
 
@@ -350,60 +350,6 @@ function check_os() {
   fi
 }
 
-#
-# Generate repo file under /etc/apt/sources.list.d/
-#
-function apt_add_repo() {
-  local -r repo_name="$1"
-  local -r repo_data="$3" # "http(s)://host/path/uri argument0 .. argumentN"
-  local -r include_src="${4:-yes}"
-  local -r kr_path="${5:-/usr/share/keyrings/${repo_name}.gpg}"
-  local -r repo_path="${6:-/etc/apt/sources.list.d/${repo_name}.list}"
-
-  echo "deb [signed-by=${kr_path}] ${repo_data}" > "${repo_path}"
-  if [[ "${include_src}" == "yes" ]] ; then
-    echo "deb-src [signed-by=${kr_path}] ${repo_data}" >> "${repo_path}"
-  fi
-
-  apt-get update -qq
-}
-
-#
-# Generate repo file under /etc/yum.repos.d/
-#
-function dnf_add_repo() {
-  local -r repo_name="$1"
-  local -r repo_url="$3" # "http(s)://host/path/filename.repo"
-  local -r kr_path="${5:-/etc/pki/rpm-gpg/${repo_name}.gpg}"
-  local -r repo_path="${6:-/etc/yum.repos.d/${repo_name}.repo}"
-
-  curl -s -L "${repo_url}" \
-    | dd of="${repo_path}" status=progress
-#    | perl -p -e "s{^gpgkey=.*$}{gpgkey=file://${kr_path}}" \
-}
-
-#
-# Keyrings default to
-# /usr/share/keyrings/${repo_name}.gpg (debian/ubuntu) or
-# /etc/pki/rpm-gpg/${repo_name}.gpg    (rocky/RHEL)
-#
-function os_add_repo() {
-  local -r repo_name="$1"
-  local -r signing_key_url="$2"
-  local -r repo_data="$3" # "http(s)://host/path/uri argument0 .. argumentN"
-  local kr_path
-  if is_debuntu ; then kr_path="${5:-/usr/share/keyrings/${repo_name}.gpg}"
-                  else kr_path="${5:-/etc/pki/rpm-gpg/${repo_name}.gpg}" ; fi
-
-  mkdir -p "$(dirname "${kr_path}")"
-
-  curl -fsS --retry-connrefused --retry 10 --retry-max-time 30 "${signing_key_url}" \
-    | gpg --import --no-default-keyring --keyring "${kr_path}"
-
-  if is_debuntu ; then apt_add_repo "${repo_name}" "${signing_key_url}" "${repo_data}" "${4:-yes}" "${kr_path}" "${6:-}"
-                  else dnf_add_repo "${repo_name}" "${signing_key_url}" "${repo_data}" "${4:-yes}" "${kr_path}" "${6:-}" ; fi
-}
-
 function configure_dkms_certs() {
   if test -v PSN && [[ -z "${PSN}" ]]; then
       echo "No signing secret provided.  skipping";
diff --git a/templates/dask/util_functions b/templates/dask/util_functions
index d67da1fc1..ce6964e94 100644
--- a/templates/dask/util_functions
+++ b/templates/dask/util_functions
@@ -510,8 +510,8 @@ function install_conda_packages() {
     if [[ "$retval" == "0" ]] ; then
       is_installed="1"
       pushd "${DASK_CONDA_ENV}"
-      time (
-        tar czf "${local_tarball}" .
+      time ( set -e
+        tar czf "${local_tarball}" . && tar tzf "${local_tarball}"
         gcloud storage cp "${local_tarball}" "${gcs_tarball}"
         rm "${local_tarball}"
       )
diff --git a/templates/generate-action.pl b/templates/generate-action.pl
index 7cc954a67..950bd15fe 100644
--- a/templates/generate-action.pl
+++ b/templates/generate-action.pl
@@ -10,7 +10,19 @@
 my $action = $ARGV[0];
 my $v = { template_path => "${action}.in" };
 
-sub usage{ die "Usage: $0 <action>" }
+sub usage{
+  # TODO: use File::Find to list the available actions for the user
+  my $message = <<EOF;
+This script evaluates a template to generate an initialization action.
+The output is printed to STDOUT.
+
+Action templates reside under templates/$action and end in .sh.in
+
+The <action> argument is the destination action name, not the source.
+EOF
+  print STDERR $message;
+  die "Usage:$/$0 <action>"
+}
 
 usage unless( $action && -f "$ENV{PWD}/templates/$v->{template_path}" );
 
diff --git a/templates/gpu/install_functions b/templates/gpu/install_functions
index 2ea8ca4d2..f6aa9fcf9 100644
--- a/templates/gpu/install_functions
+++ b/templates/gpu/install_functions
@@ -798,7 +798,7 @@ function download_gpu_monitoring_agent(){
 
 function install_gpu_monitoring_agent_dependency(){
   cd /opt/google/compute-gpu-monitoring/linux
-  /usr/bin/python3 -m venv venv
+  /opt/conda/miniconda3/bin/python3 -m venv venv
   (
     source venv/bin/activate
     pip install wheel
@@ -829,7 +829,7 @@ function install_gpu_agent() {
     | sed -e 's/-u --format=/--format=/' \
     | dd status=none of="${install_dir}/report_gpu_metrics.py"
   local venv="${install_dir}/venv"
-  /usr/bin/python3 -m venv "${venv}"
+  /opt/conda/miniconda3/bin/python3 -m venv "${venv}"
 (
   source "${venv}/bin/activate"
   python3 -m pip install --upgrade pip