refine sftp cache bits

ci/build.sh

@@ -7,7 +7,18 @@ export BUILD_TYPE=DEBUG
 export ESCAPETEST=yes
 export EXPLICIT_ROLE=hub
 export TEST_MACHINE=chroot
-# TODO maybe seed the cache? cp -R buildscripts/ci/cache ~/.cache
+
+set +x # hide secrets
+eval $(ssh-agent -s)
+if [ -z "$SECRET" ]; then
+  echo "Need sftp cache ssh secret key. Provide with SECRET env variable"
+  exit 1
+else
+  echo "$SECRET" | ssh-add -
+fi
+ssh-add -l
+set -x # stop hiding secrets
+
 time ./buildscripts/build-scripts/build-environment-check
 time ./buildscripts/build-scripts/install-dependencies
 time ./buildscripts/build-scripts/configure # 3 minutes locally

ci/clean-build-package.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+set -ex
 # clean up docker stuff
 name=cfengine-build-package
 # TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine

ci/docker-build-package.sh

@@ -11,55 +11,29 @@ name=cfengine-build-package
 label=PACKAGES_HUB_x86_64_linux_ubuntu_20
 export JOB_BASE_NAME=label=$label
 
-# start ssh-agent for local sftp cache pull
-eval "$(ssh-agent -s)"
 
-# add private key
-set +x # hide secrets
+docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" .
+
+# add secret key to enable push up to sftp cache
 if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then
-  echo "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" | ssh-add -
+  export SECRET="$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE"
 else
-  if ! pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec  | ssh-add -; then
-    echo "Need the ssh private key for build artifacts cache, neither env var nor mystiko was available."
+  if ! export SECRET=$(pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec); then
+    echo "The sftp cache ssh secret key must be provided, either with environment variable GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE or access to mystiko path developers/CFEngine/jenkins/sftp-cache.sec"
     exit 1
   fi
 fi
-set -x # done hiding secrets
-
-
-# todo, check the image against the Dockerfile for up-to-date ness?
-docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" .
-# todo, check if already running and up-to-date?
 # send in JOB_BASE_NAME to enable use of retrieved or generated deps cache
-docker run -d --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name || true
+docker run -d --env SECRET --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name
 
 # copy local caches to docker container
 mkdir -p "${NTECH_ROOT}/packages"
 mkdir -p "${NTECH_ROOT}/cache"
 
-# pre-seed cache from sftp buildcache if possible
-# requires either environment var with private key or mystiko+pass
-# clean up any lingering revision file previously generated, if you are changing deps locally and iterating this is important
-[ -f "${NTECH_ROOT}/buildscripts/deps-packaging/revision" ] && rm "${NTECH_ROOT}/buildscripts/deps-packaging/revision"
-cd "${NTECH_ROOT}/buildscripts/deps-packaging"
-# see buildscripts/build-scripts/autogen for a similar workaround to ensure it stays 7 on bootstrap-oslo-dc jobs
-git config --add core.abbrev 7 # hack to match smaller commit sha on bootstrap-oslo-dc (debian-9)
-revision=$(git log --pretty='format:%h' -1 -- .)
-cd - # back to previous directory
-PKGS_DIR="${NTECH_ROOT}/cache/buildscripts_cache/pkgs/${label}"
-mkdir -p "${PKGS_DIR}"
-
 # setup host key trust
 pubkey="build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y="
-if ! grep "$pubkey" ~/.ssh/known_hosts; then
-  mkdir -p ~/.ssh
-  echo "$pubkey" >> ~/.ssh/known_hosts
-fi
 #echo "build-artifacts-cache.cloud.cfengine.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINMJKl282VQSz4EMMypJjATu21A9SxQA1XoTslIOID16" >> ~/.ssh/known_hosts
 
-echo -e "cd /export/sftp_dirs_cache/${label}\n get -Ra *${revision}* ${PKGS_DIR}" | \
- sftp -oPubkeyAcceptedKeyTypes=+ssh-rsa -b - [email protected] || true # allow failure, fallback is github cache or building deps
-
 # ending with /. in srcpath copies contents to destpath
 docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache
 
@@ -73,13 +47,6 @@ done
 docker exec -i $name bash -c "mkdir -p ~/.ssh"
 docker exec -i $name bash -c "echo $pubkey >> ~/.ssh/known_hosts"
 
-# add secret key to enable push up to sftp cache
-if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then
-  echo "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" |  docker exec -i $name bash -c 'cat > /sftp-cache.sec'
-else
-  pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec  | docker exec -i $name bash -c 'cat > /sftp-cache.sec'
-fi
-docker exec -i $name bash -c 'chmod 400 /sftp-cache.sec'
 docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh'
 docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'