Refactored sftp cache ssh key usage into the docker container for ful…

…l pull/push usage

Before I was pulling the sftp cache on the host outside the container because
I couldn't figure out how to push the sftp cache private key inside the container.

When I changed a dependency, zlib in this case, the build scripts wanted to push
the new packages up to the sftp cache which failed due to lack of private key in the container.

This commit takes either a github secret or mystiko password and sets it as an environment variable inside the container for the build to use.

Ticket: ENT-10419
Changelog: none
(cherry picked from commit b989b9d)

ci/build.sh

@@ -7,7 +7,18 @@ export BUILD_TYPE=DEBUG
 export ESCAPETEST=yes
 export EXPLICIT_ROLE=hub
 export TEST_MACHINE=chroot
-# TODO maybe seed the cache? cp -R buildscripts/ci/cache ~/.cache
+
+set +x # hide secrets
+eval $(ssh-agent -s)
+if [ -z "$SECRET" ]; then
+  echo "Need sftp cache ssh secret key. Provide with SECRET env variable"
+  exit 1
+else
+  echo "$SECRET" | ssh-add -
+fi
+ssh-add -l
+set -x # stop hiding secrets
+
 time ./buildscripts/build-scripts/build-environment-check
 time ./buildscripts/build-scripts/install-dependencies
 time ./buildscripts/build-scripts/configure # 3 minutes locally

ci/clean-build-package.sh

@@ -1,6 +1,8 @@
+#!/usr/bin/env bash
+set -ex
 # clean up docker stuff
 name=cfengine-build-package
 # TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine
 docker stop $name
-docker rm $name
-docker rmi $name
+docker rm -f $name
+docker rmi -f $name

ci/docker-build-package.sh

@@ -10,44 +10,30 @@ NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
 name=cfengine-build-package
 label=PACKAGES_HUB_x86_64_linux_ubuntu_20
 export JOB_BASE_NAME=label=$label
-# todo, check the image against the Dockerfile for up-to-date ness?
-docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true
-# todo, check if already running and up-to-date?
-# send in JOB_BASE_NAME to enable use of retrieved or generated deps cache
-docker run -d --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name || true
 
-# copy local caches to docker container
-mkdir -p "${NTECH_ROOT}/packages"
-mkdir -p "${NTECH_ROOT}/cache"
 
-# pre-seed cache from sftp buildcache if possible
-# requires either environment var with private key or mystiko+pass
-eval "$(ssh-agent -s)"
+docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" .
+
+# add secret key to enable push up to sftp cache
 set +x # hide secrets
 if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then
-  echo "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" | ssh-add -
+  export SECRET="$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE"
 else
-  if ! pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec | ssh-add -; then
-    echo "Need the ssh private key for build artifacts cache, neither env var nor mystiko was available."
+  if ! export SECRET="$(pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec)"; then
+    echo "The sftp cache ssh secret key must be provided, either with environment variable GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE or access to mystiko path developers/CFEngine/jenkins/sftp-cache.sec"
     exit 1
   fi
 fi
 set -x # done hiding secrets
-# clean up any lingering revision file previously generated, if you are changing deps locally and iterating this is important
-[ -f "${NTECH_ROOT}/buildscripts/deps-packaging/revision" ] && rm "${NTECH_ROOT}/buildscripts/deps-packaging/revision"
-cd "${NTECH_ROOT}/buildscripts/deps-packaging"
-# see buildscripts/build-scripts/autogen for a similar workaround to ensure it stays 7 on bootstrap-oslo-dc jobs
-git config --add core.abbrev 7 # hack to match smaller commit sha on bootstrap-oslo-dc (debian-9)
-revision=$(git log --pretty='format:%h' -1 -- .)
-cd - # back to previous directory
-PKGS_DIR="${NTECH_ROOT}/cache/buildscripts_cache/pkgs/${label}"
-mkdir -p "${PKGS_DIR}"
+# send in JOB_BASE_NAME to enable use of retrieved or generated deps cache
+docker run -d --env SECRET --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name
 
-# setup host key trust
-echo "build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y=" >>~/.ssh/known_hosts
+# copy local caches to docker container
+mkdir -p "${NTECH_ROOT}/packages"
+mkdir -p "${NTECH_ROOT}/cache"
 
-echo -e "cd /export/sftp_dirs_cache/${label}\n get -Ra *${revision}* ${PKGS_DIR}" | \
- sftp -oPubkeyAcceptedKeyTypes=+ssh-rsa -b - jenkins_sftp_cache@build-artifacts-cache.cloud.cfengine.com
+# setup host key trust
+pubkey="build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y="
 
 # ending with /. in srcpath copies contents to destpath
 docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache
@@ -58,6 +44,10 @@ do
   docker exec -i $name bash -c "git config --global --add safe.directory /data/$i"
 done
 
+# add build artifacts host public keys to container for use there
+docker exec -i $name bash -c "mkdir -p ~/.ssh"
+docker exec -i $name bash -c "echo $pubkey >> ~/.ssh/known_hosts"
+
 docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh'
 docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'