Merge pull request #1278 from cfengine/ENT-9157/3.18

ENT-9517: Modified zlib URL to adjust for new release (3.18)

ci/build.sh

@@ -7,7 +7,18 @@ export BUILD_TYPE=DEBUG
 export ESCAPETEST=yes
 export EXPLICIT_ROLE=hub
 export TEST_MACHINE=chroot
-# TODO maybe seed the cache? cp -R buildscripts/ci/cache ~/.cache
+
+set +x # hide secrets
+eval $(ssh-agent -s)
+if [ -z "$SECRET" ]; then
+  echo "Need sftp cache ssh secret key. Provide with SECRET env variable"
+  exit 1
+else
+  echo "$SECRET" | ssh-add -
+fi
+ssh-add -l
+set -x # stop hiding secrets
+
 time ./buildscripts/build-scripts/build-environment-check
 time ./buildscripts/build-scripts/install-dependencies
 time ./buildscripts/build-scripts/configure # 3 minutes locally

ci/clean-build-package.sh

@@ -1,6 +1,8 @@
+#!/usr/bin/env bash
+set -ex
 # clean up docker stuff
 name=cfengine-build-package
 # TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine
 docker stop $name
-docker rm $name
-docker rmi $name
+docker rm -f $name
+docker rmi -f $name

ci/docker-build-package.sh

@@ -10,44 +10,30 @@ NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
 name=cfengine-build-package
 label=PACKAGES_HUB_x86_64_linux_ubuntu_20
 export JOB_BASE_NAME=label=$label
-# todo, check the image against the Dockerfile for up-to-date ness?
-docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true
-# todo, check if already running and up-to-date?
-# send in JOB_BASE_NAME to enable use of retrieved or generated deps cache
-docker run -d --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name || true
 
-# copy local caches to docker container
-mkdir -p "${NTECH_ROOT}/packages"
-mkdir -p "${NTECH_ROOT}/cache"
 
-# pre-seed cache from sftp buildcache if possible
-# requires either environment var with private key or mystiko+pass
-eval "$(ssh-agent -s)"
+docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" .
+
+# add secret key to enable push up to sftp cache
 set +x # hide secrets
 if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then
-  echo "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" | ssh-add -
+  export SECRET="$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE"
 else
-  if ! pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec | ssh-add -; then
-    echo "Need the ssh private key for build artifacts cache, neither env var nor mystiko was available."
+  if ! export SECRET="$(pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec)"; then
+    echo "The sftp cache ssh secret key must be provided, either with environment variable GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE or access to mystiko path developers/CFEngine/jenkins/sftp-cache.sec"
     exit 1
   fi
 fi
 set -x # done hiding secrets
-# clean up any lingering revision file previously generated, if you are changing deps locally and iterating this is important
-[ -f "${NTECH_ROOT}/buildscripts/deps-packaging/revision" ] && rm "${NTECH_ROOT}/buildscripts/deps-packaging/revision"
-cd "${NTECH_ROOT}/buildscripts/deps-packaging"
-# see buildscripts/build-scripts/autogen for a similar workaround to ensure it stays 7 on bootstrap-oslo-dc jobs
-git config --add core.abbrev 7 # hack to match smaller commit sha on bootstrap-oslo-dc (debian-9)
-revision=$(git log --pretty='format:%h' -1 -- .)
-cd - # back to previous directory
-PKGS_DIR="${NTECH_ROOT}/cache/buildscripts_cache/pkgs/${label}"
-mkdir -p "${PKGS_DIR}"
+# send in JOB_BASE_NAME to enable use of retrieved or generated deps cache
+docker run -d --env SECRET --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name
 
-# setup host key trust
-echo "build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y=" >>~/.ssh/known_hosts
+# copy local caches to docker container
+mkdir -p "${NTECH_ROOT}/packages"
+mkdir -p "${NTECH_ROOT}/cache"
 
-echo -e "cd /export/sftp_dirs_cache/${label}\n get -Ra *${revision}* ${PKGS_DIR}" | \
- sftp -oPubkeyAcceptedKeyTypes=+ssh-rsa -b - jenkins_sftp_cache@build-artifacts-cache.cloud.cfengine.com
+# setup host key trust
+pubkey="build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y="
 
 # ending with /. in srcpath copies contents to destpath
 docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache
@@ -58,6 +44,10 @@ do
   docker exec -i $name bash -c "git config --global --add safe.directory /data/$i"
 done
 
+# add build artifacts host public keys to container for use there
+docker exec -i $name bash -c "mkdir -p ~/.ssh"
+docker exec -i $name bash -c "echo $pubkey >> ~/.ssh/known_hosts"
+
 docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh'
 docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'
 

deps-packaging/zlib/source

@@ -1 +1 @@
-http://zlib.net/
+https://zlib.net/fossils/