diff --git a/ci/build.sh b/ci/build.sh index 201c01982..b6feafec1 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -7,7 +7,18 @@ export BUILD_TYPE=DEBUG export ESCAPETEST=yes export EXPLICIT_ROLE=hub export TEST_MACHINE=chroot -# TODO maybe seed the cache? cp -R buildscripts/ci/cache ~/.cache + +set +x # hide secrets +eval $(ssh-agent -s) +if [ -z "$SECRET" ]; then + echo "Need sftp cache ssh secret key. Provide with SECRET env variable" + exit 1 +else + echo "$SECRET" | ssh-add - +fi +ssh-add -l +set -x # stop hiding secrets + time ./buildscripts/build-scripts/build-environment-check time ./buildscripts/build-scripts/install-dependencies time ./buildscripts/build-scripts/configure # 3 minutes locally diff --git a/ci/clean-build-package.sh b/ci/clean-build-package.sh index 01376fc22..a36691cda 100755 --- a/ci/clean-build-package.sh +++ b/ci/clean-build-package.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash +set -ex # clean up docker stuff name=cfengine-build-package # TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine diff --git a/ci/docker-build-package.sh b/ci/docker-build-package.sh index 2ce5dd148..126d89ff5 100755 --- a/ci/docker-build-package.sh +++ b/ci/docker-build-package.sh @@ -11,55 +11,29 @@ name=cfengine-build-package label=PACKAGES_HUB_x86_64_linux_ubuntu_20 export JOB_BASE_NAME=label=$label -# start ssh-agent for local sftp cache pull -eval "$(ssh-agent -s)" -# add private key -set +x # hide secrets +docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . + +# add secret key to enable push up to sftp cache if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then - echo "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" | ssh-add - + export SECRET="$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" else - if ! pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec | ssh-add -; then - echo "Need the ssh private key for build artifacts cache, neither env var nor mystiko was available." + if ! export SECRET=$(pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec); then + echo "The sftp cache ssh secret key must be provided, either with environment variable GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE or access to mystiko path developers/CFEngine/jenkins/sftp-cache.sec" exit 1 fi fi -set -x # done hiding secrets - - -# todo, check the image against the Dockerfile for up-to-date ness? -docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . -# todo, check if already running and up-to-date? # send in JOB_BASE_NAME to enable use of retrieved or generated deps cache -docker run -d --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name || true +docker run -d --env SECRET --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name # copy local caches to docker container mkdir -p "${NTECH_ROOT}/packages" mkdir -p "${NTECH_ROOT}/cache" -# pre-seed cache from sftp buildcache if possible -# requires either environment var with private key or mystiko+pass -# clean up any lingering revision file previously generated, if you are changing deps locally and iterating this is important -[ -f "${NTECH_ROOT}/buildscripts/deps-packaging/revision" ] && rm "${NTECH_ROOT}/buildscripts/deps-packaging/revision" -cd "${NTECH_ROOT}/buildscripts/deps-packaging" -# see buildscripts/build-scripts/autogen for a similar workaround to ensure it stays 7 on bootstrap-oslo-dc jobs -git config --add core.abbrev 7 # hack to match smaller commit sha on bootstrap-oslo-dc (debian-9) -revision=$(git log --pretty='format:%h' -1 -- .) -cd - # back to previous directory -PKGS_DIR="${NTECH_ROOT}/cache/buildscripts_cache/pkgs/${label}" -mkdir -p "${PKGS_DIR}" - # setup host key trust pubkey="build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y=" -if ! grep "$pubkey" ~/.ssh/known_hosts; then - mkdir -p ~/.ssh - echo "$pubkey" >> ~/.ssh/known_hosts -fi #echo "build-artifacts-cache.cloud.cfengine.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINMJKl282VQSz4EMMypJjATu21A9SxQA1XoTslIOID16" >> ~/.ssh/known_hosts -echo -e "cd /export/sftp_dirs_cache/${label}\n get -Ra *${revision}* ${PKGS_DIR}" | \ - sftp -oPubkeyAcceptedKeyTypes=+ssh-rsa -b - jenkins_sftp_cache@build-artifacts-cache.cloud.cfengine.com || true # allow failure, fallback is github cache or building deps - # ending with /. in srcpath copies contents to destpath docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache @@ -73,13 +47,6 @@ done docker exec -i $name bash -c "mkdir -p ~/.ssh" docker exec -i $name bash -c "echo $pubkey >> ~/.ssh/known_hosts" -# add secret key to enable push up to sftp cache -if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then - echo "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" | docker exec -i $name bash -c 'cat > /sftp-cache.sec' -else - pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec | docker exec -i $name bash -c 'cat > /sftp-cache.sec' -fi -docker exec -i $name bash -c 'chmod 400 /sftp-cache.sec' docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh' docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'