ENT-9170: Added deplyment tests for built enterprise hub package (3.18)

.github/workflows/build-using-buildscripts.yml

@@ -9,6 +9,8 @@ on:
         required: true
       GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO:
         required: true
+      GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE:
+        required: true
 
 jobs:
   build_cfengine_hub_package:
@@ -80,7 +82,7 @@ jobs:
           ssh-known-hosts: github.com
 
       - name: get PACKAGE_SHA for package cache
-        run: echo "PACKAGE_SHA=$(mission-portal/ci/package-sha.sh)" | tee -a ${GITHUB_ENV}
+        run: echo "PACKAGE_SHA=$(buildscripts/ci/package-sha.sh)" | tee -a ${GITHUB_ENV}
 
       - name: get SHA of buildscripts/deps-packaging last commit
         run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV}
@@ -105,7 +107,12 @@ jobs:
             deps
 
       - name: Build package in docker
-        run: test ! -f packages/cfe*deb && buildscripts/ci/docker.sh || true
+        env:
+            GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }}
+        run: |
+            if [ ! -f packages/cfe*deb ]; then
+                buildscripts/ci/docker-build-package.sh
+            fi
 
       - name: Save dependency cache
         uses: actions/cache/save@v3
@@ -118,3 +125,12 @@ jobs:
         with:
           path: packages
           key: packages-${{ env.PACKAGE_SHA }}
+
+      - name: Save artifacts
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifacts
+          path: |
+            artifacts
+            packages

.github/workflows/ci.yml

@@ -10,3 +10,8 @@ jobs:
   build_cfengine_hub_package:
     uses: ./.github/workflows/build-using-buildscripts.yml
     secrets: inherit
+
+  deployment_tests:
+    needs: build_cfengine_hub_package
+    uses: ./.github/workflows/deployment-tests.yml
+    secrets: inherit

.github/workflows/deployment-tests.yml

@@ -0,0 +1,139 @@
+name: Deployment tests for built package
+
+on:
+  workflow_call:
+    secrets:
+      GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO:
+        required: true
+      GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO:
+        required: true
+      GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO:
+        required: true
+      GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE:
+        required: true
+
+jobs:
+  deployment_tests:
+    name: Run simple deployment tests
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout Together Action
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/together-javascript-action
+          ref: main
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_TOGETHER_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Action step
+        uses: ./
+        id: together
+        with:
+          myToken: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout Core
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/core
+          path: core
+          ref: ${{steps.together.outputs.core || github.base_ref}}
+          submodules: recursive
+
+      - name: Checkout Masterfiles
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/masterfiles
+          path: masterfiles
+          ref: ${{steps.together.outputs.masterfiles || github.base_ref}}
+
+      - name: Checkout Buildscripts (current project)
+        uses: actions/checkout@v3
+        with:
+          path: buildscripts
+          fetch-depth: 20
+
+      - name: Checkout Nova
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/nova
+          path: nova
+          ref: ${{steps.together.outputs.nova || github.base_ref}}
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Checkout Enterprise
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/enterprise
+          path: enterprise
+          ref: ${{steps.together.outputs.enterprise || github.base_ref}}
+          submodules: recursive
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Checkout Mission Portal
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/mission-portal
+          path: mission-portal
+          ref: ${{steps.together.outputs.mission-portal || github.base_ref}}
+          submodules: recursive
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: get PACKAGE_SHA for package cache
+        run: echo "PACKAGE_SHA=$(buildscripts/ci/package-sha.sh)" | tee -a ${GITHUB_ENV}
+
+      - name: get SHA of buildscripts/deps-packaging last commit
+        run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV}
+        working-directory: buildscripts/deps-packaging
+
+      - name: restore packages cache
+        uses: actions/cache/restore@v3
+        with:
+          path: packages
+          key: packages-${{ env.PACKAGE_SHA }}
+          restore-keys: |
+            packages-${{ env.PACKAGE_SHA }}
+
+      - name: Restore dependency cache
+        uses: actions/cache/restore@v3
+        with:
+          path: cache
+          key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
+          restore-keys: |
+            deps-${{ github.base_ref }}
+            deps-master
+            deps
+
+      - name: Build package in docker
+        env:
+            GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }}
+        run: |
+            if [ ! -f packages/cfe*deb ]; then
+                buildscripts/ci/docker-build-package.sh
+            fi
+
+      - name: Run deployment tests
+        run: buildscripts/ci/docker-deployment-tests.sh
+
+      - name: Save dependency cache
+        uses: actions/cache/save@v3
+        with:
+          path: cache
+          key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
+
+      - name: Save packages cache
+        uses: actions/cache/save@v3
+        with:
+          path: packages
+          key: packages-${{ env.PACKAGE_SHA }}
+
+      - name: Save artifacts
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifacts
+          path: |
+            artifacts
+            packages

build-scripts/autogen

@@ -28,12 +28,27 @@ for p in $projects
 do
     (cd $BASEDIR/$p && NO_CONFIGURE=1 ./autogen.sh) || false
 done
+# %h (abbreviated commit hash) is not deterministic for length on different systems
+# so far (up to Aug 2023) this didn't matter because one system (bootstrap-oslo-dc)
+# was responsible for doing this autogen work and all other systems used the result.
+# When we migrated from travis to github actions we needed things to be stable between
+# bootstrap-oslo-dc and other systems so will force a length of 7 and check that
+# the result is unique.
+export CORE_ABBREV=7 # adjust this up if need be
+git config --global --add core.abbrev $CORE_ABBREV
 
 for i in $GITSHAOF
 do
     if [ -d $BASEDIR/$i ] && [ ! -f $BASEDIR/$i/revision ]
     then
         R=$(cd $BASEDIR/$i && git log --pretty='format:%h' -1 -- .) || false
+        (
+            cd $BASEDIR/$i
+            if ! git show $R --oneline >/dev/null; then
+                echo "abbreviated commit hash of $CORE_ABBREV is not unique. Consider increasing the value in the script $0."
+                exit 1
+            fi
+        )
         echo $R | tr -d '\n' > $BASEDIR/$i/revision
     fi
 done

ci/Dockerfile-cfengine-deployment-tests

@@ -0,0 +1,3 @@
+FROM ubuntu:20.04
+RUN apt-get update -y && apt-get install -y systemd sudo
+CMD [ "/lib/systemd/systemd" ]

ci/clean-deployment-tests.sh

@@ -0,0 +1,6 @@
+# clean up docker stuff
+name=cfengine-deployment-tests
+# TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine
+docker stop $name
+docker rm $name
+docker rmi $name

ci/deployment-tests.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2024
+# I am redirecting many sudo run commands to logfiles which can be owned by the non-priv user
+# copied from mission-portal/ci/run.sh for selenium tests
+# todo refactor to share some of this instead of copy/pasting
+set -ex
+
+# find the dir one level up from here, home of all the repositories
+COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")"
+# NTECH_ROOT should be the same, but if available use it so user can do their own thing.
+NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+USER=${USER:-$(whoami)}
+
+if [ ! -d /var/cfengine ]; then
+  # ci and local buildscripts should place built packages in $NTECH_ROOT/packages
+  sudo dpkg -i "$NTECH_ROOT"/packages/cfengine-nova-hub*deb
+fi
+
+# now that cfengine is probably installed, run cf-support if there is an error
+trap failure ERR
+
+function failure() {
+  sudo mkdir -p "${NTECH_ROOT}/artifacts"
+  sudo chown "$USER" "${NTECH_ROOT}/artifacts"
+  cd "${NTECH_ROOT}/artifacts"
+  sudo cf-support --yes > $$.cfsupportlog 2>&1 || cat $$.cfsupportlog
+  rm $$.cfsupportlog
+}
+
+AGENT_LOG="${NTECH_ROOT}/artifacts/agent.log"
+if [ -f "$AGENT_LOG" ]; then
+  mv "$AGENT_LOG" "${AGENT_LOG}.$(date +%s)"
+fi
+mkdir -p "${NTECH_ROOT}/artifacts"
+touch "$AGENT_LOG"
+if [ ! -f /var/cfengine/policy_server.dat ]; then
+  sudo /var/cfengine/bin/cf-agent -B "$(hostname -I | awk ' {print $1}')" >>"$AGENT_LOG" 2>&1
+fi
+
+# make artifacts directory to be slurped by CI (jenkins, github, ...)
+mkdir -p "${NTECH_ROOT}/artifacts"
+
+{
+  sudo /var/cfengine/bin/cf-agent -KIf update.cf
+  sudo /var/cfengine/bin/cf-agent -KI
+  sudo /var/cfengine/bin/cf-agent -KI
+} >>"$AGENT_LOG" 2>&1
+
+if grep -i error "$AGENT_LOG" >/dev/null; then
+  echo "FAIL test, errors in $AGENT_LOG"
+  grep -i error "$AGENT_LOG"
+fi
+
+apt-get -y install python3-psycopg2
+export REPORTING_TEST_DELAY=5
+cd "${NTECH_ROOT}/nova/tests/reporting"
+python3 deployment_test.py

ci/docker-build-package.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# run the build in a docker container
+set -ex
+
+# find the dir two levels up from here, home of all the repositories
+COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")"
+# NTECH_ROOT should be the same, but if available use it so user can do their own thing.
+NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+
+name=cfengine-build-package
+label=PACKAGES_HUB_x86_64_linux_ubuntu_20
+export JOB_BASE_NAME=label=$label
+# todo, check the image against the Dockerfile for up-to-date ness?
+docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true
+# todo, check if already running and up-to-date?
+# send in JOB_BASE_NAME to enable use of retrieved or generated deps cache
+docker run -d --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name || true
+
+# copy local caches to docker container
+mkdir -p "${NTECH_ROOT}/packages"
+mkdir -p "${NTECH_ROOT}/cache"
+
+# pre-seed cache from sftp buildcache if possible
+# requires either environment var with private key or mystiko+pass
+eval "$(ssh-agent -s)"
+set +x # hide secrets
+if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then
+  echo "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" | ssh-add -
+else
+  if ! pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec | ssh-add -; then
+    echo "Need the ssh private key for build artifacts cache, neither env var nor mystiko was available."
+    exit 1
+  fi
+fi
+set -x # done hiding secrets
+# clean up any lingering revision file previously generated, if you are changing deps locally and iterating this is important
+[ -f "${NTECH_ROOT}/buildscripts/deps-packaging/revision" ] && rm "${NTECH_ROOT}/buildscripts/deps-packaging/revision"
+cd "${NTECH_ROOT}/buildscripts/deps-packaging"
+# see buildscripts/build-scripts/autogen for a similar workaround to ensure it stays 7 on bootstrap-oslo-dc jobs
+git config --add core.abbrev 7 # hack to match smaller commit sha on bootstrap-oslo-dc (debian-9)
+revision=$(git log --pretty='format:%h' -1 -- .)
+cd - # back to previous directory
+PKGS_DIR="${NTECH_ROOT}/cache/buildscripts_cache/pkgs/${label}"
+mkdir -p "${PKGS_DIR}"
+
+# setup host key trust
+echo "build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y=" >>~/.ssh/known_hosts
+
+echo -e "cd /export/sftp_dirs_cache/${label}\n get -Ra *${revision}* ${PKGS_DIR}" | \
+ sftp -oPubkeyAcceptedKeyTypes=+ssh-rsa -b - [email protected]
+
+# ending with /. in srcpath copies contents to destpath
+docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache
+
+# in order for build-scripts/autogen to generate a revision file:
+for i in core buildscripts buildscripts/deps-packaging enterprise nova masterfiles
+do
+  docker exec -i $name bash -c "git config --global --add safe.directory /data/$i"
+done
+
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh'
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'
+
+# save back cache and packages to host for handling by CI and such
+docker cp $name:/root/.cache/. "${NTECH_ROOT}/cache/"
+docker cp $name:/data/packages/. "${NTECH_ROOT}/packages/"
+
+rc=1 # if we find no packages, fail
+for f in packages/*.deb; do
+  [ -f "$f" ] && rc=0
+  break
+done
+exit $rc

ci/docker-deployment-tests.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# run the build in a docker container
+set -ex
+
+# find the dir two levels up from here, home of all the repositories
+COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")"
+# NTECH_ROOT should be the same, but if available use it so user can do their own thing.
+NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+
+name=cfengine-deployment-tests
+# todo, check the image against the Dockerfile for up-to-date ness?
+if ! docker images | grep $name; then
+  docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true
+fi
+
+# todo, check if already running and up-to-date?
+# we want a fresh container, stop and remove any that exist by this $name
+if docker ps -a | grep $name; then
+  docker ps -a | grep $name | awk '{print $1}' | xargs docker stop
+  docker ps -a | grep $name | awk '{print $1}' | xargs docker rm
+fi
+docker run -d --privileged -v "${NTECH_ROOT}":/data --name $name $name || true
+
+if [ ! -d "${NTECH_ROOT}/packages" ]; then
+  echo "${NTECH_ROOT}/packages directory should exist and have a cfengine-nova-hub package there"
+  exit 1
+fi
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/deployment-tests.sh'