Added deplyment tests for built enterprise hub package

Can be run in docker or on a plain host (vm/metal)

Also added step in docker build to get sftp cached deps packages

Requires either GH env var ssh private key or mystiko (password-store repo)

Ticket: ENT-9170
Changelog: none

.github/workflows/build-using-buildscripts.yml

@@ -9,6 +9,8 @@ on:
         required: true
       GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO:
         required: true
+      GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE:
+        required: true
 
 jobs:
   build_cfengine_hub_package:
@@ -105,7 +107,12 @@ jobs:
             deps
 
       - name: Build package in docker
-        run: test ! -f packages/cfe*deb && buildscripts/ci/docker.sh || true
+        env:
+            GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }}
+        run: |
+            if ! -f packages/cfe*deb; then
+                buildscripts/ci/docker-build-package.sh
+            fi
 
       - name: Save dependency cache
         uses: actions/cache/save@v3

.github/workflows/ci.yml

@@ -10,3 +10,8 @@ jobs:
   build_cfengine_hub_package:
     uses: ./.github/workflows/build-using-buildscripts.yml
     secrets: inherit
+
+  deployment_tests:
+    needs: build_cfengine_hub_package
+    uses: ./.github/workflows/deployment-tests.yml
+    secrets: inherit

.github/workflows/deployment-tests.yml

@@ -0,0 +1,130 @@
+name: Deployment tests for built package
+
+on:
+  workflow_call:
+    secrets:
+      GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO:
+        required: true
+      GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO:
+        required: true
+      GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO:
+        required: true
+      GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE:
+        required: true
+
+jobs:
+  deployment_tests:
+    name: Run simple deployment tests
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout Together Action
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/together-javascript-action
+          ref: main
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_TOGETHER_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Action step
+        uses: ./
+        id: together
+        with:
+          myToken: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout Core
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/core
+          path: core
+          ref: ${{steps.together.outputs.core || github.base_ref}}
+          submodules: recursive
+
+      - name: Checkout Masterfiles
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/masterfiles
+          path: masterfiles
+          ref: ${{steps.together.outputs.masterfiles || github.base_ref}}
+
+      - name: Checkout Buildscripts (current project)
+        uses: actions/checkout@v3
+        with:
+          path: buildscripts
+          fetch-depth: 20
+
+      - name: Checkout Nova
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/nova
+          path: nova
+          ref: ${{steps.together.outputs.nova || github.base_ref}}
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Checkout Enterprise
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/enterprise
+          path: enterprise
+          ref: ${{steps.together.outputs.enterprise || github.base_ref}}
+          submodules: recursive
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Checkout Mission Portal
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/mission-portal
+          path: mission-portal
+          ref: ${{steps.together.outputs.mission-portal || github.base_ref}}
+          submodules: recursive
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: get PACKAGE_SHA for package cache
+        run: echo "PACKAGE_SHA=$(mission-portal/ci/package-sha.sh)" | tee -a ${GITHUB_ENV}
+
+      - name: get SHA of buildscripts/deps-packaging last commit
+        run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV}
+        working-directory: buildscripts/deps-packaging
+
+      - name: restore packages cache
+        uses: actions/cache/restore@v3
+        with:
+          path: packages
+          key: packages-${{ env.PACKAGE_SHA }}
+          restore-keys: |
+            packages-${{ env.PACKAGE_SHA }}
+
+      - name: Restore dependency cache
+        uses: actions/cache/restore@v3
+        with:
+          path: cache
+          key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
+          restore-keys: |
+            deps-${{ github.base_ref }}
+            deps-master
+            deps
+
+      - name: Build package in docker
+        env:
+            GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }}
+        run: |
+            if ! -f packages/cfe*deb; then
+                buildscripts/ci/docker-build-package.sh
+            fi
+
+      - name: Run deployment tests
+        run: buildscripts/ci/docker-deployment-tests.sh
+
+      - name: Save dependency cache
+        uses: actions/cache/save@v3
+        with:
+          path: cache
+          key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
+
+      - name: Save packages cache
+        uses: actions/cache/save@v3
+        with:
+          path: packages
+          key: packages-${{ env.PACKAGE_SHA }}

ci/Dockerfile-cfengine-deployment-tests

@@ -0,0 +1,3 @@
+FROM ubuntu:20.04
+RUN apt-get update -y && apt-get install -y systemd sudo
+CMD [ "/lib/systemd/systemd" ]

ci/clean-deployment-tests.sh

@@ -0,0 +1,6 @@
+# clean up docker stuff
+name=cfengine-deployment-tests
+# TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine
+docker stop $name
+docker rm $name
+docker rmi $name

ci/deployment-tests.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# copied from mission-portal/ci/run.sh for selenium tests
+# todo refactor to share some of this instead of copy/pasting
+set -ex
+
+# find the dir one level up from here, home of all the repositories
+COMPUTED_ROOT=$(readlink -e $(dirname "$0")/../../)
+# NTECH_ROOT should be the same, but if available use it so user can do their own thing.
+NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+USER=${USER:-$(whoami)}
+
+if [ ! -d /var/cfengine ]; then
+  # ci and local buildscripts should place built packages in $NTECH_ROOT/packages
+  sudo dpkg -i "$NTECH_ROOT"/packages/cfengine-nova-hub*deb
+fi
+
+# now that cfengine is probably installed, run cf-support if there is an error
+trap failure ERR
+
+function failure() {
+  sudo mkdir -p "${NTECH_ROOT}/artifacts"
+  sudo chown $USER "${NTECH_ROOT}/artifacts"
+  cd "${NTECH_ROOT}/artifacts"
+  sudo cf-support --yes 2>&1 > $$.cfsupportlog || cat $$.cfsupportlog
+  rm $$.cfsupportlog
+}
+
+AGENT_LOG="${NTECH_ROOT}/artifacts/agent.log"
+if [ -f "$AGENT_LOG" ]; then
+  mv "$AGENT_LOG" "$AGENT_LOG".$(date +%s)
+fi
+touch "$AGENT_LOG"
+if [ ! -f /var/cfengine/policy_server.dat ]; then
+  sudo /var/cfengine/bin/cf-agent -B $(hostname -I | awk ' {print $1}') >>"$AGENT_LOG"
+fi
+
+# make artifacts directory to be slurped by CI (jenkins, github, ...)
+mkdir -p "${NTECH_ROOT}/artifacts"
+
+sudo /var/cfengine/bin/cf-agent -KIf update.cf 2>&1 >>"$AGENT_LOG"
+sudo /var/cfengine/bin/cf-agent -KI 2>&1 >>"$AGENT_LOG"
+sudo /var/cfengine/bin/cf-agent -KI 2>&1 >>"$AGENT_LOG"
+
+if grep -i error "$AGENT_LOG" >/dev/null; then
+  echo "FAIL test, errors in $AGENT_LOG"
+  grep -i error "$AGENT_LOG"
+fi
+
+apt-get -y install python3-psycopg2
+export REPORTING_TEST_DELAY=5
+cd "${NTECH_ROOT}/nova/tests/reporting"
+python3 deployment_test.py

ci/docker-build-package.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# run the build in a docker container
+set -ex
+
+# find the dir two levels up from here, home of all the repositories
+COMPUTED_ROOT=$(readlink -e $(dirname "$0")/../../)
+# NTECH_ROOT should be the same, but if available use it so user can do their own thing.
+NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+
+name=cfengine-build-package
+label=PACKAGES_HUB_x86_64_linux_ubuntu_20
+export JOB_BASE_NAME=label=$label
+# todo, check the image against the Dockerfile for up-to-date ness?
+docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true
+# todo, check if already running and up-to-date?
+# send in JOB_BASE_NAME to enable use of retrieved or generated deps cache
+docker run -d --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name || true
+
+# copy local caches to docker container
+mkdir -p "${NTECH_ROOT}/packages"
+mkdir -p "${NTECH_ROOT}/cache"
+
+# pre-seed cache from sftp buildcache if possible
+# requires either environment var with private key or mystiko+pass
+eval $(ssh-agent -s)
+set +x # hide secrets
+if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then
+  echo "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" | ssh-add -
+else
+  if ! pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec | ssh-add -; then
+    echo "Need the ssh private key for build artifacts cache, neither env var nor mystiko was available."
+    exit 1
+  fi
+fi
+set -x # done hiding secrets
+# clean up any lingering revision file previously generated, if you are changing deps locally and iterating this is important
+[ -f "${NTECH_ROOT}/buildscripts/deps-packaging/revision" ] && rm "${NTECH_ROOT}/buildscripts/deps-packaging/revision"
+cd "${NTECH_ROOT}/buildscripts/deps-packaging"
+# see buildscripts/build-scripts/autogen for a similar workaround to ensure it stays 7 on bootstrap-oslo-dc jobs
+git config --add core.abbrev 7 # hack to match smaller commit sha on bootstrap-oslo-dc (debian-9)
+revision=$(git log --pretty='format:%h' -1 -- .)
+cd - # back to previous directory
+PKGS_DIR="${NTECH_ROOT}/cache/buildscripts_cache/pkgs/${label}"
+mkdir -p "${PKGS_DIR}"
+
+# setup host key trust
+echo "build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y=" >>~/.ssh/known_hosts
+
+echo -e "cd /export/sftp_dirs_cache/${label}\n get -Ra *${revision}* ${PKGS_DIR}" | \
+ sftp -oPubkeyAcceptedKeyTypes=+ssh-rsa -b - [email protected]
+
+# ending with /. in srcpath copies contents to destpath
+docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache
+
+# in order for build-scripts/autogen to generate a revision file:
+for i in core buildscripts buildscripts/deps-packaging enterprise nova masterfiles
+do
+  docker exec -i $name bash -c "git config --global --add safe.directory /data/$i"
+done
+
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh'
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'
+
+# save back cache and packages to host for handling by CI and such
+docker cp $name:/root/.cache/. "${NTECH_ROOT}/cache/"
+docker cp $name:/data/packages/. "${NTECH_ROOT}/packages/"
+
+# if no packages, then fail
+[ -f packages/*.deb ] || [ -f packages/*.rpm ]

ci/docker-deployment-tests.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# run the build in a docker container
+set -ex
+
+# find the dir two levels up from here, home of all the repositories
+COMPUTED_ROOT=$(readlink -e $(dirname "$0")/../../)
+# NTECH_ROOT should be the same, but if available use it so user can do their own thing.
+NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+
+name=cfengine-deployment-tests
+# todo, check the image against the Dockerfile for up-to-date ness?
+if ! docker images | grep $name; then
+  docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true
+fi
+
+# todo, check if already running and up-to-date?
+# we want a fresh container, stop and remove any that exist by this $name
+if docker ps -a | grep $name; then
+  docker ps -a | grep $name | awk '{print $1}' | xargs docker stop
+  docker ps -a | grep $name | awk '{print $1}' | xargs docker rm
+fi
+docker run -d --privileged -v "${NTECH_ROOT}":/data --name $name $name || true
+
+if [ ! -d "${NTECH_ROOT}/packages" ]; then
+  echo "${NTECH_ROOT}/packages directory should exist and have a cfengine-nova-hub package there"
+  exit 1
+fi
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/deployment-tests.sh'

ci/docker.sh

@@ -0,0 +1 @@
+docker-build-package.sh