ENT-9167: Added ci docker scripts for manual and github actions use (3.18)

.github/workflows/build-using-buildscripts.yml

@@ -0,0 +1,120 @@
+name: Build dependencies and CFEngine hub package
+
+on:
+  workflow_call:
+    secrets:
+      GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO:
+        required: true
+      GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO:
+        required: true
+      GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO:
+        required: true
+
+jobs:
+  build_cfengine_hub_package:
+    name: Build package and run selenium tests
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout Together Action
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/together-javascript-action
+          ref: v1.7
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_TOGETHER_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Action step
+        uses: ./
+        id: together
+        with:
+          myToken: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout Core
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/core
+          path: core
+          ref: ${{steps.together.outputs.core || github.base_ref}}
+          submodules: recursive
+
+      - name: Checkout Masterfiles
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/masterfiles
+          path: masterfiles
+          ref: ${{steps.together.outputs.masterfiles || github.base_ref}}
+
+      - name: Checkout Buildscripts (current project)
+        uses: actions/checkout@v3
+        with:
+          path: buildscripts
+          fetch-depth: 20
+
+      - name: Checkout Nova
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/nova
+          path: nova
+          ref: ${{steps.together.outputs.nova || github.base_ref}}
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Checkout Enterprise
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/enterprise
+          path: enterprise
+          ref: ${{steps.together.outputs.enterprise || github.base_ref}}
+          submodules: recursive
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Checkout Mission Portal
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/mission-portal
+          path: mission-portal
+          ref: ${{steps.together.outputs.mission-portal || github.base_ref}}
+          submodules: recursive
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: get PACKAGE_SHA for package cache
+        run: echo "PACKAGE_SHA=$(mission-portal/ci/package-sha.sh)" | tee -a ${GITHUB_ENV}
+
+      - name: get SHA of buildscripts/deps-packaging last commit
+        run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV}
+        working-directory: buildscripts/deps-packaging
+
+      - name: restore packages cache
+        uses: actions/cache/restore@v3
+        with:
+          path: packages
+          key: packages-${{ env.PACKAGE_SHA }}
+          restore-keys: |
+            packages-${{ env.PACKAGE_SHA }}
+
+      - name: Restore dependency cache
+        uses: actions/cache/restore@v3
+        with:
+          path: cache
+          key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
+          restore-keys: |
+            deps-${{ github.base_ref }}
+            deps-master
+            deps
+
+      - name: Build package in docker
+        run: test ! -f packages/cfe*deb && buildscripts/ci/docker.sh || true
+
+      - name: Save dependency cache
+        uses: actions/cache/save@v3
+        with:
+          path: cache
+          key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
+
+      - name: Save packages cache
+        uses: actions/cache/save@v3
+        with:
+          path: packages
+          key: packages-${{ env.PACKAGE_SHA }}

.github/workflows/ci.yml

@@ -0,0 +1,12 @@
+name: Continuous Integration
+
+on:
+  # run this workflow on pull_request activity
+  # this includes opening and pushing more commits
+  pull_request:
+    branches: [ master, 3.21.x, 3.18.x ]
+
+jobs:
+  build_cfengine_hub_package:
+    uses: ./.github/workflows/build-using-buildscripts.yml
+    secrets: inherit

.gitignore

@@ -1,2 +1,5 @@
 workdir*
 revision
+*.html
+ci/cache
+ci/packages

README.md

@@ -1,4 +1,56 @@
-## Prerequisites
+This repository contains the necessary tools to build and test cfengine packages for various platforms.
+
+## Hardware requirements
+
+By experimentation I have found that building hub packages, which includes php dependency requires more than 1.6G of RAM/swap. 2.6G worked for me, less might work as well.
+
+## Example build of Community Agent
+
+A minimal example would be to build packages for cfengine community agent.
+This should be done in an isolated environment such as a dedicated host, virtual machine or linux container.
+
+Install necessary distribution packages. For example on debian/ubuntu:
+
+```
+apt update -y
+apt upgrade -y
+apt install -y git autoconf automake m4 make bison flex binutils libtool gcc g++ libc-dev libpam0g-dev python3 psmisc libtokyocabinet-dev libssl-dev libpcre3-dev default-jre-headless build-essential fakeroot ntp dpkg-dev debhelper pkg-config nfs-common sudo apt-utils wget libncurses5 rsync libexpat1-dev libexpat1 curl
+apt purge -y emacs emacs24 libltdl-dev libltdl7
+```
+
+Get the cfengine source code:
+
+```
+mkdir $HOME/cfengine
+cd $HOME/cfengine
+git clone --recursive --depth 1 https://github.com/cfengine/core
+git clone --depth 1 https://github.com/cfengine/buildscripts
+git clone --depth 1 https://github.com/cfengine/masterfiles
+```
+
+Set some environment variables:
+
+```
+export NO_CONFIGURE=1
+export PROJECT=community
+export BUILD_TYPE=DEBUG
+export EXPLICIT_ROLE=agent
+```
+
+Execute the build steps and see that packages are generated:
+
+```
+./buildscripts/build-scripts/autogen
+./buildscripts/build-scripts/clean-buildmachine
+./buildscripts/build-scripts/build-environment-check
+./buildscripts/build-scripts/install-dependencies
+./buildscripts/build-scripts/configure
+./buildscripts/build-scripts/compile
+./buildscripts/build-scripts/package
+ls -l cfengine-community/*.deb
+```
+
+## General Build Machine Prerequisites
 
 Due to sheer diversity of the environments, build machine is expected to provide
 strict minimum amount of software (don't forget --no-install-recommends on

build-scripts/clean-dependencies

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. `dirname "$0"`/functions
+. detect-environment
+. compile-options
+
+set -ex
+for dep in $DEPS
+do
+    rm -rf $dep
+done

ci/Dockerfile-cfengine-build-package

@@ -0,0 +1,5 @@
+FROM ubuntu:20.04
+RUN apt-get update -y && apt-get install -y systemd wget sudo
+ADD "${NTECH_ROOT}/buildscripts/ci/setup.sh" /
+RUN /bin/bash -c '/setup.sh 2>&1 > setup.log'
+CMD [ "/lib/systemd/systemd" ]

ci/README

@@ -0,0 +1,73 @@
+In this 'buildscripts/ci' directory you will find the means to build cfengine packages.
+Note that you should ensure that the various projects are all on the same version: core, enterprise, masterfiles, nova, mission-portal all should be on say 3.21.x or 3.18.x or master.
+
+Currently a full build with no dependencies cached takes around 53 minutes with deps and packaging.
+
+Two options: containerize build or "normal" machine (such as virtual machine or actual hardware).
+
+The build is designed for ubuntu-20.04 but could be adjusted in various shell scripts for other platforms. (TODO: do this!)
+
+# containerized build
+./clean.sh # cleans any leftover docker bits
+./run.sh # builds a docker image, sets it up, runs it in the background, copies local cache into container, runs the build and saves the results
+./shell.sh # runs bash in the container for debugging
+
+See /data/buildscripts/ci/build.sh for required environment variables and steps to try manually
+
+# virtual or real machine
+
+For virtual machine such as with vagrant, at $NTECH_ROOT (aka top-level directory containing all CFEngine repositories), init an ubuntu-20.04 vagrant machine so it has access to all your repositories.
+
+vagrant init ubuntu/focal64
+vagrant ssh
+
+# note however, that currently you will need to copy /vagrant to a non shared filesystem most likely as vboxsf type doesn't support required hard and soft links for packaging :(
+set -ex
+repos="\
+buildscripts \
+core \
+enterprise \
+masterfiles \
+mission-portal \
+nova \
+"
+
+mkdir -p $HOME/workspace
+pwd
+for repo in $repos; do
+  pwd
+  cp -R "$repo" $HOME/workspace
+done
+
+# always be careful of trailing slashes with rsync, check it again if you are typing!
+rsync -avz /vagrant/cache/ $HOME/.cache/
+
+
+And then do the following as you would on a real machine:
+
+cd $HOME/workspace
+./buildscripts/ci/setup.sh # install needed dependencies/packages/etc
+./buildscripts/ci/setup-projects.sh # items which must be done AFTER the container is created with setup.sh, in non-container case must be executed second
+./buildscripts/ci/build.sh # run the build, should generate a package
+
+don't use ./clean.sh ./run.sh or ./shell.sh those are specifically for containerized/docker build
+
+If you are running selenium tests in mission-portal, you might need to do a `make -C mission-portal clean` before creating packages as the selenium tests copy some things from the distribution into the repository directory that cause problems with packaging, such as the api directory from nova.
+
+# manual debugging on-host
+
+cat buildscripts/ci/build.sh
+# now export the environment variables there (and adjust as needed)
+export BUILD_TYPE=DEBUG
+export ESCAPETEST=yes
+export TEST_MACHINE=chroot
+
+then take each step one-at-a-time or re-run as needed
+
+./buildscripts/build-scripts/install-dependencies
+
+Note that you can provide an argument to install-dependencies to just build ONE dependency, such as
+
+./buildscripts/build-scripts/install-dependencies lmdb
+
+If you change the version of a dependency the cache for that should be skipped and the dep rebuilt from scratch.

ci/build.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# build cfengine hub package
+set -ex
+export PROJECT=nova
+export NO_CONFIGURE=1
+export BUILD_TYPE=DEBUG
+export ESCAPETEST=yes
+export EXPLICIT_ROLE=hub
+export TEST_MACHINE=chroot
+# TODO maybe seed the cache? cp -R buildscripts/ci/cache ~/.cache
+time ./buildscripts/build-scripts/build-environment-check
+time ./buildscripts/build-scripts/install-dependencies
+time ./buildscripts/build-scripts/configure # 3 minutes locally
+time ./buildscripts/build-scripts/generate-source-tarballs # 1m49
+time ./buildscripts/build-scripts/compile
+time sudo apt remove -y 'cfbuild*' || true
+time sudo apt remove -y 'cfengine-*' || true
+time sudo rm -rf /var/cfengine
+time sudo rm -rf /opt/cfengine
+time ./buildscripts/build-scripts/install-dependencies
+time ./buildscripts/build-scripts/package
+sudo mkdir -p packages
+sudo cp cfengine-nova-hub/*.deb packages/ || true
+sudo cp cfengine-nova-hub/*.rpm packages/ || true
+
+# todo maybe save the cache cp -R ~/.cache buildscripts/ci/cache
+
+# clean up
+time sudo apt remove -y 'cfbuild*' || true
+time sudo apt remove -y 'cfengine-*' || true
+time sudo rm -rf /var/cfengine
+time sudo rm -rf /opt/cfengine

ci/clean.sh

@@ -0,0 +1,6 @@
+# clean up docker stuff
+name=cfengine-build-package
+# TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine
+docker stop $name
+docker rm $name
+docker rmi $name

ci/docker.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# run the build in a docker container
+set -ex
+
+# find the dir two levels up from here, home of all the repositories
+COMPUTED_ROOT=$(readlink -e $(dirname "$0")/../../)
+# NTECH_ROOT should be the same, but if available use it so user can do their own thing.
+NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+
+name=cfengine-build-package
+# todo, check the image against the Dockerfile for up-to-date ness?
+docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true
+# todo, check if already running and up-to-date?
+docker run -d --privileged -v ${NTECH_ROOT}:/data --name $name $name || true
+
+# copy local caches to docker container
+mkdir -p "${NTECH_ROOT}/packages"
+mkdir -p "${NTECH_ROOT}/cache"
+# ending with /. in srcpath copies contents to destpath
+docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache
+
+# in order for build-scripts/autogen to generate a revision file:
+for i in core buildscripts buildscripts/deps-packaging enterprise nova masterfiles
+do
+  docker exec -i $name bash -c "git config --global --add safe.directory /data/$i"
+done
+
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh'
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'
+
+# save back cache and packages to host for handling by CI and such
+docker cp $name:/root/.cache/. "${NTECH_ROOT}/cache/"
+docker cp $name:/data/packages/. "${NTECH_ROOT}/packages/"
+
+# if no packages, then fail
+[ -f packages/*.deb ] || [ -f packages/*.rpm ]