Merge pull request #1239 from cfengine/ENT-9167/master

ENT-9167: Added ci docker scripts for manual and github actions use

.github/workflows/build-using-buildscripts.yml

@@ -78,3 +78,43 @@ jobs:
           submodules: recursive
           ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO }}
           ssh-known-hosts: github.com
+
+      - name: get PACKAGE_SHA for package cache
+        run: echo "PACKAGE_SHA=$(mission-portal/ci/package-sha.sh)" | tee -a ${GITHUB_ENV}
+
+      - name: get SHA of buildscripts/deps-packaging last commit
+        run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV}
+        working-directory: buildscripts/deps-packaging
+
+      - name: restore packages cache
+        uses: actions/cache/restore@v3
+        with:
+          path: packages
+          key: packages-${{ env.PACKAGE_SHA }}
+          restore-keys: |
+            packages-${{ env.PACKAGE_SHA }}
+
+      - name: Restore dependency cache
+        uses: actions/cache/restore@v3
+        with:
+          path: cache
+          key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
+          restore-keys: |
+            deps-${{ github.base_ref }}
+            deps-master
+            deps
+
+      - name: Build package in docker
+        run: test ! -f packages/cfe*deb && buildscripts/ci/docker.sh || true
+
+      - name: Save dependency cache
+        uses: actions/cache/save@v3
+        with:
+          path: cache
+          key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
+
+      - name: Save packages cache
+        uses: actions/cache/save@v3
+        with:
+          path: packages
+          key: packages-${{ env.PACKAGE_SHA }}

.gitignore

@@ -1,3 +1,5 @@
 workdir*
 revision
 *.html
+ci/cache
+ci/packages

README.md

@@ -1,5 +1,9 @@
 This repository contains the necessary tools to build and test cfengine packages for various platforms.
 
+## Hardware requirements
+
+By experimentation I have found that building hub packages, which includes php dependency requires more than 1.6G of RAM/swap. 2.6G worked for me, less might work as well.
+
 ## Example build of Community Agent
 
 A minimal example would be to build packages for cfengine community agent.

build-scripts/clean-dependencies

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. `dirname "$0"`/functions
+. detect-environment
+. compile-options
+
+set -ex
+for dep in $DEPS
+do
+    rm -rf $dep
+done

ci/Dockerfile-cfengine-build-package

@@ -0,0 +1,5 @@
+FROM ubuntu:20.04
+RUN apt-get update -y && apt-get install -y systemd wget sudo
+ADD "${NTECH_ROOT}/buildscripts/ci/setup.sh" /
+RUN /bin/bash -c '/setup.sh 2>&1 > setup.log'
+CMD [ "/lib/systemd/systemd" ]

ci/README

@@ -0,0 +1,73 @@
+In this 'buildscripts/ci' directory you will find the means to build cfengine packages.
+Note that you should ensure that the various projects are all on the same version: core, enterprise, masterfiles, nova, mission-portal all should be on say 3.21.x or 3.18.x or master.
+
+Currently a full build with no dependencies cached takes around 53 minutes with deps and packaging.
+
+Two options: containerize build or "normal" machine (such as virtual machine or actual hardware).
+
+The build is designed for ubuntu-20.04 but could be adjusted in various shell scripts for other platforms. (TODO: do this!)
+
+# containerized build
+./clean.sh # cleans any leftover docker bits
+./run.sh # builds a docker image, sets it up, runs it in the background, copies local cache into container, runs the build and saves the results
+./shell.sh # runs bash in the container for debugging
+
+See /data/buildscripts/ci/build.sh for required environment variables and steps to try manually
+
+# virtual or real machine
+
+For virtual machine such as with vagrant, at $NTECH_ROOT (aka top-level directory containing all CFEngine repositories), init an ubuntu-20.04 vagrant machine so it has access to all your repositories.
+
+vagrant init ubuntu/focal64
+vagrant ssh
+
+# note however, that currently you will need to copy /vagrant to a non shared filesystem most likely as vboxsf type doesn't support required hard and soft links for packaging :(
+set -ex
+repos="\
+buildscripts \
+core \
+enterprise \
+masterfiles \
+mission-portal \
+nova \
+"
+
+mkdir -p $HOME/workspace
+pwd
+for repo in $repos; do
+  pwd
+  cp -R "$repo" $HOME/workspace
+done
+
+# always be careful of trailing slashes with rsync, check it again if you are typing!
+rsync -avz /vagrant/cache/ $HOME/.cache/
+
+
+And then do the following as you would on a real machine:
+
+cd $HOME/workspace
+./buildscripts/ci/setup.sh # install needed dependencies/packages/etc
+./buildscripts/ci/setup-projects.sh # items which must be done AFTER the container is created with setup.sh, in non-container case must be executed second
+./buildscripts/ci/build.sh # run the build, should generate a package
+
+don't use ./clean.sh ./run.sh or ./shell.sh those are specifically for containerized/docker build
+
+If you are running selenium tests in mission-portal, you might need to do a `make -C mission-portal clean` before creating packages as the selenium tests copy some things from the distribution into the repository directory that cause problems with packaging, such as the api directory from nova.
+
+# manual debugging on-host
+
+cat buildscripts/ci/build.sh
+# now export the environment variables there (and adjust as needed)
+export BUILD_TYPE=DEBUG
+export ESCAPETEST=yes
+export TEST_MACHINE=chroot
+
+then take each step one-at-a-time or re-run as needed
+
+./buildscripts/build-scripts/install-dependencies
+
+Note that you can provide an argument to install-dependencies to just build ONE dependency, such as
+
+./buildscripts/build-scripts/install-dependencies lmdb
+
+If you change the version of a dependency the cache for that should be skipped and the dep rebuilt from scratch.

ci/build.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# build cfengine hub package
+set -ex
+export PROJECT=nova
+export NO_CONFIGURE=1
+export BUILD_TYPE=DEBUG
+export ESCAPETEST=yes
+export EXPLICIT_ROLE=hub
+export TEST_MACHINE=chroot
+# TODO maybe seed the cache? cp -R buildscripts/ci/cache ~/.cache
+time ./buildscripts/build-scripts/build-environment-check
+time ./buildscripts/build-scripts/install-dependencies
+time ./buildscripts/build-scripts/configure # 3 minutes locally
+time ./buildscripts/build-scripts/generate-source-tarballs # 1m49
+time ./buildscripts/build-scripts/compile
+time sudo apt remove -y 'cfbuild*' || true
+time sudo apt remove -y 'cfengine-*' || true
+time sudo rm -rf /var/cfengine
+time sudo rm -rf /opt/cfengine
+time ./buildscripts/build-scripts/install-dependencies
+time ./buildscripts/build-scripts/package
+sudo mkdir -p packages
+sudo cp cfengine-nova-hub/*.deb packages/ || true
+sudo cp cfengine-nova-hub/*.rpm packages/ || true
+
+# todo maybe save the cache cp -R ~/.cache buildscripts/ci/cache
+
+# clean up
+time sudo apt remove -y 'cfbuild*' || true
+time sudo apt remove -y 'cfengine-*' || true
+time sudo rm -rf /var/cfengine
+time sudo rm -rf /opt/cfengine

ci/clean.sh

@@ -0,0 +1,6 @@
+# clean up docker stuff
+name=cfengine-build-package
+# TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine
+docker stop $name
+docker rm $name
+docker rmi $name

ci/docker.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# run the build in a docker container
+set -ex
+
+# find the dir two levels up from here, home of all the repositories
+COMPUTED_ROOT=$(readlink -e $(dirname "$0")/../../)
+# NTECH_ROOT should be the same, but if available use it so user can do their own thing.
+NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+
+name=cfengine-build-package
+# todo, check the image against the Dockerfile for up-to-date ness?
+docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true
+# todo, check if already running and up-to-date?
+docker run -d --privileged -v ${NTECH_ROOT}:/data --name $name $name || true
+
+# copy local caches to docker container
+mkdir -p "${NTECH_ROOT}/packages"
+mkdir -p "${NTECH_ROOT}/cache"
+# ending with /. in srcpath copies contents to destpath
+docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache
+
+# in order for build-scripts/autogen to generate a revision file:
+for i in core buildscripts buildscripts/deps-packaging enterprise nova masterfiles
+do
+  docker exec -i $name bash -c "git config --global --add safe.directory /data/$i"
+done
+
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh'
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'
+
+# save back cache and packages to host for handling by CI and such
+docker cp $name:/root/.cache/. "${NTECH_ROOT}/cache/"
+docker cp $name:/data/packages/. "${NTECH_ROOT}/packages/"
+
+# if no packages, then fail
+[ -f packages/*.deb ] || [ -f packages/*.rpm ]

ci/setup-projects.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+echo "=== tool versions (npm, node, composer) ==="
+which npm
+npm --version
+which node
+node --version
+which composer
+composer --version
+
+set -ex
+(
+if test -f "mission-portal/public/scripts/package.json"; then
+	cd mission-portal/public/scripts
+	# install dependencies from npmjs
+	npm i
+fi
+)
+
+# install composer and friends
+(
+if test -f "mission-portal/composer.json"; then
+	cd mission-portal
+	# install PHP dependencies from composer
+	composer install
+fi
+)
+
+(
+if test -f "nova/api/http/composer.json"; then
+	cd nova/api/http/
+	# install PHP dependencies from composer
+	composer install --ignore-platform-reqs
+fi
+)
+
+(
+if test -f "mission-portal/public/themes/default/bootstrap/cfengine_theme.less"; then
+	cd mission-portal/public/themes/default/bootstrap
+	npx -p less lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css
+fi
+)
+
+(
+if test -f "mission-portal/ldap/composer.json"; then
+	cd mission-portal/ldap
+	# install PHP dependencies from composer
+	composer install
+fi
+)
+
+# packages needed for autogen are installed in setup.sh
+NO_CONFIGURE=1 PROJECT=nova ./buildscripts/build-scripts/autogen
+
+# remove unwanted dependencies
+sudo apt-get -qy purge libltdl-dev libltdl7 #libtool

ci/setup.sh

@@ -0,0 +1,47 @@
+# setup build host on ubuntu 20
+set -ex
+PREFIX=/var/cfengine
+
+# Github Actions provides machines with various packages installed,
+# what confuses our build system into thinking that it's an RPM distro.
+sudo rm -f /bin/rpm
+
+# Install dependencies
+sudo apt-get update -y
+
+# install apt-utils so that debconf can configure installed packages
+sudo apt-get install apt-utils
+
+# git is needed for build-scripts/autogen to determine revision for such things as deps-packaging
+sudo apt-get install -qy git
+
+# python3-pip is needed for cfengine-nova-hub.deb packaging
+sudo apt-get install -qy python3 python3-pip
+
+# Install Python2 and psycopg2
+sudo apt-get -qy install python2
+wget https://bootstrap.pypa.io/pip/2.7/get-pip.py -O get-pip.py
+sudo python2 get-pip.py
+sudo pip install psycopg2-binary
+
+# install composer and friends
+sudo apt-get -qq -y install curl php7.4-cli php7.4-curl php7.4-zip php7.4-mbstring php7.4-xml php7.4-gd composer php7.4-ldap
+# packages needed for autogen
+sudo apt-get -qy install git autoconf automake m4 make bison flex \
+ binutils libtool gcc g++ libc-dev libpam0g-dev python2 python3 psmisc
+
+# packages needed for buildscripts
+sudo apt-get -qy install libncurses5 rsync
+# packages needed for building 
+sudo apt-get -qy install bison flex binutils build-essential fakeroot ntp \
+ dpkg-dev libpam0g-dev python2 python3 debhelper pkg-config psmisc nfs-common
+
+# remove unwanted packages
+sudo apt-get -qq purge apache* "postgresql*" redis*
+
+# packages needed for installing Mission portal dependencies
+# remove any nodejs or node- packages currently in place
+sudo apt-get remove -y 'nodejs*' 'node-*'
+# replace with exact version we want
+wget -O - https://deb.nodesource.com/setup_12.x | sudo -E bash -
+sudo apt-get install -y nodejs

ci/shell.sh

@@ -0,0 +1,2 @@
+# shell into the docker container
+docker exec -it cfengine-build-package bash

deps-packaging/libattr/source

@@ -1 +1 @@
-http://nongnu.uib.no/attr/
+https://download-mirror.savannah.gnu.org/releases/attr/