From eb0d49f343976b52969cc4ee52a717b6de6d92ac Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 18 Jul 2023 10:44:38 -0500 Subject: [PATCH 1/2] Added ci scripts using docker for github actions Includes unused build-scripts/clean-dependencies which I found helpful while iterating locally. Ticket: ENT-9167 Changelog: none --- .../workflows/build-using-buildscripts.yml | 40 ++++++++++ .gitignore | 2 + README.md | 4 + build-scripts/clean-dependencies | 11 +++ ci/Dockerfile-cfengine-build-package | 5 ++ ci/README | 73 +++++++++++++++++++ ci/build.sh | 32 ++++++++ ci/clean.sh | 6 ++ ci/docker.sh | 36 +++++++++ ci/setup-projects.sh | 56 ++++++++++++++ ci/setup.sh | 47 ++++++++++++ ci/shell.sh | 2 + 12 files changed, 314 insertions(+) create mode 100755 build-scripts/clean-dependencies create mode 100644 ci/Dockerfile-cfengine-build-package create mode 100644 ci/README create mode 100755 ci/build.sh create mode 100755 ci/clean.sh create mode 100755 ci/docker.sh create mode 100755 ci/setup-projects.sh create mode 100755 ci/setup.sh create mode 100755 ci/shell.sh diff --git a/.github/workflows/build-using-buildscripts.yml b/.github/workflows/build-using-buildscripts.yml index d3cf658e3..4c002c2ac 100644 --- a/.github/workflows/build-using-buildscripts.yml +++ b/.github/workflows/build-using-buildscripts.yml @@ -78,3 +78,43 @@ jobs: submodules: recursive ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO }} ssh-known-hosts: github.com + + - name: get PACKAGE_SHA for package cache + run: echo "PACKAGE_SHA=$(mission-portal/ci/package-sha.sh)" | tee -a ${GITHUB_ENV} + + - name: get SHA of buildscripts/deps-packaging last commit + run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV} + working-directory: buildscripts/deps-packaging + + - name: restore packages cache + uses: actions/cache/restore@v3 + with: + path: packages + key: packages-${{ env.PACKAGE_SHA }} + restore-keys: | + packages-${{ env.PACKAGE_SHA }} + + - name: Restore dependency cache + uses: actions/cache/restore@v3 + with: + path: cache + key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }} + restore-keys: | + deps-${{ github.base_ref }} + deps-master + deps + + - name: Build package in docker + run: test ! -f packages/cfe*deb && buildscripts/ci/docker.sh || true + + - name: Save dependency cache + uses: actions/cache/save@v3 + with: + path: cache + key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }} + + - name: Save packages cache + uses: actions/cache/save@v3 + with: + path: packages + key: packages-${{ env.PACKAGE_SHA }} diff --git a/.gitignore b/.gitignore index 8fb9867ad..92e2d4b56 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ workdir* revision *.html +ci/cache +ci/packages diff --git a/README.md b/README.md index 476936176..20c45df9c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ This repository contains the necessary tools to build and test cfengine packages for various platforms. +## Hardware requirements + +By experimentation I have found that building hub packages, which includes php dependency requires more than 1.6G of RAM/swap. 2.6G worked for me, less might work as well. + ## Example build of Community Agent A minimal example would be to build packages for cfengine community agent. diff --git a/build-scripts/clean-dependencies b/build-scripts/clean-dependencies new file mode 100755 index 000000000..d929141ce --- /dev/null +++ b/build-scripts/clean-dependencies @@ -0,0 +1,11 @@ +#!/bin/sh + +. `dirname "$0"`/functions +. detect-environment +. compile-options + +set -ex +for dep in $DEPS +do + rm -rf $dep +done diff --git a/ci/Dockerfile-cfengine-build-package b/ci/Dockerfile-cfengine-build-package new file mode 100644 index 000000000..65f1e8951 --- /dev/null +++ b/ci/Dockerfile-cfengine-build-package @@ -0,0 +1,5 @@ +FROM ubuntu:20.04 +RUN apt-get update -y && apt-get install -y systemd wget sudo +ADD "${NTECH_ROOT}/buildscripts/ci/setup.sh" / +RUN /bin/bash -c '/setup.sh 2>&1 > setup.log' +CMD [ "/lib/systemd/systemd" ] diff --git a/ci/README b/ci/README new file mode 100644 index 000000000..ba9ebad15 --- /dev/null +++ b/ci/README @@ -0,0 +1,73 @@ +In this 'buildscripts/ci' directory you will find the means to build cfengine packages. +Note that you should ensure that the various projects are all on the same version: core, enterprise, masterfiles, nova, mission-portal all should be on say 3.21.x or 3.18.x or master. + +Currently a full build with no dependencies cached takes around 53 minutes with deps and packaging. + +Two options: containerize build or "normal" machine (such as virtual machine or actual hardware). + +The build is designed for ubuntu-20.04 but could be adjusted in various shell scripts for other platforms. (TODO: do this!) + +# containerized build +./clean.sh # cleans any leftover docker bits +./run.sh # builds a docker image, sets it up, runs it in the background, copies local cache into container, runs the build and saves the results +./shell.sh # runs bash in the container for debugging + +See /data/buildscripts/ci/build.sh for required environment variables and steps to try manually + +# virtual or real machine + +For virtual machine such as with vagrant, at $NTECH_ROOT (aka top-level directory containing all CFEngine repositories), init an ubuntu-20.04 vagrant machine so it has access to all your repositories. + +vagrant init ubuntu/focal64 +vagrant ssh + +# note however, that currently you will need to copy /vagrant to a non shared filesystem most likely as vboxsf type doesn't support required hard and soft links for packaging :( +set -ex +repos="\ +buildscripts \ +core \ +enterprise \ +masterfiles \ +mission-portal \ +nova \ +" + +mkdir -p $HOME/workspace +pwd +for repo in $repos; do + pwd + cp -R "$repo" $HOME/workspace +done + +# always be careful of trailing slashes with rsync, check it again if you are typing! +rsync -avz /vagrant/cache/ $HOME/.cache/ + + +And then do the following as you would on a real machine: + +cd $HOME/workspace +./buildscripts/ci/setup.sh # install needed dependencies/packages/etc +./buildscripts/ci/setup-projects.sh # items which must be done AFTER the container is created with setup.sh, in non-container case must be executed second +./buildscripts/ci/build.sh # run the build, should generate a package + +don't use ./clean.sh ./run.sh or ./shell.sh those are specifically for containerized/docker build + +If you are running selenium tests in mission-portal, you might need to do a `make -C mission-portal clean` before creating packages as the selenium tests copy some things from the distribution into the repository directory that cause problems with packaging, such as the api directory from nova. + +# manual debugging on-host + +cat buildscripts/ci/build.sh +# now export the environment variables there (and adjust as needed) +export BUILD_TYPE=DEBUG +export ESCAPETEST=yes +export TEST_MACHINE=chroot + +then take each step one-at-a-time or re-run as needed + +./buildscripts/build-scripts/install-dependencies + +Note that you can provide an argument to install-dependencies to just build ONE dependency, such as + +./buildscripts/build-scripts/install-dependencies lmdb + +If you change the version of a dependency the cache for that should be skipped and the dep rebuilt from scratch. diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 000000000..201c01982 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# build cfengine hub package +set -ex +export PROJECT=nova +export NO_CONFIGURE=1 +export BUILD_TYPE=DEBUG +export ESCAPETEST=yes +export EXPLICIT_ROLE=hub +export TEST_MACHINE=chroot +# TODO maybe seed the cache? cp -R buildscripts/ci/cache ~/.cache +time ./buildscripts/build-scripts/build-environment-check +time ./buildscripts/build-scripts/install-dependencies +time ./buildscripts/build-scripts/configure # 3 minutes locally +time ./buildscripts/build-scripts/generate-source-tarballs # 1m49 +time ./buildscripts/build-scripts/compile +time sudo apt remove -y 'cfbuild*' || true +time sudo apt remove -y 'cfengine-*' || true +time sudo rm -rf /var/cfengine +time sudo rm -rf /opt/cfengine +time ./buildscripts/build-scripts/install-dependencies +time ./buildscripts/build-scripts/package +sudo mkdir -p packages +sudo cp cfengine-nova-hub/*.deb packages/ || true +sudo cp cfengine-nova-hub/*.rpm packages/ || true + +# todo maybe save the cache cp -R ~/.cache buildscripts/ci/cache + +# clean up +time sudo apt remove -y 'cfbuild*' || true +time sudo apt remove -y 'cfengine-*' || true +time sudo rm -rf /var/cfengine +time sudo rm -rf /opt/cfengine diff --git a/ci/clean.sh b/ci/clean.sh new file mode 100755 index 000000000..ca54f7738 --- /dev/null +++ b/ci/clean.sh @@ -0,0 +1,6 @@ +# clean up docker stuff +name=cfengine-build-package +# TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine +docker stop $name +docker rm $name +docker rmi $name diff --git a/ci/docker.sh b/ci/docker.sh new file mode 100755 index 000000000..765d5b8c1 --- /dev/null +++ b/ci/docker.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# run the build in a docker container +set -ex + +# find the dir two levels up from here, home of all the repositories +COMPUTED_ROOT=$(readlink -e $(dirname "$0")/../../) +# NTECH_ROOT should be the same, but if available use it so user can do their own thing. +NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT} + +name=cfengine-build-package +# todo, check the image against the Dockerfile for up-to-date ness? +docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true +# todo, check if already running and up-to-date? +docker run -d --privileged -v ${NTECH_ROOT}:/data --name $name $name || true + +# copy local caches to docker container +mkdir -p "${NTECH_ROOT}/packages" +mkdir -p "${NTECH_ROOT}/cache" +# ending with /. in srcpath copies contents to destpath +docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache + +# in order for build-scripts/autogen to generate a revision file: +for i in core buildscripts buildscripts/deps-packaging enterprise nova masterfiles +do + docker exec -i $name bash -c "git config --global --add safe.directory /data/$i" +done + +docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh' +docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh' + +# save back cache and packages to host for handling by CI and such +docker cp $name:/root/.cache/. "${NTECH_ROOT}/cache/" +docker cp $name:/data/packages/. "${NTECH_ROOT}/packages/" + +# if no packages, then fail +[ -f packages/*.deb ] || [ -f packages/*.rpm ] diff --git a/ci/setup-projects.sh b/ci/setup-projects.sh new file mode 100755 index 000000000..f74dcb585 --- /dev/null +++ b/ci/setup-projects.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +echo "=== tool versions (npm, node, composer) ===" +which npm +npm --version +which node +node --version +which composer +composer --version + +set -ex +( +if test -f "mission-portal/public/scripts/package.json"; then + cd mission-portal/public/scripts + # install dependencies from npmjs + npm i +fi +) + +# install composer and friends +( +if test -f "mission-portal/composer.json"; then + cd mission-portal + # install PHP dependencies from composer + composer install +fi +) + +( +if test -f "nova/api/http/composer.json"; then + cd nova/api/http/ + # install PHP dependencies from composer + composer install --ignore-platform-reqs +fi +) + +( +if test -f "mission-portal/public/themes/default/bootstrap/cfengine_theme.less"; then + cd mission-portal/public/themes/default/bootstrap + npx -p less lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css +fi +) + +( +if test -f "mission-portal/ldap/composer.json"; then + cd mission-portal/ldap + # install PHP dependencies from composer + composer install +fi +) + +# packages needed for autogen are installed in setup.sh +NO_CONFIGURE=1 PROJECT=nova ./buildscripts/build-scripts/autogen + +# remove unwanted dependencies +sudo apt-get -qy purge libltdl-dev libltdl7 #libtool diff --git a/ci/setup.sh b/ci/setup.sh new file mode 100755 index 000000000..124e48be8 --- /dev/null +++ b/ci/setup.sh @@ -0,0 +1,47 @@ +# setup build host on ubuntu 20 +set -ex +PREFIX=/var/cfengine + +# Github Actions provides machines with various packages installed, +# what confuses our build system into thinking that it's an RPM distro. +sudo rm -f /bin/rpm + +# Install dependencies +sudo apt-get update -y + +# install apt-utils so that debconf can configure installed packages +sudo apt-get install apt-utils + +# git is needed for build-scripts/autogen to determine revision for such things as deps-packaging +sudo apt-get install -qy git + +# python3-pip is needed for cfengine-nova-hub.deb packaging +sudo apt-get install -qy python3 python3-pip + +# Install Python2 and psycopg2 +sudo apt-get -qy install python2 +wget https://bootstrap.pypa.io/pip/2.7/get-pip.py -O get-pip.py +sudo python2 get-pip.py +sudo pip install psycopg2-binary + +# install composer and friends +sudo apt-get -qq -y install curl php7.4-cli php7.4-curl php7.4-zip php7.4-mbstring php7.4-xml php7.4-gd composer php7.4-ldap +# packages needed for autogen +sudo apt-get -qy install git autoconf automake m4 make bison flex \ + binutils libtool gcc g++ libc-dev libpam0g-dev python2 python3 psmisc + +# packages needed for buildscripts +sudo apt-get -qy install libncurses5 rsync +# packages needed for building +sudo apt-get -qy install bison flex binutils build-essential fakeroot ntp \ + dpkg-dev libpam0g-dev python2 python3 debhelper pkg-config psmisc nfs-common + +# remove unwanted packages +sudo apt-get -qq purge apache* "postgresql*" redis* + +# packages needed for installing Mission portal dependencies +# remove any nodejs or node- packages currently in place +sudo apt-get remove -y 'nodejs*' 'node-*' +# replace with exact version we want +wget -O - https://deb.nodesource.com/setup_12.x | sudo -E bash - +sudo apt-get install -y nodejs diff --git a/ci/shell.sh b/ci/shell.sh new file mode 100755 index 000000000..e5f5d8171 --- /dev/null +++ b/ci/shell.sh @@ -0,0 +1,2 @@ +# shell into the docker container +docker exec -it cfengine-build-package bash From 437b108b1626a1cc729d65866602c6c60aefda8a Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 18 Jul 2023 10:45:27 -0500 Subject: [PATCH 2/2] Fixed non-working source URL for attr dependency Ticket: ENT-9167 Changelog: none --- deps-packaging/libattr/source | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps-packaging/libattr/source b/deps-packaging/libattr/source index b58ee64cb..4273e5d82 100644 --- a/deps-packaging/libattr/source +++ b/deps-packaging/libattr/source @@ -1 +1 @@ -http://nongnu.uib.no/attr/ +https://download-mirror.savannah.gnu.org/releases/attr/