Add new Docker based build system

The new system is primarily targeting efficiency: - Smaller and quicker to build Docker build environment images - Storage of Docker images in registry instead of cache that expires - Leverages much more performant namespace.so GitHub actions runners - Each step is optimized for parallelism to leverage new runners - Heavy cross branch, more persistent, caching of compilation objects
beyond-all-reason · Oct 14, 2024 · 89fae25 · 89fae25
1 parent d416225
commit 89fae25
Show file tree

Hide file tree

Showing 13 changed files with 479 additions and 0 deletions.
diff --git a/.github/workflows/docker-images-build.yml b/.github/workflows/docker-images-build.yml
@@ -0,0 +1,42 @@
+# Builds docker images used by engine-build.yml
+name: Docker Images Build
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths:
+      - 'docker-build-v2/amd64-linux/**'
+      - 'docker-build-v2/amd64-windows/**'
+  pull_request:
+    paths:
+      - 'docker-build-v2/amd64-linux/**'
+      - 'docker-build-v2/amd64-windows/**'
+jobs:
+  build-images:
+    name: Build ${{ matrix.system }} docker image
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        system:
+          - amd64-linux
+          - amd64-windows
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: docker-build-v2/${{ matrix.system }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ghcr.io/${{ github.repository_owner }}/recoil-build-${{ matrix.system }}:latest
diff --git a/.github/workflows/engine-build.yml b/.github/workflows/engine-build.yml
@@ -0,0 +1,154 @@
+# Engine CI workflow that compiles engine using build-docker-v2 scripts with
+# https://namespace.so/ GitHub Actions Runners.
+#
+# The worklow has multiple layers of caching for ccache compilation objects:
+#
+# - Local cache on runner's machine SSD disk, very efficient but prone to
+#   cache misses when workflow can't be scheduled next to the storage
+# - Remote, higher latency, S3 based storage using Cloudflare R2 bucket
+# - For pull requests only: GitHub Actions Cache archive build by mounting
+#   overlay filesystem on top of two other read-only caches (because pull
+#   requests can't have write access for security reasons) to reduce build
+#   times for subsequent pushes to the same PR branch.
+#
+# Because ccache can't use S3 based storage directly, we are leveraging
+# bazel-remote project that supports it, and ccache supports storing artifacts
+# in bazel remote caching HTTP layout.
+
+name: Build Engine v2
+on:
+  workflow_dispatch:
+    inputs:
+      recache:
+        description: "Recache ccache build objects"
+        type: boolean
+        default: false
+  # TODO: Uncomment after it's merged and docker images are availabe
+  # pull_request:
+  #   paths-ignore:
+  #     - 'doc/**'
+  # push:
+  #   branches:
+  #     - master
+  #     # TODO: add release branches once their naming is finalized
+  #   paths-ignore:
+  #     - 'doc/**'
+jobs:
+  build-engine:
+    strategy:
+      matrix:
+        system:
+          - amd64-linux
+          - amd64-windows
+    runs-on:
+      - nscloud-ubuntu-24.04-amd64-8x16-with-cache
+      - nscloud-cache-tag-engine-${{ matrix.system }}
+      - nscloud-cache-size-20gb
+      - nscloud-git-mirror-5gb
+      - nscloud-exp-container-image-cache
+    steps:
+      - name: Checkout code
+        uses: namespacelabs/nscloud-checkout-action@v5
+        with:
+          fetch-depth: 0
+          submodules: recursive
+          dissociate: true
+          path: src
+      - name: Setup ccache cache
+        uses: namespacelabs/nscloud-cache-action@v1
+        with:
+          path: |
+            bazel-remote-data
+            tools
+      - name: Restore pull request's bazel remote cache overlay
+        id: pr-cache-restore
+        if: github.event_name == 'pull_request'
+        uses: actions/cache/restore@v4
+        with:
+          path: bazel-remote-data-overlay.tar
+          key: pr-bazel-remote-data-${{ matrix.system }}-${{ github.run_id }}
+          restore-keys: pr-bazel-remote-data-${{ matrix.system }}-
+      - name: Mount bazel remote overlay
+        id: mount-overlay
+        if: github.event_name == 'pull_request'
+        run: |
+          sudo apt-get install --yes fuse-overlayfs
+          sudo tar --acls --xattrs --xattrs-include='*' -xf bazel-remote-data-overlay.tar || mkdir bazel-remote-data-overlay
+          mkdir -p overlay-workdir bazel-remote-data-merged
+          sudo fuse-overlayfs -o lowerdir=bazel-remote-data,upperdir=bazel-remote-data-overlay,workdir=overlay-workdir bazel-remote-data-merged
+      - name: Start remote ccache fetcher
+        uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635  # v1.0.7
+        env:
+          BAZEL_REMOTE_S3_ENDPOINT: ${{ vars.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          BAZEL_REMOTE_S3_BUCKET: ${{ vars.R2_BUCKET_BUILD_CACHE }}
+          BAZEL_REMOTE_S3_ACCESS_KEY_ID: ${{ github.event_name == 'pull_request' && vars.R2_RO_ACCESS_KEY_ID || vars.R2_ACCESS_KEY_ID }}
+          BAZEL_REMOTE_S3_SECRET_ACCESS_KEY: ${{ github.event_name == 'pull_request' && vars.R2_RO_ACCESS_KEY_SECRET || secrets.R2_ACCESS_KEY_SECRET }}
+        with:
+          run: |
+            if ! sha256sum --status -c <<< "8679a76074b1408a95d2b3ec0f5b1a6d0c20500cfc24c3a87ef08c1b60200f8c tools/bazel-remote"; then
+              curl -L https://github.com/buchgr/bazel-remote/releases/download/v2.4.4/bazel-remote-2.4.4-linux-x86_64 -o bazel-remote
+              chmod +x bazel-remote
+              mv bazel-remote tools/bazel-remote
+            fi
+
+            cat > remote_ccache.conf <<EOF
+            max_size = 10G
+            cache_dir = /build/cache
+            remote_storage = http://127.0.0.1:8085|layout=bazel
+            remote_only = true
+            EOF
+
+            tools/bazel-remote \
+              --dir bazel-remote-data${{ github.event_name == 'pull_request' && '-merged' || '' }} \
+              --s3.auth_method access_key \
+              --s3.region auto \
+              --max_size 5 \
+              --num_uploaders ${{ github.event_name == 'pull_request' && '0' || '100' }} \
+              --disable_http_ac_validation \
+              --http_address 127.0.0.1:8085 \
+              --access_log_level none &
+          wait-on: |
+            http-get://127.0.0.1:8085/cas/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+      - name: Pull builder image
+        run: docker pull ghcr.io/${{ github.repository_owner }}/recoil-build-${{ matrix.system }}:latest
+      - name: Build
+        # Instead of manually running docker, it would be cool to just run whole GitHub actions
+        # in container, but unfortunately ubuntu 18.04 is just too old to be able to do it ;(.
+        run: |
+          mkdir -p artifacts
+          docker run -i --rm \
+            -v /etc/passwd:/etc/passwd:ro \
+            -v /etc/group:/etc/group:ro \
+            --user="$(id -u):$(id -g)" \
+            -v $(pwd)/src:/build/src:ro \
+            -v $(pwd)/remote_ccache.conf:/build/ccache.conf:ro \
+            -v $(pwd)/artifacts:/build/artifacts:rw \
+            -e CCACHE_${{ inputs.recache && 'RECACHE' || 'NORECACHE' }}=1 \
+            --network host \
+            ghcr.io/${{ github.repository_owner }}/recoil-build-${{ matrix.system }}:latest \
+            bash <<EOF
+          set -e
+          cd /build/src/docker-build-v2/scripts
+          ./build.sh
+          ./split-debug-info.sh
+          ./package.sh
+
+          EOF
+      - name: Save
+        if: github.event_name != 'pull_request'
+        uses: namespace-actions/upload-artifact@v0
+        with:
+          name: output-${{ matrix.system }}
+          path: ./artifacts
+      - name: Unmount bazel remote overlay
+        if: always() && steps.mount-overlay.outcome == 'success'
+        run: |
+          sudo fusermount -u bazel-remote-data-merged
+          sudo tar --acls --xattrs --xattrs-include='*' -cf bazel-remote-data-overlay.tar bazel-remote-data-overlay
+      - name: Save pull request's bazel remote cache overlay
+        id: pr-cache-save
+        if: always() && steps.pr-cache-restore.outcome == 'success'
+        uses: actions/cache/save@v4
+        with:
+          path: bazel-remote-data-overlay.tar
+          key: ${{ steps.pr-cache-restore.outputs.cache-primary-key }}
diff --git a/docker-build-v2/README.md b/docker-build-v2/README.md
@@ -0,0 +1,79 @@
+# Engine Docker Build v2
+
+This directory contains work in progress next version of the Docker engine
+build scripts.
+
+## Local usage
+
+To execute build locally use `build.sh` script
+
+```console
+$ docker-build-v2/build.sh
+USAGE: docker-build-v2/build.sh {windows|linux} [cmake_flag...]
+```
+
+For example
+
+```shell
+docker-build-v2/build.sh windows
+```
+
+will:
+
+1. Automatically fetch/update Docker image with engine build environment from
+   [GitHub packages](https://github.com/beyond-all-reason?tab=packages&repo_name=spring)
+2. Configure the release configuration of engine build
+3. Compile and install engine using following paths in the repository root:
+   - `.cache`: compilation cache
+   - `build-windows`: compilation output
+   - `build-windows/install`: ready to use installation
+
+### Custom build config
+
+Because script takes cmake arguments compilation can be easily adjusted this
+way. For example to compile Linux release with Tracy support and skip building
+headless:
+
+```shell
+docker-build-v2/build.sh linux -DBUILD_spring-headless=OFF -DTRACY_ENABLE=ON
+```
+
+### Custom Docker image
+
+Before downloading official Docker build environment image, `build.sh` will
+first lookup if there exists locally a Docker image with tag
+`recoil-build-amd64-windows` (and `-linux` for Linux). If you want to adjust
+the Docker build image, test local changes, build it with that tag:
+
+```shell
+docker build -t recoil-build-amd64-windows docker-build-v2/amd64-windows
+```
+
+and `build.sh` will use it.
+
+## Overview
+
+There are two separate build images, one for Windows, one for Linux. The Docker
+images are built as part of GitHub actions workflow and stored in the GitHub
+Package repository. The images are relatively small (~300-400MiB compressed)
+and building them takes 2-3 minutes.
+
+Each of the images contains a complete required build environment with all
+dependencies installed (including
+[mingwlibs](https://github.com/beyond-all-reason/mingwlibs64) etc.), configured
+for proper resolution from engine CMake configuration, and caching with
+[ccache](https://ccache.dev/).
+
+The build step is then a platform agnostic invocation of CMake with generic
+release build configuration.
+
+To sum up, there is a separation:
+
+- Build environment: dependencies, toolchain, etc. are part of the Docker
+  image.
+- Build options: e.g. optimization level, are in the platform agnostic build
+  script `scripts/build.sh` stored in repository.
+- Post build scripts: platform agnostic scripts:
+  - `script/split-debug-info.sh`: Splits debug information from binaries
+  - `script/package.sh`: Creates 2 archives, one with engine and one with
+     debug symbols.
diff --git a/docker-build-v2/amd64-linux/Dockerfile b/docker-build-v2/amd64-linux/Dockerfile
@@ -0,0 +1,53 @@
+FROM docker.io/ubuntu:18.04
+ENV ENGINE_PLATFORM=amd64-linux
+
+RUN apt-get update \
+ && apt-get install --no-install-recommends --yes software-properties-common \
+ && add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
+ && apt-get update \
+ && apt-get install --no-install-recommends --yes \
+    curl gcc-13 g++-13 git ninja-build curl p7zip-full python3-pip python3-setuptools \
+    libsdl2-dev libopenal-dev libfreetype6-dev libfontconfig1-dev \
+ && apt-get remove --purge --yes software-properties-common \
+ && apt-get autoremove --purge --yes \
+ && apt-get upgrade --yes \
+ && rm -rf /var/lib/apt/lists/*
+
+# We need a newer ccache for compression support
+ARG CCACHE_VERSION="4.10.2"
+RUN curl -L -O https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \
+ && tar -xf ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \
+ && mv ccache-${CCACHE_VERSION}-linux-x86_64/ccache /usr/bin \
+ && rm -rf ccache-${CCACHE_VERSION}-linux-x86_64*
+
+# And need newer zstd :(, fortunatelly it's quick build
+RUN git clone --depth 1 --branch v1.5.6 https://github.com/facebook/zstd.git \
+ && cd zstd \
+ && CC=g++-13 make -j$(nproc) \
+ && mv programs/zstd /usr/local/bin \
+ && cd .. \
+ && rm -rf zstd
+
+RUN pip3 install --upgrade pip \
+ && pip3 install scikit-build \
+ && pip3 install cmake==3.25.*
+
+WORKDIR /build
+RUN mkdir src cache out artifacts && chmod a+rwx cache out artifacts
+
+# Fetch library dependencies and configure resolution
+RUN git clone --depth=1 https://github.com/beyond-all-reason/spring-static-libs.git -b 18.04 spring-static-libs
+ENV PKG_CONFIG_LIBDIR=/build/spring-static-libs/lib/pkgconfig
+ENV PKG_CONFIG="pkg-config --define-prefix --static"
+ENV CMAKE_PREFIX_PATH=/build/spring-static-libs/
+ENV PREFER_STATIC_LIBS=TRUE
+
+# Set up default cmake toolchain
+COPY toolchain.cmake .
+ENV CMAKE_TOOLCHAIN_FILE=/build/toolchain.cmake
+
+# Configure ccache caching
+COPY ccache.conf .
+ENV CCACHE_CONFIGPATH=/build/ccache.conf
+ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
+ENV CMAKE_C_COMPILER_LAUNCHER=ccache
diff --git a/docker-build-v2/amd64-linux/ccache.conf b/docker-build-v2/amd64-linux/ccache.conf
@@ -0,0 +1,2 @@
+max_size = 10G
+cache_dir = /build/cache
diff --git a/docker-build-v2/amd64-linux/toolchain.cmake b/docker-build-v2/amd64-linux/toolchain.cmake
@@ -0,0 +1,7 @@
+SET(CMAKE_SYSTEM_NAME Linux)
+SET(CMAKE_C_COMPILER "gcc-13")
+SET(CMAKE_CXX_COMPILER "g++-13")
+SET(CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=gold")
+SET(CMAKE_MODULE_LINKER_FLAGS_INIT "-fuse-ld=gold")
+SET(CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=gold")
+SET(CMAKE_DISABLE_PRECOMPILE_HEADERS ON)  # Little usage, only rmlui, improves ccache hit ratio
diff --git a/docker-build-v2/amd64-windows/Dockerfile b/docker-build-v2/amd64-windows/Dockerfile
@@ -0,0 +1,28 @@
+FROM docker.io/ubuntu:24.04
+ENV ENGINE_PLATFORM=amd64-windows
+
+RUN apt-get update \
+ && apt-get upgrade --yes \
+ && apt-get install --no-install-recommends --yes \
+    g++-mingw-w64-x86-64-posix git ninja-build ccache pipx p7zip-full binutils zstd \
+ && rm -rf /var/lib/apt/lists/*
+
+# Using `pipx --global` would be better but it's available only since pipx 1.5
+RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install cmake==3.25.*
+
+WORKDIR /build
+RUN mkdir src cache out artifacts && chmod a+rwx cache out artifacts
+
+# Fetch library dependencies and configure resolution
+RUN git clone --depth=1 https://github.com/beyond-all-reason/mingwlibs64.git
+ENV MINGWLIBS=/build/mingwlibs64
+
+# Set up default cmake toolchain
+COPY toolchain.cmake .
+ENV CMAKE_TOOLCHAIN_FILE=/build/toolchain.cmake
+
+# Configure ccache caching
+COPY ccache.conf .
+ENV CCACHE_CONFIGPATH=/build/ccache.conf
+ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
+ENV CMAKE_C_COMPILER_LAUNCHER=ccache
diff --git a/docker-build-v2/amd64-windows/ccache.conf b/docker-build-v2/amd64-windows/ccache.conf
@@ -0,0 +1,2 @@
+max_size = 10G
+cache_dir = /build/cache
diff --git a/docker-build-v2/amd64-windows/toolchain.cmake b/docker-build-v2/amd64-windows/toolchain.cmake
@@ -0,0 +1,10 @@
+SET(CMAKE_SYSTEM_NAME Windows)
+SET(CMAKE_C_COMPILER "x86_64-w64-mingw32-gcc-posix")
+SET(CMAKE_CXX_COMPILER "x86_64-w64-mingw32-g++-posix")
+SET(CMAKE_RC_COMPILER "x86_64-w64-mingw32-windres")
+SET(WINDRES_BIN "x86_64-w64-mingw32-windres")
+SET(CMAKE_DLLTOOL "x86_64-w64-mingw32-dlltool")
+SET(DLLTOOL "x86_64-w64-mingw32-dlltool")
+SET(CMAKE_CXX_FLAGS_INIT "-static-libstdc++ -static-libgcc")
+SET(CMAKE_C_FLAGS_INIT "-static-libstdc++ -static-libgcc")
+SET(CMAKE_DISABLE_PRECOMPILE_HEADERS ON)  # Little usage, only rmlui, improves ccache hit ratio