From 85510009b4e4f4d76c7492180ddf014a6de0f12c Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Sat, 25 Oct 2025 16:54:39 +0200 Subject: [PATCH 1/5] Setup multi arch builds --- .gitlab-ci.yml | 2 +- .gitlab/ci/container-boot.gitlab-ci.yml | 1 + .gitlab/ci/container-build.gitlab-ci.yml | 12 +++++----- container/ruby/Dockerfile | 4 ++-- support/helpers.sh | 30 ++++++++++++------------ 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29bde8f..84dd9da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ stages: - container - container:boot -default: +.default-retry: retry: max: 2 when: diff --git a/.gitlab/ci/container-boot.gitlab-ci.yml b/.gitlab/ci/container-boot.gitlab-ci.yml index e765790..a2686d7 100644 --- a/.gitlab/ci/container-boot.gitlab-ci.yml +++ b/.gitlab/ci/container-boot.gitlab-ci.yml @@ -1,6 +1,7 @@ .container:boot: extends: - .dind + - .default-retry stage: container:boot image: ghcr.io/code0-tech/build-images/reticulum-builder:207.1-ruby-3.2.2 variables: diff --git a/.gitlab/ci/container-build.gitlab-ci.yml b/.gitlab/ci/container-build.gitlab-ci.yml index 4eb11e0..bd2c016 100644 --- a/.gitlab/ci/container-build.gitlab-ci.yml +++ b/.gitlab/ci/container-build.gitlab-ci.yml @@ -1,11 +1,15 @@ .image-build-base: extends: - .dind + - .default-retry image: ghcr.io/code0-tech/build-images/reticulum-builder:207.1-ruby-3.2.2 stage: container + variables: + PLATFORM_ARGS: "--platform linux/amd64,linux/arm64" script: - source support/helpers.sh - docker_login + - docker_setup_multi_arch - 'image=$(echo "$CI_JOB_NAME" | cut -d : -f 2)' - '[ -z "$NEED_PROJECT_DOWNLOAD" ] || download_project $image' retry: @@ -17,18 +21,14 @@ - .image-build-base script: - !reference [.image-build-base, script] - - build_image $image $CI_PIPELINE_ID - - push_image $image $CI_PIPELINE_ID + - build_image $image $CI_PIPELINE_ID "--push" .variant-image-build-base: extends: - .image-build-base script: - !reference [.image-build-base, script] - - build_image $image $CI_PIPELINE_ID "--build-arg VARIANT=$VARIANT" - - image_tag=$(get_image_tag $CI_PIPELINE_ID $VARIANT) - - retag_image $image $CI_PIPELINE_ID $image_tag - - push_image $image $image_tag + - build_image $image $CI_PIPELINE_ID "--build-arg VARIANT=$VARIANT --push" $(get_image_tag $CI_PIPELINE_ID $VARIANT) container:mise: extends: diff --git a/container/ruby/Dockerfile b/container/ruby/Dockerfile index 903c159..5ea7195 100644 --- a/container/ruby/Dockerfile +++ b/container/ruby/Dockerfile @@ -2,8 +2,8 @@ ARG RETICULUM_IMAGE_TAG=local FROM ghcr.io/code0-tech/reticulum/ci-builds/mise:$RETICULUM_IMAGE_TAG -ARG RUBY_VERSION=3.2.2 +ARG RUBY_VERSION=3.4.7 RUN apk add --update --no-cache build-base tzdata zlib-dev perl linux-headers libffi readline yaml-dev -RUN mise install-into ruby@$RUBY_VERSION /usr/local/share/ruby +RUN mise install-into ruby@$RUBY_VERSION /usr/local/share/ruby; exit_code=$?; [ $exit_code -ne 0 ] && tail -n 500 /tmp/ruby-build.*.log; exit $exit_code ENV PATH=/usr/local/share/ruby/bin:$PATH diff --git a/support/helpers.sh b/support/helpers.sh index ff92608..f23beb1 100644 --- a/support/helpers.sh +++ b/support/helpers.sh @@ -13,10 +13,22 @@ function docker_login() { echo $C0_GH_TOKEN | docker login -u $ --password-stdin ghcr.io } +function docker_setup_multi_arch() { + docker context create build-context + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap \ + --use \ + build-context + docker run --privileged --rm tonistiigi/binfmt --install all +} + function build_image() { image=$1 reticulum_tag=$2 build_args=$3 + reticulum_push_tag=${4:-$reticulum_tag} echo "Building image for $image" @@ -25,27 +37,15 @@ function build_image() { container/$image/renderDockerfile fi - docker build \ - -t "ghcr.io/code0-tech/reticulum/ci-builds/$image:$reticulum_tag" \ + docker buildx build \ + -t "ghcr.io/code0-tech/reticulum/ci-builds/$image:$reticulum_push_tag" \ -f "container/$image/Dockerfile" \ --build-arg RETICULUM_IMAGE_TAG=$reticulum_tag \ $build_args \ + $PLATFORM_ARGS \ . } -function push_image() { - image=$1 - reticulum_tag=$2 - docker push "ghcr.io/code0-tech/reticulum/ci-builds/$image:$reticulum_tag" -} - -function retag_image() { - image=$1 - reticulum_tag=$2 - variant_tag=$3 - docker image tag "ghcr.io/code0-tech/reticulum/ci-builds/$image:$reticulum_tag" "ghcr.io/code0-tech/reticulum/ci-builds/$image:$variant_tag" -} - function get_image_tag() { reticulum_tag=$1 reticulum_variant=$2 From 31d5a3c124dcf340637990162ab37f48cfc8ddc9 Mon Sep 17 00:00:00 2001 From: "code0-release-tools[bot]" <163606586+code0-release-tools[bot]@users.noreply.github.com> Date: Tue, 28 Oct 2025 19:07:22 +0000 Subject: [PATCH 2/5] Update sagittarius version to d42a043e04b --- versions/sagittarius | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/sagittarius b/versions/sagittarius index 08a8140..285cfec 100644 --- a/versions/sagittarius +++ b/versions/sagittarius @@ -1 +1 @@ -56198dce107a9c09cc5eca0773f239d9c3eba598 \ No newline at end of file +d42a043e04ba3d78185f0c4981cc7e95145a9408 \ No newline at end of file From eb1f66a6cbe6f26c460aa2cab85ffd31df08297f Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Sat, 1 Nov 2025 23:18:09 +0100 Subject: [PATCH 3/5] Build each architecture on native runner --- .gitlab/ci/container-boot.gitlab-ci.yml | 12 +- .gitlab/ci/container-build.gitlab-ci.yml | 138 ++++++++++++++++++++--- support/helpers.sh | 20 +++- 3 files changed, 142 insertions(+), 28 deletions(-) diff --git a/.gitlab/ci/container-boot.gitlab-ci.yml b/.gitlab/ci/container-boot.gitlab-ci.yml index a2686d7..0eae79c 100644 --- a/.gitlab/ci/container-boot.gitlab-ci.yml +++ b/.gitlab/ci/container-boot.gitlab-ci.yml @@ -3,7 +3,7 @@ - .dind - .default-retry stage: container:boot - image: ghcr.io/code0-tech/build-images/reticulum-builder:207.1-ruby-3.2.2 + image: ghcr.io/code0-tech/build-images/reticulum-builder:227.1-ruby-3.4.7-amd64 variables: COMPOSE_FILE: support/docker-compose.yml @@ -11,7 +11,7 @@ extends: - .container:boot needs: - - container:sagittarius + - manifest:sagittarius parallel: matrix: - SAGITTARIUS_VARIANT: @@ -47,8 +47,8 @@ container:boot:aquila: extends: - .container:boot needs: - - container:aquila - - container:sagittarius + - manifest:aquila + - manifest:sagittarius variables: SAGITTARIUS_VARIANT: ce script: @@ -68,7 +68,7 @@ container:boot:taurus: extends: - .container:boot needs: - - container:taurus + - manifest:taurus script: - bundle install - docker compose up nats -d @@ -81,7 +81,7 @@ container:boot:draco: extends: - .container:boot needs: - - container:draco + - manifest:draco parallel: matrix: - DRACO_VARIANT: diff --git a/.gitlab/ci/container-build.gitlab-ci.yml b/.gitlab/ci/container-build.gitlab-ci.yml index bd2c016..96cface 100644 --- a/.gitlab/ci/container-build.gitlab-ci.yml +++ b/.gitlab/ci/container-build.gitlab-ci.yml @@ -2,14 +2,14 @@ extends: - .dind - .default-retry - image: ghcr.io/code0-tech/build-images/reticulum-builder:207.1-ruby-3.2.2 + tags: + - c0-$PLATFORM + image: ghcr.io/code0-tech/build-images/reticulum-builder:227.1-ruby-3.4.7-$PLATFORM stage: container - variables: - PLATFORM_ARGS: "--platform linux/amd64,linux/arm64" - script: + before_script: - source support/helpers.sh - docker_login - - docker_setup_multi_arch + - docker_setup_builder - 'image=$(echo "$CI_JOB_NAME" | cut -d : -f 2)' - '[ -z "$NEED_PROJECT_DOWNLOAD" ] || download_project $image' retry: @@ -19,73 +19,157 @@ .single-image-build-base: extends: - .image-build-base + parallel: + matrix: + - PLATFORM: + - amd64 + - arm64 script: - - !reference [.image-build-base, script] - - build_image $image $CI_PIPELINE_ID "--push" + - > + build_image + $image + $CI_PIPELINE_ID + "--output type=registry,push-by-digest=true --metadata-file manifest-${CI_JOB_ID}.json" + "" + artifacts: + paths: + - manifest-*.json + expire_in: 1 day .variant-image-build-base: extends: - .image-build-base script: - - !reference [.image-build-base, script] - - build_image $image $CI_PIPELINE_ID "--build-arg VARIANT=$VARIANT --push" $(get_image_tag $CI_PIPELINE_ID $VARIANT) + - > + build_image + $image + $CI_PIPELINE_ID + "--build-arg VARIANT=$VARIANT --output type=registry,push-by-digest=true --metadata-file manifest-${CI_JOB_ID}.json" + "" + artifacts: + paths: + - manifest-*.json + expire_in: 1 day + +.manifest-create-base: + extends: + - .image-build-base + variables: + PLATFORM: amd64 + script: + - create_manifest $image $(get_image_tag $CI_PIPELINE_ID $VARIANT) container:mise: extends: - .single-image-build-base +manifest:mise: + extends: + - .manifest-create-base + needs: + - container:mise + container:rust: extends: - .single-image-build-base needs: - - container:mise + - manifest:mise + +manifest:rust: + extends: + - .manifest-create-base + needs: + - container:rust container:aquila: extends: - .single-image-build-base needs: - - container:rust + - manifest:rust variables: NEED_PROJECT_DOWNLOAD: 'true' +manifest:aquila: + extends: + - .manifest-create-base + needs: + - container:aquila + container:draco: extends: - .variant-image-build-base needs: - - container:rust + - manifest:rust variables: NEED_PROJECT_DOWNLOAD: 'true' parallel: matrix: - VARIANT: - rest + PLATFORM: + - amd64 + - arm64 + +manifest:draco: + extends: + - .manifest-create-base + parallel: + matrix: + - VARIANT: + - rest + needs: + - job: container:draco + parallel: + matrix: + - VARIANT: ['$[[ matrix.VARIANT ]]'] + PLATFORM: + - amd64 + - arm64 container:taurus: extends: - .single-image-build-base needs: - - container:rust + - manifest:rust variables: NEED_PROJECT_DOWNLOAD: 'true' +manifest:taurus: + extends: + - .manifest-create-base + needs: + - container:taurus + container:ruby: extends: - .single-image-build-base needs: - - container:mise + - manifest:mise + +manifest:ruby: + extends: + - .manifest-create-base + needs: + - container:ruby container:postgresql: extends: - .single-image-build-base needs: - - container:mise + - manifest:mise + +manifest:postgresql: + extends: + - .manifest-create-base + needs: + - container:postgresql container:sagittarius: extends: - .variant-image-build-base needs: - - container:ruby - - container:postgresql + - manifest:ruby + - manifest:postgresql variables: NEED_PROJECT_DOWNLOAD: 'true' parallel: @@ -93,3 +177,23 @@ container:sagittarius: - VARIANT: - ce - ee + PLATFORM: + - amd64 + - arm64 + +manifest:sagittarius: + extends: + - .manifest-create-base + parallel: + matrix: + - VARIANT: + - ce + - ee + needs: + - job: container:sagittarius + parallel: + matrix: + - VARIANT: [ '$[[ matrix.VARIANT ]]' ] + PLATFORM: + - amd64 + - arm64 diff --git a/support/helpers.sh b/support/helpers.sh index f23beb1..ec678ac 100644 --- a/support/helpers.sh +++ b/support/helpers.sh @@ -13,7 +13,7 @@ function docker_login() { echo $C0_GH_TOKEN | docker login -u $ --password-stdin ghcr.io } -function docker_setup_multi_arch() { +function docker_setup_builder() { docker context create build-context docker buildx create \ --name container-builder \ @@ -21,14 +21,13 @@ function docker_setup_multi_arch() { --bootstrap \ --use \ build-context - docker run --privileged --rm tonistiigi/binfmt --install all } function build_image() { image=$1 reticulum_tag=$2 build_args=$3 - reticulum_push_tag=${4:-$reticulum_tag} + reticulum_push_tag=${4-":$reticulum_tag"} echo "Building image for $image" @@ -38,14 +37,25 @@ function build_image() { fi docker buildx build \ - -t "ghcr.io/code0-tech/reticulum/ci-builds/$image:$reticulum_push_tag" \ + -t "ghcr.io/code0-tech/reticulum/ci-builds/$image$reticulum_push_tag" \ -f "container/$image/Dockerfile" \ --build-arg RETICULUM_IMAGE_TAG=$reticulum_tag \ $build_args \ - $PLATFORM_ARGS \ . } +function create_manifest() { + image=$1 + reticulum_tag=$2 + + args=(-t "ghcr.io/code0-tech/reticulum/ci-builds/$image:$reticulum_tag") + for manifest in manifest-*.json; do + args+=("$(jq -r '."image.name"' $manifest)@$(jq -r '."containerimage.digest"' $manifest)") + done + + docker buildx imagetools create "${args[@]}" +} + function get_image_tag() { reticulum_tag=$1 reticulum_variant=$2 From c46ea49bed511a8896031b89a7a76ae07444b222 Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Sun, 2 Nov 2025 01:13:39 +0100 Subject: [PATCH 4/5] Add boot tests for each platform --- .gitlab/ci/container-boot.gitlab-ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/container-boot.gitlab-ci.yml b/.gitlab/ci/container-boot.gitlab-ci.yml index 0eae79c..c5da2d3 100644 --- a/.gitlab/ci/container-boot.gitlab-ci.yml +++ b/.gitlab/ci/container-boot.gitlab-ci.yml @@ -2,10 +2,17 @@ extends: - .dind - .default-retry + tags: + - c0-$PLATFORM stage: container:boot - image: ghcr.io/code0-tech/build-images/reticulum-builder:227.1-ruby-3.4.7-amd64 + image: ghcr.io/code0-tech/build-images/reticulum-builder:227.1-ruby-3.4.7-$PLATFORM variables: COMPOSE_FILE: support/docker-compose.yml + parallel: + matrix: + - PLATFORM: + - amd64 + - arm64 .container:boot:sagittarius: extends: @@ -17,6 +24,9 @@ - SAGITTARIUS_VARIANT: - ce - ee + PLATFORM: + - amd64 + - arm64 before_script: - docker compose up postgres -d - docker compose up sagittarius-$SAGITTARIUS_SERVICE -d @@ -86,6 +96,9 @@ container:boot:draco: matrix: - DRACO_VARIANT: - rest + PLATFORM: + - amd64 + - arm64 script: - bundle install - docker compose up nats -d From a0c3264b030f769b34ec26332950b3384600f50f Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Sun, 2 Nov 2025 02:12:17 +0100 Subject: [PATCH 5/5] Add patch to fix ruby stack size --- container/ruby/Dockerfile | 4 +- ...-make-get_main_stack-portable-on-lin.patch | 92 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 container/ruby/patches/0001-thread_pthread.c-make-get_main_stack-portable-on-lin.patch diff --git a/container/ruby/Dockerfile b/container/ruby/Dockerfile index 5ea7195..62a11a8 100644 --- a/container/ruby/Dockerfile +++ b/container/ruby/Dockerfile @@ -5,5 +5,7 @@ FROM ghcr.io/code0-tech/reticulum/ci-builds/mise:$RETICULUM_IMAGE_TAG ARG RUBY_VERSION=3.4.7 RUN apk add --update --no-cache build-base tzdata zlib-dev perl linux-headers libffi readline yaml-dev -RUN mise install-into ruby@$RUBY_VERSION /usr/local/share/ruby; exit_code=$?; [ $exit_code -ne 0 ] && tail -n 500 /tmp/ruby-build.*.log; exit $exit_code +COPY container/ruby/patches /ruby-patches +ENV MISE_RUBY_APPLY_PATCHES=/ruby-patches/0001-thread_pthread.c-make-get_main_stack-portable-on-lin.patch +RUN mise install-into ruby@$RUBY_VERSION /usr/local/share/ruby; exit_code=$?; rm -rf /ruby-patches; [ $exit_code -ne 0 ] && tail -n 500 /tmp/ruby-build.*.log; exit $exit_code ENV PATH=/usr/local/share/ruby/bin:$PATH diff --git a/container/ruby/patches/0001-thread_pthread.c-make-get_main_stack-portable-on-lin.patch b/container/ruby/patches/0001-thread_pthread.c-make-get_main_stack-portable-on-lin.patch new file mode 100644 index 0000000..4fe80c2 --- /dev/null +++ b/container/ruby/patches/0001-thread_pthread.c-make-get_main_stack-portable-on-lin.patch @@ -0,0 +1,92 @@ +From 2cd56ff98bdb291b9b5e2d0e3fd4cd315076c904 Mon Sep 17 00:00:00 2001 +From: Natanael Copa +Date: Wed, 14 Mar 2018 14:14:11 +0100 +Subject: [PATCH] thread_pthread.c: make get_main_stack portable on linux + +We can not rely on the calculated stack size from pthread_getattr_np() +for the main thread, because the way this is calculated may differ +depending on implementation. The _np means non-portable. + +So we test if it is a known implementation (__GLIBC__) and fallback to a +portable way on Linux by parsing /proc/self/maps to get the stack for +main thread. We also add a 100MB safety limit so we don't reserve_stack +with an insane value. + +Credit goes to Szabolcs Nagy for this code. +--- + thread_pthread.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++++--- + 1 file changed, 49 insertions(+), 3 deletions(-) + +diff --git a/thread_pthread.c b/thread_pthread.c +index 951885ffa0..cf90321d1d 100644 +--- a/thread_pthread.c ++++ b/thread_pthread.c +@@ -530,9 +530,6 @@ hpux_attr_getstackaddr(const pthread_attr_t *attr, void **addr) + # define MAINSTACKADDR_AVAILABLE 0 + # endif + #endif +-#if MAINSTACKADDR_AVAILABLE && !defined(get_main_stack) +-# define get_main_stack(addr, size) get_stack(addr, size) +-#endif + + #ifdef STACKADDR_AVAILABLE + /* +@@ -614,6 +611,55 @@ get_stack(void **addr, size_t *size) + return 0; + #undef CHECK_ERR + } ++ ++#if defined(__linux__) && !defined(__GLIBC__) && defined(HAVE_GETRLIMIT) ++ ++#ifndef PAGE_SIZE ++#include ++#define PAGE_SIZE sysconf(_SC_PAGE_SIZE) ++#endif ++ ++static int ++get_main_stack(void **addr, size_t *size) ++{ ++ size_t start, end, limit, prevend = 0; ++ struct rlimit r; ++ FILE *f; ++ char buf[PATH_MAX+80], s[8]; ++ int n; ++ STACK_GROW_DIR_DETECTION; ++ ++ f = fopen("/proc/self/maps", "re"); ++ if (!f) ++ return -1; ++ n = 0; ++ while (fgets(buf, sizeof buf, f)) { ++ n = sscanf(buf, "%zx-%zx %*s %*s %*s %*s %7s", &start, &end, s); ++ if (n >= 2) { ++ if (n == 3 && strcmp(s, "[stack]") == 0) ++ break; ++ prevend = end; ++ } ++ n = 0; ++ } ++ fclose(f); ++ if (n == 0) ++ return -1; ++ ++ limit = 100 << 20; /* 100MB stack limit */ ++ if (getrlimit(RLIMIT_STACK, &r)==0 && r.rlim_cur < limit) ++ limit = r.rlim_cur & -PAGE_SIZE; ++ if (limit > end) limit = end; ++ if (prevend < end - limit) prevend = end - limit; ++ if (start > prevend) start = prevend; ++ *addr = IS_STACK_DIR_UPPER() ? (void *)start : (void *)end; ++ *size = end - start; ++ return 0; ++} ++#else ++# define get_main_stack(addr, size) get_stack(addr, size) ++#endif ++ + #endif + + static struct { +-- +2.16.2 +