diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 237195c..108556a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,17 +1,38 @@ -agents: - provider: aws - instanceType: m6g.xlarge - imagePrefix: drivah-ubuntu-2204-aarch64 +env: + TARGET_VERSION: 18.16.1 steps: - - label: ":buildah: Building Container Images for ARM64" - branches: "*" - command: | - buildah --version - drivah build --changed-since=main ./containers/arm64 + - label: Annotate build with node version + command: buildkite-agent annotate "node.js v$${OVERRIDE_TARGET_VERSION:-$$TARGET_VERSION} with glibc 2.17" --style 'info' + agents: + queue: kibana-default - # - label: ":buildah: Building Container Images for AMD64" - # branches: "*" - # command: | - # buildah --version - # drivah build --changed-since=main ./.buildkite/containers/amd64 + - label: Build custom node.js artifacts with glibc 2.17 for x64 + command: + - scripts/create_build_images.sh + - scripts/build_nodejs.sh + - scripts/upload_nodejs_artifacts.sh + env: + ARCH: amd64 + agents: + queue: c2-16 + timeout_in_minutes: 60 # ideally runs in ~30m + + - label: Build custom node.js artifacts with glibc 2.17 for arm64 + command: + - scripts/create_build_images.sh + - scripts/build_nodejs.sh + - scripts/upload_nodejs_artifacts.sh + env: + ARCH: arm64 + agents: + queue: c2-60 # cross-compiling takes a while + timeout_in_minutes: 180 # ideally runs in ~2hr + + - wait + + - label: Fix SHASUMS256.txt with newly built files' hashes + command: + - scripts/replace_sha_hashes.sh + agents: + queue: kibana-default diff --git a/.gitignore b/.gitignore index d2be9d8..61b15c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ .vscode .idea -workdir \ No newline at end of file +workdir/ diff --git a/README.md b/README.md index bf59127..019d76b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,46 @@ # kibana-custom-nodejs-builds -Contains configuration and sources to build node.js for custom platforms +Contains configuration and sources to build node.js + +The main usecase right now is building `node.js@18+` with `glibc@2.17`, which is required for some older platforms. (More context: https://github.com/nodejs/unofficial-builds/pull/69) + +## Running locally +You can run most scripts locally on Mac/Linux. You'll need a few of the build/infra tools: + - Docker + - node.js + - gsutil (`brew install google-cloud-sdk`) + +Export some env variables required for the builds +```sh + export ARCH="arm64" + export TARGET_VERSION="18.16.1" +``` + +Then run individual scripts locally: +```sh + ./scripts/create_build_images.sh + ./scripts/build_nodejs.sh +``` + +## Docker image for node.js builds +One of the main components we need to create in this step is a docker image for an environment that's set up for building `node.js`. + +The bits for this component are in the [build-image-config](./build-image-config/) folder. + +The docker image uses mounted directories as working directories, as well as outputting the artifacts in these directories. + + +## Scripts for running the builds +Most of the `buildkite` logic is sheltered in the [scripts](./scripts/) directory. + + + +## Context +During development, we found some more information that can be helpful as context, should anyone find this repo again + + - This repository is only needed while + - centos:7 / RHEL7 is supported by Elastic, and we ship node.js with Kibana + - the unofficial-builds repo accepts a linux/arm64 build (https://github.com/nodejs/unofficial-builds/pull/83) + - The created Docker images needn't be pushed + - they can be used once for the build, then rebuilt in case we need to run it again + - I decided to remove the `VARIATION` attribute on the node.js: + - build would result in the variation showing up in the file and folder names, making the logistics more difficult, if we want to keep this mostly transparent for Kibana diff --git a/containers/custom-nodejs-builder-amd64/Dockerfile b/build-image-config/Dockerfile similarity index 89% rename from containers/custom-nodejs-builder-amd64/Dockerfile rename to build-image-config/Dockerfile index a1d6faa..34ece57 100644 --- a/containers/custom-nodejs-builder-amd64/Dockerfile +++ b/build-image-config/Dockerfile @@ -22,7 +22,6 @@ RUN ulimit -n 1024 \ glibc-devel COPY --chown=node:node entrypoint.sh /home/node/entrypoint.sh -COPY --chown=node:node re2_entrypoint.sh /home/node/re2_entrypoint.sh USER node diff --git a/containers/custom-nodejs-builder-arm64/entrypoint.sh b/build-image-config/entrypoint.sh old mode 100644 new mode 100755 similarity index 78% rename from containers/custom-nodejs-builder-arm64/entrypoint.sh rename to build-image-config/entrypoint.sh index fffa7f4..90e0398 --- a/containers/custom-nodejs-builder-arm64/entrypoint.sh +++ b/build-image-config/entrypoint.sh @@ -5,7 +5,7 @@ set -x release_url_base="$1" full_version="$2" -config_flags=${3:-""} #"--without-dtrace --without-npm --without-etw" +config_flags=${3:-""} if [[ $(arch) == x86_64 ]]; then architecture="x64"; @@ -13,6 +13,9 @@ else architecture="arm64" fi +ls -la "/home/node/workdir/src" +ls -la "/home/node/workdir/src/node-${full_version}" + cd "/home/node/workdir/src/node-${full_version}" # Compile from source @@ -25,10 +28,11 @@ export CXX="ccache g++" make -j"$(getconf _NPROCESSORS_ONLN)" binary V= \ DESTCPU="$architecture" \ ARCH="$architecture" \ - VARIATION="glibc-217" \ DISTTYPE="release" \ RELEASE_URLBASE="$release_url_base" \ CONFIG_FLAGS="$config_flags" mkdir -p /home/node/workdir/dist/ +chmod a+w /home/node/workdir/dist mv node-*.tar.?z /home/node/workdir/dist/ +chmod a+rwx /home/node/workdir/dist/* diff --git a/catalog-info.yaml b/catalog-info.yaml index 7800189..2c815f7 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -8,21 +8,3 @@ metadata: links: - title: Pipeline url: https://buildkite.com/elastic/kibana-custom-nodejs-builds - -spec: - type: buildkite-pipeline - owner: group:kibana-operations - system: buildkite - implementation: - apiVersion: buildkite.elastic.dev/v1 - kind: Pipeline - metadata: - name: kibana-custom-nodejs-builds - spec: - repository: elastic/kibana-custom-nodejs-builds - pipeline_file: ".buildkite/pipeline.yml" - teams: - kibana-operations: - access_level: MANAGE_BUILD_AND_READ - everyone: - access_level: READ_ONLY diff --git a/containers/custom-nodejs-builder-amd64/drivah.toml b/containers/custom-nodejs-builder-amd64/drivah.toml deleted file mode 100644 index 60292f6..0000000 --- a/containers/custom-nodejs-builder-amd64/drivah.toml +++ /dev/null @@ -1,13 +0,0 @@ -[container.image] -names = ["docker.elastic.co/ci-agent-images/kibana/custom-nodejs-builder"] -tags = ["amd64-0.1"] - -[container.image.build_args] -GROUP_ID = 1000 -USER_ID = 1000 - -[docker] -build_flags = [ - "--progress=plain", - "--platform=linux/amd64" -] diff --git a/containers/custom-nodejs-builder-amd64/entrypoint.sh b/containers/custom-nodejs-builder-amd64/entrypoint.sh deleted file mode 100644 index fffa7f4..0000000 --- a/containers/custom-nodejs-builder-amd64/entrypoint.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -release_url_base="$1" -full_version="$2" -config_flags=${3:-""} #"--without-dtrace --without-npm --without-etw" - -if [[ $(arch) == x86_64 ]]; then - architecture="x64"; -else - architecture="arm64" -fi - -cd "/home/node/workdir/src/node-${full_version}" - -# Compile from source -export CCACHE_DIR="/home/node/workdir/.ccache-${architecture}" -export CC="ccache gcc" -export CXX="ccache g++" - -. /opt/rh/devtoolset-9/enable - -make -j"$(getconf _NPROCESSORS_ONLN)" binary V= \ - DESTCPU="$architecture" \ - ARCH="$architecture" \ - VARIATION="glibc-217" \ - DISTTYPE="release" \ - RELEASE_URLBASE="$release_url_base" \ - CONFIG_FLAGS="$config_flags" - -mkdir -p /home/node/workdir/dist/ -mv node-*.tar.?z /home/node/workdir/dist/ diff --git a/containers/custom-nodejs-builder-amd64/re2_entrypoint.sh b/containers/custom-nodejs-builder-amd64/re2_entrypoint.sh deleted file mode 100644 index a8165d2..0000000 --- a/containers/custom-nodejs-builder-amd64/re2_entrypoint.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -re2_full_version="$1" -node_full_version="$2" -node_download_base_url="$3" - -if [[ $(arch) == x86_64 ]]; then - architecture="x64"; -else - architecture="arm64" -fi - -cd /home/node/workdir -mkdir -p dist/ -mkdir -p src/ - -## Download and unpack Node.js binary if needed. -node_folder_name="node-${node_full_version}-linux-${architecture}" -npm_binary="/home/node/workdir/${node_folder_name}/bin/npm" -if [ ! -f "$npm_binary" ]; then - if [ ! -f "/home/node/workdir/$node_folder_name.tar.xz" ]; then - curl -fsSLO --compressed "${node_download_base_url}/${node_folder_name}.tar.xz" - fi - - tar -xf "/home/node/workdir/${node_folder_name}.tar.xz" -fi - -cd src - -## Download re2 source if needed. -re2_source_folder="/home/node/workdir/src/node-re2-${re2_full_version}" -if [ ! -d "$re2_source_folder" ]; then - git clone --recurse-submodules --depth 1 --branch 1.17.4 https://github.com/uhop/node-re2.git "${re2_source_folder}" -fi - -cd "$re2_source_folder" -export PATH="/home/node/workdir/${node_folder_name}/bin:$PATH" -export DEVELOPMENT_SKIP_GETTING_ASSET=true - -export CCACHE_DIR="/home/node/workdir/.ccache-re2-${architecture}" -export CC="ccache gcc" -export CXX="ccache g++" - -. /opt/rh/devtoolset-9/enable - -npm i --unsafe-perm=true -npm run build --if-present -npm test - -mkdir -p /home/node/workdir/dist/ -cp "${re2_source_folder}/build/Release/re2.node" "/home/node/workdir/dist/linux-${architecture}-108" -gzip -f "/home/node/workdir/dist/linux-${architecture}-108" diff --git a/containers/custom-nodejs-builder-arm64/Dockerfile b/containers/custom-nodejs-builder-arm64/Dockerfile deleted file mode 100644 index a1d6faa..0000000 --- a/containers/custom-nodejs-builder-arm64/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM centos:7 - -ARG GROUP_ID=1000 -ARG USER_ID=1000 - -RUN groupadd --force --gid $GROUP_ID node -RUN adduser --gid $GROUP_ID --uid $USER_ID node - -RUN ulimit -n 1024 \ - && yum install -y epel-release \ - && yum install -y centos-release-scl-rh \ - && yum upgrade -y \ - && yum install -y \ - git \ - curl \ - make \ - python2 \ - python3 \ - ccache \ - xz-utils \ - devtoolset-9 \ - glibc-devel - -COPY --chown=node:node entrypoint.sh /home/node/entrypoint.sh -COPY --chown=node:node re2_entrypoint.sh /home/node/re2_entrypoint.sh - -USER node - -VOLUME /home/node/workdir - -ENTRYPOINT [ "/home/node/entrypoint.sh" ] diff --git a/containers/custom-nodejs-builder-arm64/drivah.toml b/containers/custom-nodejs-builder-arm64/drivah.toml deleted file mode 100644 index 1b040fb..0000000 --- a/containers/custom-nodejs-builder-arm64/drivah.toml +++ /dev/null @@ -1,15 +0,0 @@ -[container.image] -names = ["docker.elastic.co/ci-agent-images/kibana/custom-nodejs-builder"] -tags = ["arm64-0.1"] - -[container.image.build_args] -GROUP_ID = 1000 -USER_ID = 1000 - -[docker] -build_flags = [ - "--progress=plain", - "--platform=linux/arm64", - "--build-arg=\"GROUP_ID=1000\"", - "--build-arg=\"USER_ID=1000\"" -] diff --git a/containers/custom-nodejs-builder-arm64/re2_entrypoint.sh b/containers/custom-nodejs-builder-arm64/re2_entrypoint.sh deleted file mode 100644 index a8165d2..0000000 --- a/containers/custom-nodejs-builder-arm64/re2_entrypoint.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -re2_full_version="$1" -node_full_version="$2" -node_download_base_url="$3" - -if [[ $(arch) == x86_64 ]]; then - architecture="x64"; -else - architecture="arm64" -fi - -cd /home/node/workdir -mkdir -p dist/ -mkdir -p src/ - -## Download and unpack Node.js binary if needed. -node_folder_name="node-${node_full_version}-linux-${architecture}" -npm_binary="/home/node/workdir/${node_folder_name}/bin/npm" -if [ ! -f "$npm_binary" ]; then - if [ ! -f "/home/node/workdir/$node_folder_name.tar.xz" ]; then - curl -fsSLO --compressed "${node_download_base_url}/${node_folder_name}.tar.xz" - fi - - tar -xf "/home/node/workdir/${node_folder_name}.tar.xz" -fi - -cd src - -## Download re2 source if needed. -re2_source_folder="/home/node/workdir/src/node-re2-${re2_full_version}" -if [ ! -d "$re2_source_folder" ]; then - git clone --recurse-submodules --depth 1 --branch 1.17.4 https://github.com/uhop/node-re2.git "${re2_source_folder}" -fi - -cd "$re2_source_folder" -export PATH="/home/node/workdir/${node_folder_name}/bin:$PATH" -export DEVELOPMENT_SKIP_GETTING_ASSET=true - -export CCACHE_DIR="/home/node/workdir/.ccache-re2-${architecture}" -export CC="ccache gcc" -export CXX="ccache g++" - -. /opt/rh/devtoolset-9/enable - -npm i --unsafe-perm=true -npm run build --if-present -npm test - -mkdir -p /home/node/workdir/dist/ -cp "${re2_source_folder}/build/Release/re2.node" "/home/node/workdir/dist/linux-${architecture}-108" -gzip -f "/home/node/workdir/dist/linux-${architecture}-108" diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh new file mode 100755 index 0000000..8b80773 --- /dev/null +++ b/scripts/build_nodejs.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +source ./scripts/common.sh + +# TARGET_VERSION provided in env +# ARCH provided in env +assert_correct_arch $ARCH + +TARGET_NODE_VERSION="v$TARGET_VERSION" +BUILD_IMAGE_NAME=$(get_build_image_name) +TARGET_PLATFORM="linux/$ARCH" +RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" + +echo "Running node.js build in folder: `pwd`" + +echo '--- Downloading node source' +retry 5 15 curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ + https://nodejs.org/download/release/$TARGET_NODE_VERSION/node-$TARGET_NODE_VERSION.tar.xz +tar -xf ./workdir/src/node-$TARGET_NODE_VERSION.tar.xz -C ./workdir/src +chmod -R a+rwx ./workdir/ + + +echo "--- Buidling node for $TARGET_PLATFORM" +docker run --rm -it --platform $TARGET_PLATFORM \ + -v ./workdir:/home/node/workdir:Z \ + $BUILD_IMAGE_NAME \ + $RELEASE_URL_BASE \ + $TARGET_NODE_VERSION diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 0000000..c324ec2 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,74 @@ + +BUCKET_NAME="kibana-custom-node-artifacts" + +export TARGET_VERSION=${OVERRIDE_TARGET_VERSION:-$TARGET_VERSION} + +function assert_correct_arch() { + ARCH=$1 + + if [[ "$ARCH" == "arm64" || "$ARCH" == "amd64" ]]; then + # we're good, supported architecture + echo "Building for architecture: $ARCH" + else + echo "ARCH (=$ARCH) env variable is not one of: arm64, amd64" + exit 1 + fi +} + +function get_build_image_name() { + NODE_VERSION=${1:-$TARGET_VERSION} + PLATFORM=${2:-$ARCH} + + echo "docker.elastic.co/elastic/nodejs-custom:$NODE_VERSION-$PLATFORM" +} + +function retry() { + local retries=$1; shift + local delay=$1; shift + local attempts=1 + + until "$@"; do + retry_exit_status=$? + echo "Exited with $retry_exit_status" >&2 + if (( retries == "0" )); then + return $retry_exit_status + elif (( attempts == retries )); then + echo "Failed $attempts retries" >&2 + return $retry_exit_status + else + echo "Retrying $((retries - attempts)) more times..." >&2 + attempts=$((attempts + 1)) + sleep "$delay" + fi + done +} + +function replace_shasums_in_folder() { + local working_directory=$1 + + cd $working_directory + + # Check if SHASUMS256.txt file exists + if [ ! -f "./SHASUMS256.txt" ]; then + echo "SHASUMS256.txt file does not exist in folder: $working_directory" + exit 1 + fi + + # Loop through files in folder + for file in *; do + if [ "$file" != "SHASUMS256.txt" ] && [ -f "$file" ]; then + # Calculate SHA256 hash + new_sha256=$(shasum -a 256 "$file" | cut -d' ' -f1) + # Replace hashes in SHASUMS256.txt + node -e """ + lines = fs.readFileSync('SHASUMS256.txt').toString().split('\n') + output = lines.map(l => l.endsWith('$file') ? '$new_sha256 $file' : l) + fs.writeFileSync('SHASUMS256.txt', output.join('\n')) + """ + fi + done + + echo "`pwd`/SHASUMS256.txt updated" + + cd - +} diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh new file mode 100755 index 0000000..1798eb4 --- /dev/null +++ b/scripts/create_build_images.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail + +source ./scripts/common.sh + +# TARGET_VERSION provided in env +# ARCH provided in env +assert_correct_arch $ARCH + +TARGET_PLATFORM="linux/$ARCH" +IMAGE_NAME=$(get_build_image_name) +SCRIPT_DIR=$(dirname $0) +DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" + + +echo "--- Creating multiarch driver" +if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then + docker buildx create --name multiarch --driver docker-container --use +fi + + +echo "--- Building node.js build image ($ARCH)" +DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ + --platform $TARGET_PLATFORM \ + --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ + --load \ + --tag $IMAGE_NAME \ + $DOCKER_BUILD_CONTEXT_DIR diff --git a/scripts/replace_sha_hashes.sh b/scripts/replace_sha_hashes.sh new file mode 100755 index 0000000..a77c759 --- /dev/null +++ b/scripts/replace_sha_hashes.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +source ./scripts/common.sh + +# TARGET_VERSION provided in env + +ARTIFACT_BASE_PATH="node-glibc-217/dist/v$TARGET_VERSION" +SHASUMS_LOCATION="https://nodejs.org/dist/v$TARGET_VERSION/SHASUMS256.txt" + +ARTIFACT_DIST_DIR="./workdir/download" + +echo "--- Downloading node.js $TARGET_VERSION glibc-217 artifacts" +mkdir -p $ARTIFACT_DIST_DIR + +gsutil cp -r gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH/* "$ARTIFACT_DIST_DIR/" + +echo "--- Downloading SHASUMS256.txt" +retry 5 10 curl -fsSLO $SHASUMS_LOCATION +mv SHASUMS256.txt $ARTIFACT_DIST_DIR/SHASUMS256.txt + +echo "--- Replacing checksums with local file hashes" +replace_shasums_in_folder $ARTIFACT_DIST_DIR + +echo "--- Uploading SHASUMS256.txt to $BUCKET_NAME" +gsutil cp -r "$ARTIFACT_DIST_DIR/SHASUMS256.txt" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH/ diff --git a/scripts/upload_nodejs_artifacts.sh b/scripts/upload_nodejs_artifacts.sh new file mode 100755 index 0000000..4295aeb --- /dev/null +++ b/scripts/upload_nodejs_artifacts.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +source ./scripts/common.sh + +# TARGET_VERSION provided in env + +ARTIFACT_BASE_PATH="node-glibc-217/dist/v$TARGET_VERSION/" +ARTIFACT_DIST_DIR="./workdir/dist" + +echo "--- Uploading build artifacts" +gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH