Skip to content

Commit

Permalink
Merge pull request #4072 from vgteam/arm-ci-build
Browse files Browse the repository at this point in the history 
Restrict CI to resources available
  • Loading branch information
@adamnovak
adamnovak authored Sep 7, 2023
2 parents 16bd80e + 56877a1 commit bcd5712
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 29 deletions.
62 changes: 36 additions & 26 deletions .gitlab-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,18 @@ before_script:
# Configure Docker to use a mirror for Docker Hub and restart the daemon
- |
if [[ ! -z "${DOCKER_HUB_MIRROR}" ]] ; then
export SINGULARITY_DOCKER_HUB_MIRROR="${DOCKER_HUB_MIRROR}"
echo "[registry.\"docker.io\"]" >buildkitd.toml
echo " mirrors = [\"${DOCKER_HUB_MIRROR##*://}\"]" >>buildkitd.toml
if [[ "${DOCKER_HUB_MIRROR}" == https* ]] ; then
# Set up a secure mirror
echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"]}" | sudo tee /etc/docker/daemon.json
else
# Set up an insecure mirror
echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json
echo "[registry.\"${DOCKER_HUB_MIRROR##*://}\"]" >>buildkitd.toml
echo " http = true" >>buildkitd.toml
echo " insecure = true" >>buildkitd.toml
fi
fi
# Restart or start the Docker daemon
Expand All @@ -31,7 +37,7 @@ before_script:
- mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 > ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx
# Connect to the Kubernetes-based builder "buildkit" if appropriate
# See vgci/buildkit-deployment.yml
- if [[ "${CI_BUILDKIT_DRIVER}" == "kubernetes" ]] ; then docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" ; else docker buildx create --use --name=container-builder --driver=docker-container ; fi
- if [[ "${CI_BUILDKIT_DRIVER}" == "kubernetes" ]] ; then docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" ; else docker buildx create --use --name=container-builder --driver=docker-container --config ./buildkitd.toml ; fi
# Report on the builders, and make sure they exist.
- docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found! Are we on the right Gitlab runner?" && exit 1)
# Prune down build cache to make space. This will hang if the builder isn't findable.
Expand Down Expand Up @@ -78,16 +84,17 @@ local-build-test-job:
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash
- export NVM_DIR="$HOME/.nvm" && . "$NVM_DIR/nvm.sh"
script:
- THREADS=8
- nvm version
- python3 ./configure.py
- source ./source_me.sh
- make get-deps
- make -j8
- make -j${THREADS}
- echo Testing
- bin/vg test "Target to alignment extraction"
- echo Full Testing
- make test
- make static -j8
- make static -j${THREADS}
# Also test as a backend for the tube map
- git clone https://github.com/vgteam/sequenceTubeMap.git
- cd sequenceTubeMap
Expand All @@ -113,6 +120,7 @@ build-job:
- CACHE_TAG="cache-$(echo ${CI_COMMIT_BRANCH}${CI_COMMIT_TAG} | tr '/' '-')"
- MAINLINE_CACHE_TAG="cache-master"
- PLATFORMS=linux/amd64
- THREADS=8
- DOCKER_TAG=ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}
- make include/vg_git_version.hpp
- cat include/vg_git_version.hpp
Expand All @@ -121,20 +129,20 @@ build-job:
# Note that A LOCAL CACHE CAN ONLY HOLD ONE TAG/TARGET AT A TIME!
# And Quay can't use mode=max registry caching to cache the intermediate targets with the final image, just inline caching.
# So we have to do the Complicated Cache Shuffle.
# Build base image from branch and mainline base caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
# Push base image from local cache to registry cache for branch.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
# Build build image from local base cache and branch and mainline build caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
# Push build image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
# Build run image from local build cache and branch and mainline run caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
# Push run image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
# Finally, push run image to where we actually want it.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
- echo " Build base image from branch and mainline base caches to local cache"
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-from type=local,src=${HOME}/docker-cache/base --cache-to type=local,dest=${HOME}/docker-cache/base --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
- echo "Push base image from local cache to registry cache for branch."
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
- echo "Build build image from local base cache and branch and mainline build caches to local cache"
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-to type=local,dest=${HOME}/docker-cache/build --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
- echo "Push build image to registry cache for branch"
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
- echo "Build run image from local build cache and branch and mainline run caches to local cache"
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=local,dest=${HOME}/docker-cache/run --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
- echo "Push run image to registry cache for branch"
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
- echo "Finally, push run image to where we actually want it."
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
variables:
GIT_SUBMODULE_STRATEGY: recursive

Expand All @@ -148,11 +156,13 @@ production-build-job:
- /^arm/
- master
- tags
timeout: 12h
script:
- CI_REPO=${CI_REGISTRY}/vgteam/vg
- CACHE_TAG="cache-$(echo ${CI_COMMIT_BRANCH}${CI_COMMIT_TAG} | tr '/' '-')"
- MAINLINE_CACHE_TAG="cache-master"
- PLATFORMS=linux/amd64,linux/arm64
- THREADS=8 # Oversubscribe since the ARM build will take way longer anyway.
# Determine what we should be tagging vg Dockers as. If we're running on a Git tag we want to use that. Otherwise push over the tag we made already.
- if [[ ! -z "${CI_COMMIT_TAG}" ]]; then DOCKER_TAG="${CI_COMMIT_TAG}" ; else DOCKER_TAG="ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}"; fi
- make include/vg_git_version.hpp
Expand All @@ -163,23 +173,23 @@ production-build-job:
# And Quay can't use mode=max registry caching to cache the intermediate targets with the final image, just inline caching.
# So we have to do the Complicated Cache Shuffle.
# Build base image from branch and mainline base caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-from type=local,src=${HOME}/docker-cache/base --cache-to type=local,dest=${HOME}/docker-cache/base --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
# Push base image from local cache to registry cache for branch.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
# Build build image from local base cache and branch and mainline build caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-to type=local,dest=${HOME}/docker-cache/build --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
# Push build image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
# Build run image from local build cache and branch and mainline run caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=local,dest=${HOME}/docker-cache/run --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
# Push run image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
# Finally, push run image to where we actually want it.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
# Tag it latest if we pushed a real release tag
- if [[ ! -z "${CI_COMMIT_TAG}" ]]; then docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:latest -f Dockerfile --push .; fi
- if [[ ! -z "${CI_COMMIT_TAG}" ]]; then docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:latest -f Dockerfile --push .; fi
# If we wanted to run the tests under ARM emulation, we could do:
# docker buildx build --platform=linux/arm64 --build-arg THREADS=8 --target test -f Dockerfile .
# docker buildx build --platform=linux/arm64 --build-arg THREADS=${THREADS} --target test -f Dockerfile .
# But we don't, because they both don't actually pass yet on ARM and also
# manage to hit a 6 hour timeout on our extremely slow emulators.
variables:
Expand Down
2 changes: 1 addition & 1 deletion deps/backward-cpp
Submodule backward-cpp updated 14 files
+2 −0 .clang-format
+126 −38 BackwardConfig.cmake
+34 −7 CMakeLists.txt
+61 −8 README.md
+11 −1 backward.cpp
+3,942 −1,825 backward.hpp
+202 −107 test/_test_main.cpp
+45 −46 test/rectrace.cpp
+15 −15 test/select_signals.cpp
+19 −27 test/stacktrace.cpp
+37 −47 test/suicide.cpp
+21 −33 test/test.cpp
+115 −102 test/test.hpp
+28 −42 test_package/main.cpp
2 changes: 1 addition & 1 deletion deps/gbwtgraph
Submodule gbwtgraph updated 1 files
+0 −4 src/algorithms.cpp
2 changes: 1 addition & 1 deletion deps/libbdsg
Submodule libbdsg updated 2 files
+31 −0 bdsg/include/bdsg/snarl_distance_index.hpp
+266 −12 bdsg/src/snarl_distance_index.cpp

1 comment on commit bcd5712

@adamnovak
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

vg CI tests complete for merge to master. View the full report here.

16 tests passed, 0 tests failed and 0 tests skipped in 17328 seconds

Please sign in to comment.