Skip to content

Commit

Permalink
Restrict CI to resources available
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamnovak
adamnovak committed Sep 5, 2023
1 parent 647d935 commit 112e11c
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 22 deletions.
48 changes: 29 additions & 19 deletions .gitlab-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,18 @@ before_script:
# Configure Docker to use a mirror for Docker Hub and restart the daemon
- |
if [[ ! -z "${DOCKER_HUB_MIRROR}" ]] ; then
export SINGULARITY_DOCKER_HUB_MIRROR="${DOCKER_HUB_MIRROR}"
echo "[registry.\"docker.io\"]" >buildkitd.toml
echo " mirrors = [\"${DOCKER_HUB_MIRROR##*://}\"]" >>buildkitd.toml
if [[ "${DOCKER_HUB_MIRROR}" == https* ]] ; then
# Set up a secure mirror
echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"]}" | sudo tee /etc/docker/daemon.json
else
# Set up an insecure mirror
echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json
echo "[registry.\"${DOCKER_HUB_MIRROR##*://}\"]" >>buildkitd.toml
echo " http = true" >>buildkitd.toml
echo " insecure = true" >>buildkitd.toml
fi
fi
# Restart or start the Docker daemon
Expand All @@ -31,7 +37,7 @@ before_script:
- mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 > ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx
# Connect to the Kubernetes-based builder "buildkit" if appropriate
# See vgci/buildkit-deployment.yml
- if [[ "${CI_BUILDKIT_DRIVER}" == "kubernetes" ]] ; then docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" ; else docker buildx create --use --name=container-builder --driver=docker-container ; fi
- if [[ "${CI_BUILDKIT_DRIVER}" == "kubernetes" ]] ; then docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" ; else docker buildx create --use --name=container-builder --driver=docker-container --config ./buildkitd.toml ; fi
# Report on the builders, and make sure they exist.
- docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found! Are we on the right Gitlab runner?" && exit 1)
# Prune down build cache to make space. This will hang if the builder isn't findable.
Expand Down Expand Up @@ -78,16 +84,17 @@ local-build-test-job:
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash
- export NVM_DIR="$HOME/.nvm" && . "$NVM_DIR/nvm.sh"
script:
- THREADS=4
- nvm version
- python3 ./configure.py
- source ./source_me.sh
- make get-deps
- make -j8
- make -j${THREADS}
- echo Testing
- bin/vg test "Target to alignment extraction"
- echo Full Testing
- make test
- make static -j8
- make static -j${THREADS}
# Also test as a backend for the tube map
- git clone https://github.com/vgteam/sequenceTubeMap.git
- cd sequenceTubeMap
Expand All @@ -113,6 +120,7 @@ build-job:
- CACHE_TAG="cache-$(echo ${CI_COMMIT_BRANCH}${CI_COMMIT_TAG} | tr '/' '-')"
- MAINLINE_CACHE_TAG="cache-master"
- PLATFORMS=linux/amd64
- THREADS=4
- DOCKER_TAG=ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}
- make include/vg_git_version.hpp
- cat include/vg_git_version.hpp
Expand All @@ -122,19 +130,19 @@ build-job:
# And Quay can't use mode=max registry caching to cache the intermediate targets with the final image, just inline caching.
# So we have to do the Complicated Cache Shuffle.
# Build base image from branch and mainline base caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
# Push base image from local cache to registry cache for branch.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
# Build build image from local base cache and branch and mainline build caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
# Push build image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
# Build run image from local build cache and branch and mainline run caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
# Push run image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
# Finally, push run image to where we actually want it.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
variables:
GIT_SUBMODULE_STRATEGY: recursive

Expand All @@ -148,11 +156,13 @@ production-build-job:
- /^arm/
- master
- tags
timeout: 12h
script:
- CI_REPO=${CI_REGISTRY}/vgteam/vg
- CACHE_TAG="cache-$(echo ${CI_COMMIT_BRANCH}${CI_COMMIT_TAG} | tr '/' '-')"
- MAINLINE_CACHE_TAG="cache-master"
- PLATFORMS=linux/amd64,linux/arm64
- THREADS=2 # Since we run two builds at once, each gets fewer threads
# Determine what we should be tagging vg Dockers as. If we're running on a Git tag we want to use that. Otherwise push over the tag we made already.
- if [[ ! -z "${CI_COMMIT_TAG}" ]]; then DOCKER_TAG="${CI_COMMIT_TAG}" ; else DOCKER_TAG="ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}"; fi
- make include/vg_git_version.hpp
Expand All @@ -163,23 +173,23 @@ production-build-job:
# And Quay can't use mode=max registry caching to cache the intermediate targets with the final image, just inline caching.
# So we have to do the Complicated Cache Shuffle.
# Build base image from branch and mainline base caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
# Push base image from local cache to registry cache for branch.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
# Build build image from local base cache and branch and mainline build caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
# Push build image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
# Build run image from local build cache and branch and mainline run caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
# Push run image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
# Finally, push run image to where we actually want it.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
# Tag it latest if we pushed a real release tag
- if [[ ! -z "${CI_COMMIT_TAG}" ]]; then docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:latest -f Dockerfile --push .; fi
- if [[ ! -z "${CI_COMMIT_TAG}" ]]; then docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:latest -f Dockerfile --push .; fi
# If we wanted to run the tests under ARM emulation, we could do:
# docker buildx build --platform=linux/arm64 --build-arg THREADS=8 --target test -f Dockerfile .
# docker buildx build --platform=linux/arm64 --build-arg THREADS=${THREADS} --target test -f Dockerfile .
# But we don't, because they both don't actually pass yet on ARM and also
# manage to hit a 6 hour timeout on our extremely slow emulators.
variables:
Expand Down
2 changes: 1 addition & 1 deletion deps/backward-cpp
Submodule backward-cpp updated 14 files
+2 −0 .clang-format
+126 −38 BackwardConfig.cmake
+34 −7 CMakeLists.txt
+61 −8 README.md
+11 −1 backward.cpp
+3,942 −1,825 backward.hpp
+202 −107 test/_test_main.cpp
+45 −46 test/rectrace.cpp
+15 −15 test/select_signals.cpp
+19 −27 test/stacktrace.cpp
+37 −47 test/suicide.cpp
+21 −33 test/test.cpp
+115 −102 test/test.hpp
+28 −42 test_package/main.cpp
2 changes: 1 addition & 1 deletion deps/gbwtgraph
Submodule gbwtgraph updated 1 files
+0 −4 src/algorithms.cpp
2 changes: 1 addition & 1 deletion deps/libbdsg
Submodule libbdsg updated 2 files
+31 −0 bdsg/include/bdsg/snarl_distance_index.hpp
+266 −12 bdsg/src/snarl_distance_index.cpp

1 comment on commit 112e11c

@adamnovak
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

vg CI tests complete for branch arm-ci-build. View the full report here.

16 tests passed, 0 tests failed and 0 tests skipped in 25870 seconds

Please sign in to comment.