Merge remote-tracking branch 'upstream/master' into packed-edge-count

.github/workflows/testmac.yml

@@ -50,7 +50,15 @@ jobs:
         # to avoid problems with ABI changes with/without -DNDEBUG.
         # And we update libomp to make extra sure it will be picked up by the compiler.
         # We pre-install a pinned txm to work around https://github.com/anko/txm/issues/8
-        run: brew bundle cleanup --force && brew bundle install && brew update && brew install protobuf && brew install libomp && npm install -g [email protected]
+        run: |
+          brew bundle cleanup --force && \
+          brew bundle install && \
+          brew update && \
+          brew install protobuf && \
+          brew install libomp && \
+          npm install -g [email protected] && \
+          brew config && \
+          (brew doctor || echo "brew doctor is unhappy")
 
       - name: Run build and test
         run: |

.gitlab-ci.yml

@@ -24,7 +24,20 @@ before_script:
         echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json
       fi
     fi
+  # Restart or start the Docker daemon
+  - stopdocker || true
   - startdocker || true
+  # Get buildx
+  - mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 >  ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx
+  # Connect to the Kubernetes-based builder "buildkit" if appropriate
+  # See vgci/buildkit-deployment.yml
+  - if [[ "${CI_BUILDKIT_DRIVER}" == "kubernetes" ]] ; then docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" ; else docker buildx create --use --name=container-builder --driver=docker-container ; fi
+  # Report on the builders, and make sure they exist.
+  - docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found! Are we on the right Gitlab runner?" && exit 1)
+  # Prune down build cache to make space. This will hang if the builder isn't findable.
+  - (echo "y" | docker buildx prune --keep-storage 80G) || true
+  # Connect so we can upload our images
+  - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   - docker info
   - mkdir -p ~/.aws && cp "$GITLAB_SECRET_FILE_AWS_CREDENTIALS" ~/.aws/credentials
 
@@ -93,30 +106,33 @@ local-build-test-job:
 # We define one job to do the Docker container build
 build-job:
   stage: build
-  before_script:
-    # Don't bother starting the Docker daemon or installing much
-    - which docker || (sudo apt-get -q -y update && sudo apt-get -q -y install --no-upgrade docker.io)
-    # Get buildx
-    - mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.6.3/buildx-v0.6.3.linux-amd64 >  ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx
-    # Connect to the Kubernetes-based builder "buildkit"
-    # See vgci/buildkit-deployment.yml
-    - docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64"
-    # Report on the builders, and make sure they exist.
-    - docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found in our Kubernetes namespace! Are we on the right Gitlab runner?" && exit 1)
-    # Prune down build cache to make space. This will hang if the builder isn't findable.
-    - (echo "y" | docker buildx prune --keep-storage 80G) || true
   script:
+    - CI_REPO=${CI_REGISTRY}/vgteam/vg
+    - CACHE_TAG="cache-$(echo ${CI_COMMIT_BRANCH}${CI_COMMIT_TAG} | tr '/' '-')"
+    - MAINLINE_CACHE_TAG="cache-master"
+    - PLATFORMS=linux/amd64
+    - DOCKER_TAG=ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}
     - make include/vg_git_version.hpp
     - cat include/vg_git_version.hpp
-    # Build but don't push, just fill the cache
-    - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run -f Dockerfile .
-    # Run the tests
-    - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target test -f Dockerfile .
     # Connect so we can upload our images
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    # Keep trying to push until it works or we time out or run out of tries
-    - COUNT=0
-    - while ! docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run --push -t "quay.io/vgteam/vg:ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}" -f Dockerfile . ; do docker logout "${CI_REGISTRY}" ; sleep 30; docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" ; sleep 30; ((COUNT+=1)); if [[ ${COUNT} == 10 ]] ; then exit 1; fi; done
+    # Note that A LOCAL CACHE CAN ONLY HOLD ONE TAG/TARGET AT A TIME!
+    # And Quay can't use mode=max registry caching to cache the intermediate targets with the final image, just inline caching.
+    # So we have to do the Complicated Cache Shuffle.
+    # Build base image from branch and mainline base caches to local cache
+    - docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
+    # Push base image from local cache to registry cache for branch.
+    - docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
+    # Build build image from local base cache and branch and mainline build caches to local cache
+    - docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
+    # Push build image to registry cache for branch
+    - docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
+    # Build run image from local build cache and branch and mainline run caches to local cache
+    - docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
+    # Push run image to registry cache for branch
+    - docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
+    # Finally, push run image to where we actually want it.
+    - docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
   variables:
     GIT_SUBMODULE_STRATEGY: recursive
 
@@ -130,28 +146,36 @@ production-build-job:
     - /^arm/
     - master
     - tags
-  before_script:
-    # Don't bother starting the Docker daemon or installing much
-    - which docker || (sudo apt-get -q -y update && sudo apt-get -q -y install --no-upgrade docker.io)
-    # Get buildx
-    - mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 >  ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx
-    # Connect to the Kubernetes-based builder "buildkit"
-    # See vgci/buildkit-deployment.yml
-    - docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64"
-    # Report on the builders, and make sure they exist.
-    - docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found in our Kubernetes namespace! Are we on the right Gitlab runner?" && exit 1)
-    # Prune down build cache to make space. This will hang if the builder isn't findable.
-    - docker buildx prune --keep-storage 80G
-    # Connect so we can upload our images
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
-    - make include/vg_git_version.hpp
+    - CI_REPO=${CI_REGISTRY}/vgteam/vg
+    - CACHE_TAG="cache-$(echo ${CI_COMMIT_BRANCH}${CI_COMMIT_TAG} | tr '/' '-')"
+    - MAINLINE_CACHE_TAG="cache-master"
+    - PLATFORMS=linux/amd64,linux/arm64
     # Determine what we should be tagging vg Dockers as. If we're running on a Git tag we want to use that. Otherwise push over the tag we made already.
-    - if [[ ! -z "${CI_COMMIT_TAG}" ]]; then VG_DOCKER_TAG="${CI_COMMIT_TAG}" ; else VG_DOCKER_TAG="ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}"; fi
-    # Build the container for all architectures.
-    - docker buildx build --platform=linux/amd64,linux/arm64 --build-arg THREADS=8 --target run --push -t "quay.io/vgteam/vg:${VG_DOCKER_TAG}" -f Dockerfile .
+    - if [[ ! -z "${CI_COMMIT_TAG}" ]]; then DOCKER_TAG="${CI_COMMIT_TAG}" ; else DOCKER_TAG="ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}"; fi
+    - make include/vg_git_version.hpp
+    # Make sure ARM emulation is available.
+    - if [[ "${CI_BUILDKIT_DRIVER}" != "kubernetes" ]] ; then docker run --privileged --rm tonistiigi/binfmt --install all || true ; fi
+    # TODO: deduplicate this code with normal build above
+    # Note that A LOCAL CACHE CAN ONLY HOLD ONE TAG/TARGET AT A TIME!
+    # And Quay can't use mode=max registry caching to cache the intermediate targets with the final image, just inline caching.
+    # So we have to do the Complicated Cache Shuffle.
+    # Build base image from branch and mainline base caches to local cache
+    - docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
+    # Push base image from local cache to registry cache for branch.
+    - docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
+    # Build build image from local base cache and branch and mainline build caches to local cache
+    - docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
+    # Push build image to registry cache for branch
+    - docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
+    # Build run image from local build cache and branch and mainline run caches to local cache
+    - docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
+    # Push run image to registry cache for branch
+    - docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
+    # Finally, push run image to where we actually want it.
+    - docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
     # Tag it latest if we pushed a real release tag
-    - if [[ ! -z "${CI_COMMIT_TAG}" ]]; then docker buildx build --platform=linux/amd64,linux/arm64 --build-arg THREADS=8 --target run --push -t "quay.io/vgteam/vg:latest" -f Dockerfile .; fi
+    - if [[ ! -z "${CI_COMMIT_TAG}" ]]; then --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=8 --target run -t ${CI_REPO}:latest -f Dockerfile --push .; fi
     # Also run the ARM tests (emulated!)
     # But don't fail if they fail yet, because they don't yet actually work.
     - docker buildx build --platform=linux/arm64 --build-arg THREADS=8 --target test -f Dockerfile . || true

doc/publish-docs.sh

@@ -33,8 +33,13 @@ make docs
 # Get ready to deploy the docs
 
 # Make a scratch directory *outside* our normal git repo
-SCRATCH_DIR="$(pwd)/../tmp"
-mkdir -p "${SCRATCH_DIR}"
+SCRATCH_DIR="$(mktemp -d)"
+# And clean it up when we stop
+function cleanup {
+  rm -Rf ${SCRATCH_DIR}
+}
+trap cleanup EXIT
+
 
 
 # Set up our SSH key

src/clip.cpp

@@ -529,12 +529,11 @@ void clip_low_depth_nodes_and_edges_generic(MutablePathMutableHandleGraph* graph
     }
 
     // now do the edges
-    size_t edge_count = graph->get_edge_count();
     vector<edge_t> edges;
-    edges.reserve(edge_count);
     graph->for_each_edge([&](edge_t edge) {
             edges.push_back(edge);
         });
+    size_t edge_count = edges.size();
     boomphf::mphf<edge_t, BBEdgeHash> edge_hash(edge_count, edges, get_thread_count(), 2.0, false, false);
     edges.clear();
     bdsg::PackedVector<> edge_depths;