chore: Cache files downloaded from cache

To avoid a roundtrip to S3 to re-download every artifact on
cache_download, we store a local file with the file prefix that points
to the latest downloaded file version (ie hash), so we don't re-donwload
it we already have it.

This required changing the arity of the cache_download and cache_upload
scripts, so they accept the key and version for each artifact
separately.

In addition, to avoid issues with the nargo `show-program-hash` being
indeterministic, we also store "symlink" files in the cache for each
circuit and vk. These are text files named after the entire hash of the
circuits folder, that contain the nargo-computed hash of the circuit.
This acts as another key for the same artifact.

Earthfile

@@ -264,12 +264,13 @@ rollup-verifier-contract-with-cache:
   FROM +bootstrap
   ENV CI=1
   ENV USE_CACHE=1
-  LET artifact=rollup-verifier-contract-$(./noir-projects/bootstrap.sh hash).tar.gz
+  LET artifact_hash=$(./noir-projects/bootstrap.sh hash)
   # Running this directly in the 'if' means files are not permanent
-  RUN ci3/cache_download rollup-verifier-contract-3e3a78f9a68f1f1e04240acf0728522d87a313ac-linux-gnu-x86_64 || true
+  # TODO(palla/cache): Shouldn't the hash below be the artifact_hash?
+  RUN ci3/cache_download rollup-verifier-contract 3e3a78f9a68f1f1e04240acf0728522d87a313ac-linux-gnu-x86_64 || true
   IF ! [ -d /usr/src/bb ]
     COPY --dir +rollup-verifier-contract/usr/src/bb /usr/src
-    RUN ci3/cache_upload $artifact bb
+    RUN ci3/cache_upload rollup-verifier-contract artifact_hash bb
   END
   SAVE ARTIFACT /usr/src/bb /usr/src/bb
 

avm-transpiler/bootstrap.sh

@@ -8,10 +8,9 @@ hash=$(cache_content_hash ../noir/.rebuild_patterns .rebuild_patterns)
 
 function build {
   github_group "avm-transpiler build"
-  artifact=avm-transpiler-$hash.tar.gz
-  if ! cache_download $artifact; then
+  if ! cache_download avm-transpiler $hash; then
     denoise ./scripts/bootstrap_native.sh
-    cache_upload $artifact target/release
+    cache_upload avm-transpiler $hash target/release
   fi
   github_endgroup
 }

barretenberg/cpp/bootstrap.sh

@@ -23,43 +23,43 @@ hash=$(cache_content_hash .rebuild_patterns)
 
 function build_native {
   set -eu
-  if ! cache_download barretenberg-release-$hash.tar.gz; then
+  if ! cache_download barretenberg-release $hash; then
     rm -f build/CMakeCache.txt
     echo "Building with preset: $preset"
     cmake --preset $preset -Bbuild
     cmake --build build --target bb
-    cache_upload barretenberg-release-$hash.tar.gz build/bin
+    cache_upload barretenberg-release $hash build/bin
   fi
 
   (cd src/barretenberg/world_state_napi && yarn --frozen-lockfile --prefer-offline)
-  if ! cache_download barretenberg-release-world-state-$hash.tar.gz; then
+  if ! cache_download barretenberg-release-world-state $hash; then
     rm -f build-pic/CMakeCache.txt
     cmake --preset $pic_preset -DCMAKE_BUILD_TYPE=RelWithAssert
     cmake --build --preset $pic_preset --target world_state_napi
-    cache_upload barretenberg-release-world-state-$hash.tar.gz build-pic/lib/world_state_napi.node
+    cache_upload barretenberg-release-world-state $hash build-pic/lib/world_state_napi.node
   fi
 }
 
 function build_wasm {
   set -eu
-  if ! cache_download barretenberg-wasm-$hash.tar.gz; then
+  if ! cache_download barretenberg-wasm $hash; then
     rm -f build-wasm/CMakeCache.txt
     cmake --preset wasm
     cmake --build --preset wasm
     /opt/wasi-sdk/bin/llvm-strip ./build-wasm/bin/barretenberg.wasm
-    cache_upload barretenberg-wasm-$hash.tar.gz build-wasm/bin
+    cache_upload barretenberg-wasm $hash build-wasm/bin
   fi
   (cd ./build-wasm/bin && gzip barretenberg.wasm -c > barretenberg.wasm.gz)
 }
 
 function build_wasm_threads {
   set -eu
-  if ! cache_download barretenberg-wasm-threads-$hash.tar.gz; then
+  if ! cache_download barretenberg-wasm-threads $hash; then
     rm -f build-wasm-threads/CMakeCache.txt
     cmake --preset wasm-threads
     cmake --build --preset wasm-threads
     /opt/wasi-sdk/bin/llvm-strip ./build-wasm-threads/bin/barretenberg.wasm
-    cache_upload barretenberg-wasm-threads-$hash.tar.gz build-wasm-threads/bin
+    cache_upload barretenberg-wasm-threads $hash build-wasm-threads/bin
   fi
   (cd ./build-wasm-threads/bin && gzip barretenberg.wasm -c > barretenberg.wasm.gz)
 }

barretenberg/ts/bootstrap.sh

@@ -7,12 +7,12 @@ hash=$(cache_content_hash ../cpp/.rebuild_patterns .rebuild_patterns)
 
 function build {
   github_group "bb.js build"
-  if ! cache_download bb.js-$hash.tar.gz; then
+  if ! cache_download bb.js $hash; then
     denoise yarn install
     find . -exec touch -d "@0" {} + 2>/dev/null || true
 
     denoise yarn build
-    cache_upload bb.js-$hash.tar.gz dest
+    cache_upload bb.js $hash dest
   else
     denoise yarn install
   fi

build-system/s3-cache-scripts/earthly-s3-cache.sh

@@ -8,34 +8,36 @@
 set -eu
 
 # definitions
-FILE="$prefix-$(cat .content-hash).tar.gz"
+KEY="$prefix"
+HASH="$(cat .content-hash)"
+
 function s3_download() {
   if [ "${S3_BUILD_CACHE_DOWNLOAD:-true}" = "false" ] || [ "${AWS_ACCESS_KEY_ID}" == "" ] ; then
     return 1 # require a rebuild
   fi
-  /usr/src/build-system/s3-cache-scripts/cache_download "$FILE"
+  /usr/src/build-system/s3-cache-scripts/cache_download "$KEY" "$HASH"
 }
 function s3_upload() {
   if [ "${S3_BUILD_CACHE_UPLOAD:-true}" = "false" ] || [ "${AWS_ACCESS_KEY_ID}" == "" ] ; then
     return 0 # exit silently
   fi
-  /usr/src/build-system/s3-cache-scripts/cache_upload "$FILE" $build_artifacts || echo "WARNING: S3 upload failed!" >&2
+  /usr/src/build-system/s3-cache-scripts/cache_upload "$KEY" "$HASH" $build_artifacts || echo "WARNING: S3 upload failed!" >&2
 }
 function minio_download() {
   if [ -z "$S3_BUILD_CACHE_MINIO_URL" ] ; then
     return 1 # require rebuild
   fi
   # minio is S3-compatible
   S3_BUILD_CACHE_AWS_PARAMS="--endpoint-url $S3_BUILD_CACHE_MINIO_URL" AWS_SECRET_ACCESS_KEY=minioadmin AWS_ACCESS_KEY_ID=minioadmin \
-    /usr/src/build-system/s3-cache-scripts/cache_download "$FILE"
+    /usr/src/build-system/s3-cache-scripts/cache_download "$KEY" "$HASH"
 }
 function minio_upload() {
   if [ -z "$S3_BUILD_CACHE_MINIO_URL" ] ; then
     return 0 # exit silently
   fi
   # minio is S3-compatible
   S3_BUILD_CACHE_AWS_PARAMS="--endpoint-url $S3_BUILD_CACHE_MINIO_URL" AWS_SECRET_ACCESS_KEY=minioadmin AWS_ACCESS_KEY_ID=minioadmin \
-    /usr/src/build-system/s3-cache-scripts/cache_upload "$FILE" $build_artifacts || echo "WARNING Minio upload failed!" >&2
+    /usr/src/build-system/s3-cache-scripts/cache_upload "$KEY" "$HASH" $build_artifacts || echo "WARNING Minio upload failed!" >&2
 }
 
 # commands

ci3/cache_download

@@ -2,8 +2,8 @@
 set -eu -o pipefail
 [ "${BUILD_SYSTEM_DEBUG:-}" = 1 ] && set -x
 
-if [ "$#" -lt 1 ]; then
-  echo "Usage: $0 <tar.gz_file_to_download_and_extract>" >&2
+if [ "$#" -lt 2 ]; then
+  echo "Usage: $0 <key> <hash>" >&2
   exit 1
 fi
 
@@ -12,18 +12,28 @@ if [ "${USE_CACHE:-0}" -lt 1 ]; then
   echo "Not using cache for $1 because USE_CACHE=0."
   exit 1
 fi
-# Get the tar.gz file name from the argument
-TAR_FILE="$1"
-OUT_DIR="${2:-.}"
+
+KEY="$1"
+HASH="$2"
+OUT_DIR="${3:-.}"
+
+TAR_FILE="$KEY-$HASH.tar.gz"
+CACHE_FILE="$OUT_DIR/$KEY.cache"
 
 mkdir -p "$OUT_DIR"
 # Extract endpoint URL if S3_BUILD_CACHE_AWS_PARAMS is set
 if [[ -n "${S3_BUILD_CACHE_AWS_PARAMS:-}" ]]; then
   aws $S3_BUILD_CACHE_AWS_PARAMS s3 cp "s3://aztec-ci-artifacts/build-cache/$TAR_FILE" "-" | tar -xzf - -C "$OUT_DIR" 2>/dev/null
 else
+  # Do not go to S3 if we have already downloaded this same file
+  if [[ -f "$CACHE_FILE" && "$(cat "$CACHE_FILE")" == $HASH ]]; then
+    echo "File $TAR_FILE already downloaded" >&2 && exit 0
+  fi
   # Default to AWS S3 URL if no custom endpoint is set
   S3_ENDPOINT="http://aztec-ci-artifacts.s3.amazonaws.com"
   # Attempt to download and extract the cache file
   (curl -s -f "$S3_ENDPOINT/build-cache/$TAR_FILE" | tar -xzf - -C "$OUT_DIR" 2>/dev/null) || (echo "Cache download of $TAR_FILE failed." >&2 && exit 1)
+  # Record locally that we have downloaded this file
+  echo $HASH > $CACHE_FILE
 fi
 echo "Cache download and extraction of $TAR_FILE complete." >&2

ci3/cache_upload

@@ -3,12 +3,15 @@
 set -eu
 
 if [ "$#" -lt 2 ]; then
-  echo "Usage: $0 <my-artifact.tar.gz> <binary_paths_to_tar_gz_and_upload...>" >&2
+  echo "Usage: $0 <key> <hash> <binary_paths_to_tar_gz_and_upload...>" >&2
   exit 1
 fi
 
-# Name, intended to have .tar.gz ending
-name="$1"
+# Key and hash, intended to form the full tar file name
+key="$1"
+hash="$2"
+name="$key-$hash.tar.gz"
+
 # Now $@ = our binary path args
 shift 1
 
@@ -18,7 +21,10 @@ if [ -z ${S3_FORCE_UPLOAD:-} ] && aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 ls "s3:/
 fi
 # Pipe tar directly to AWS S3 cp
 if tar -czf - "$@" | aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp - "s3://aztec-ci-artifacts/build-cache/$name" >&2 ; then
+  # Record locally that we have this file version, so we don't re-download
+  echo "$hash" > "$key.cache"
   echo "Cache upload of $name complete." >&2
+  echo
 else
   echo "Cache upload of $name failed." >&2
   exit 0

l1-contracts/bootstrap.sh

@@ -7,8 +7,7 @@ export hash=$(cache_content_hash .rebuild_patterns)
 
 function build {
   github_group "l1-contracts build"
-  local artifact=l1-contracts-$hash.tar.gz
-  if ! cache_download $artifact; then
+  if ! cache_download l1-contracts $hash; then
     # Clean
     rm -rf broadcast cache out serve
 
@@ -21,7 +20,7 @@ function build {
     # Compile contracts
     forge build
 
-    cache_upload $artifact out
+    cache_upload l1-contracts $hash out
   fi
   github_endgroup
 }

noir-projects/noir-contracts/bootstrap.sh

@@ -69,11 +69,11 @@ function process_function() {
     # Build hash, check if in cache.
     # If it's in the cache it's extracted to $tmp_dir/$hash
     hash=$((echo "$BB_HASH"; echo "$bytecode_b64") | sha256sum | tr -d ' -')
-    if ! cache_download vk-$hash.tar.gz &> /dev/null; then
+    if ! cache_download vk $hash &> /dev/null; then
       # It's not in the cache. Generate the vk file and upload it to the cache.
       echo_stderr "Generating vk for function: $name..."
       echo "$bytecode_b64" | base64 -d | gunzip | $BB write_vk_for_ivc -h -b - -o $tmp_dir/$hash 2>/dev/null
-      cache_upload vk-$hash.tar.gz $tmp_dir/$hash &> /dev/null
+      cache_upload vk $hash $tmp_dir/$hash &> /dev/null
     fi
 
     # Return (echo) json containing the base64 encoded verification key.
@@ -104,10 +104,10 @@ function compile {
     "^noir-projects/noir-contracts/contracts/$contract/" \
     "^noir-projects/aztec-nr/" \
   )"
-  if ! cache_download contract-$contract_hash.tar.gz &> /dev/null; then
+  if ! cache_download contract $contract_hash &> /dev/null; then
     $NARGO compile --package $contract --silence-warnings --inliner-aggressiveness 0
     $TRANSPILER $json_path $json_path
-    cache_upload contract-$contract_hash.tar.gz $json_path &> /dev/null
+    cache_upload contract $contract_hash $json_path &> /dev/null
   fi
 
   # Pipe each contract function, one per line (jq -c), into parallel calls of process_function.

noir-projects/noir-protocol-circuits/bootstrap.sh

@@ -14,6 +14,9 @@ export NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo}
 export BB_HASH=$(cache_content_hash ../../barretenberg/cpp/.rebuild_patterns)
 export NARGO_HASH=$(cache_content_hash ../../noir/.rebuild_patterns)
 
+export PROJECT_NAME=$(basename "$PWD")
+export CIRCUITS_HASH=$(cache_content_hash ../../noir/.rebuild_patterns "^noir-projects/$PROJECT_NAME")
+
 tmp_dir=./target/tmp
 key_dir=./target/keys
 
@@ -42,6 +45,43 @@ mkdir -p $key_dir
 # Export vars needed inside compile.
 export tmp_dir key_dir ci3 ivc_regex
 
+# Tries to download the given artifact from the cache using the given hash.
+# Falls back to looking for a "symlink" (ie a txt file with the actual hash) using the CIRCUITS_HASH as the identifier.
+# This guards against indeterminism in the nargo --show-program-hash command.
+function try_cache_download {
+  set -euo pipefail
+  local name=$1
+  local hash=$2
+
+  # If we find the artifact, great
+  if cache_download $name $hash 1>&2; then
+    return 0
+  fi
+
+  # If not, try the link and the artifact pointed by it
+  if cache_download $name "$CIRCUITS_HASH-link" 1>&2; then
+    local target_hash=$(cat link.txt)
+    rm link.txt
+    if cache_download $name $target_hash 1>&2; then
+      return 0
+    fi
+  fi
+
+  # If not, fail
+  return 1
+}
+
+# Uploads a "symlink" to the cache with the given hash, using the CIRCUITS_HASH as the identifier.
+function upload_symlink() {
+  set -euo pipefail
+  local name=$1
+  local hash=$2
+  echo "$hash" > link.txt
+  cache_upload "$name" "$CIRCUITS_HASH-link" link.txt &> /dev/null
+  rm link.txt
+  echo_stderr "Uploaded $name-$CIRCUITS_HASH-link pointing to: $hash"
+}
+
 function compile {
   set -euo pipefail
   local dir=$1
@@ -50,21 +90,26 @@ function compile {
   local json_path="./target/$filename"
   local program_hash hash bytecode_hash vk vk_fields
   local program_hash_cmd="$NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2"
+
   # echo_stderr $program_hash_cmd
   program_hash=$(dump_fail "$program_hash_cmd")
   echo_stderr "Hash preimage: $NARGO_HASH-$program_hash"
   hash=$(hash_str "$NARGO_HASH-$program_hash")
-  if ! cache_download circuit-$hash.tar.gz 1>&2; then
+
+  if ! try_cache_download "circuit-$name" $hash 1>&2; then
     SECONDS=0
     rm -f $json_path
     # TODO: --skip-brillig-constraints-check added temporarily for blobs build time.
     local compile_cmd="$NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check"
     echo_stderr "$compile_cmd"
     dump_fail "$compile_cmd"
     echo_stderr "Compilation complete for: $name (${SECONDS}s)"
-    cache_upload circuit-$hash.tar.gz $json_path &> /dev/null
+    cache_upload "circuit-$name" $hash $json_path &> /dev/null
   fi
 
+  # Always upload the link to the artifact, in case the CIRCUITS_HASH changed but the artifact hash didn't
+  upload_symlink "circuit-$name" $hash
+
   if echo "$name" | grep -qE "${ivc_regex}"; then
     local proto="client_ivc"
     local write_vk_cmd="write_vk_for_ivc"
@@ -82,7 +127,8 @@ function compile {
   # Will require changing TS code downstream.
   bytecode_hash=$(jq -r '.bytecode' $json_path | sha256sum | tr -d ' -')
   hash=$(hash_str "$BB_HASH-$bytecode_hash-$proto")
-  if ! cache_download vk-$hash.tar.gz 1>&2; then
+
+  if ! try_cache_download vk-$name $hash 1>&2; then
     local key_path="$key_dir/$name.vk.data.json"
     echo_stderr "Generating vk for function: $name..."
     SECONDS=0
@@ -94,8 +140,11 @@ function compile {
     vk_fields=$(dump_fail "$vkf_cmd")
     jq -n --arg vk "$vk" --argjson vkf "$vk_fields" '{keyAsBytes: $vk, keyAsFields: $vkf}' > $key_path
     echo_stderr "Key output at: $key_path (${SECONDS}s)"
-    cache_upload vk-$hash.tar.gz $key_path &> /dev/null
+    cache_upload "vk-$name" $hash $key_path &> /dev/null
   fi
+
+  # Always upload link
+  upload_symlink "vk-$name" $hash
 }
 
 function build {
@@ -117,8 +166,7 @@ function build {
 function test {
   set -eu
   # Whether we run the tests or not is coarse grained.
-  name=$(basename "$PWD")
-  CIRCUITS_HASH=$(cache_content_hash ../../noir/.rebuild_patterns "^noir-projects/$name")
+  name=$PROJECT_NAME
   if ! test_should_run $name-tests-$CIRCUITS_HASH; then
     return
   fi

noir/bootstrap.sh

@@ -7,18 +7,19 @@ hash=$(cache_content_hash .rebuild_patterns)
 function build {
   github_group "noir build"
   # Downloads and checks for valid nargo and packages.
-  if ! cache_download noir-$hash.tar.gz; then
+  if ! cache_download noir $hash; then
     # Fake this so artifacts have a consistent hash in the cache and not git hash dependent
     export COMMIT_HASH="$(echo "$hash" | sed 's/-.*//g')"
     parallel denoise ::: ./scripts/bootstrap_native.sh ./scripts/bootstrap_packages.sh
-    cache_upload noir-$hash.tar.gz noir-repo/target/release/nargo noir-repo/target/release/acvm packages
+    cache_upload noir $hash noir-repo/target/release/nargo noir-repo/target/release/acvm packages
   fi
   github_endgroup
 }
 
 function test_hash() {
   hash_str $hash-$(cache_content_hash .rebuild_patterns_tests)
 }
+
 function test {
   test_flag=noir-test-$(test_hash)
   if test_should_run $test_flag; then