From 9c37048578c6cec83e529a68d579b5059a22bd21 Mon Sep 17 00:00:00 2001
From: Alex Goodman <wagoodman@users.noreply.github.com>
Date: Thu, 5 Sep 2024 23:42:04 -0400
Subject: [PATCH] show cache miss di

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
---
 .github/scripts/find_cache_paths.py | 18 +++++++++++++++++-
 .github/workflows/validations.yaml  | 14 ++++++++++++++
 Taskfile.yaml                       | 19 ++++++++++++++-----
 3 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/.github/scripts/find_cache_paths.py b/.github/scripts/find_cache_paths.py
index 38b8103c4c3..d6a5b7a4d51 100755
--- a/.github/scripts/find_cache_paths.py
+++ b/.github/scripts/find_cache_paths.py
@@ -48,6 +48,15 @@ def find_fingerprints_and_check_dirs(base_dir):
     return sorted(valid_paths), missing_content, fingerprint_contents
 
 
+def parse_fingerprint_contents(fingerprint_content):
+    """Parse the fingerprint content and return a dict of digest-path tuples."""
+    input_map = {}
+    for line in fingerprint_content.splitlines():
+        digest, path = line.split()
+        input_map[path] = digest
+    return input_map
+
+
 def calculate_sha256(fingerprint_contents):
     sorted_fingerprint_contents = sorted(fingerprint_contents, key=lambda x: x[0])
 
@@ -89,9 +98,16 @@ def main(file_path: str | None):
         fingerprint_file = f"{path}.fingerprint"
         if os.path.exists(fingerprint_file):
             file_digest = calculate_file_sha256(fingerprint_file)
+
+            # Parse the fingerprint file to get the digest/path tuples
+            with open(fingerprint_file, 'r') as f:
+                fingerprint_content = f.read().strip()
+                input_map = parse_fingerprint_contents(fingerprint_content)
+
             paths_with_digests.append({
                 "path": path,
-                "digest": file_digest
+                "digest": file_digest,
+                "input": input_map
             })
 
     output = {
diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml
index 67fc5cd1878..9bfd7040fd1 100644
--- a/.github/workflows/validations.yaml
+++ b/.github/workflows/validations.yaml
@@ -41,6 +41,20 @@ jobs:
       - name: Run unit tests
         run: make unit
 
+      # save .tmp/cache_paths.json and .tmp/last_cache_paths.json as workflow artifacts
+      - name: Upload cache_paths.json as an artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: cache_paths
+          path: .tmp/cache_paths.json
+
+      - name: Upload last_cache_paths.json as an artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: last_cache_paths
+          path: .tmp/last_cache_paths.json
+
+
 
   Integration-Test:
     # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
diff --git a/Taskfile.yaml b/Taskfile.yaml
index b3bacd4a564..4239e2f20f3 100644
--- a/Taskfile.yaml
+++ b/Taskfile.yaml
@@ -11,7 +11,7 @@ vars:
   TMP_DIR: .tmp
   ORAS_CACHE: "{{ .TMP_DIR }}/oras-cache"
   CACHE_PATHS_FILE: "{{ .TMP_DIR }}/cache_paths.json"
-  LAST_CACHE_PULL_FILE: "{{ .TMP_DIR }}/last_cache_pull"
+  LAST_CACHE_PULL_FILE: "{{ .TMP_DIR }}/last_cache_paths.json"
 
   # TOOLS
   ORAS: "{{ .TOOL_DIR }}/oras"
@@ -304,8 +304,10 @@ tasks:
           #echo -e "${YELLOW}• calculating fingerprint for '$dir/image-*'... ${RESET}"
           
           # find all image-* directories in this test-fixtures directory and generate sha256sum
-          find "$dir" -type d -name 'image-*' -exec bash -c 'find "$0" -type f | sort | xargs sha256sum' {} \; > "$dir/cache.fingerprint"
-          
+          pushd "$dir" > /dev/null
+            find . -type d -name 'image-*' -exec bash -c 'find "$0" -type f | sort | xargs sha256sum' {} \; > "cache.fingerprint"
+          popd > /dev/null
+        
           # for debugging...
           #cat "$dir/cache.fingerprint" | sha256sum | awk '{print $1}'
         done
@@ -335,7 +337,7 @@ tasks:
       echo "desired cache:     $WANT_FINGERPRINT"
       
       if [ -f {{ .LAST_CACHE_PULL_FILE }} ]; then
-        LAST_PULL_FINGERPRINT=$(cat {{ .LAST_CACHE_PULL_FILE }})
+        LAST_PULL_FINGERPRINT=$(cat {{ .LAST_CACHE_PULL_FILE }} | {{ .YQ }} -r '.digest')
       else
         echo -e "${BOLD}${PURPLE}empty cache, refreshing cache...${RESET}"
         {{ .TASK }} clean-cache download-test-fixture-cache
@@ -360,6 +362,13 @@ tasks:
         {{ .TASK }} clean-cache download-test-fixture-cache
       else
         echo -e "${BOLD}${PURPLE}found newer cache, but does not match what we want (skipping cache refresh)${RESET}"
+      
+        git status
+      
+        {{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .LAST_CACHE_PULL_FILE }} > .tmp/last_cache_lines
+        {{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .CACHE_PATHS_FILE }} > .tmp/cache_lines
+        diff .tmp/last_cache_lines .tmp/cache_lines || true
+      
       fi
 
   build-fixtures:
@@ -389,7 +398,7 @@ tasks:
         sh: docker manifest inspect {{ .CACHE_IMAGE }} | jq -r '.annotations.fingerprint'
     cmds:
       - "ORAS_CACHE={{ .ORAS_CACHE }} {{ .ORAS }} pull {{ .CACHE_IMAGE }}"
-      - "echo '{{ .CACHE_DIGEST }}' > {{ .LAST_CACHE_PULL_FILE }}"
+      - "cp {{ .CACHE_PATHS_FILE }} {{ .LAST_CACHE_PULL_FILE }}"
 
   upload-test-fixture-cache:
     desc: Upload the test fixture cache to ghcr.io