Sketch of conditionally executing based on previous run hashes

.github/workflows/cron.yaml

@@ -0,0 +1,73 @@
+on:
+  # TODO: change to cron-based schedule one this is working
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    name: "Test dask and distributed"
+    # TODO: change to appropriate image
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout ourselves
+      uses: actions/checkout@v4
+      with:
+        path: utils
+    - name: Checkout dask
+      uses: actions/checkout@v4
+      with:
+        repository: dask/dask
+        path: dask
+    - name: Checkout distributed
+      uses: actions/checkout@v4
+      with:
+        repository: dask/distributed
+        path: distributed
+    - name: Run
+      run: |
+        (cd dask; git rev-parse HEAD;
+         cd ../distributed; git rev-parse HEAD) | tee commit-hashes.txt
+    - name: Upload commit hashes
+      uses: actions/upload-artifact@v4
+      with:
+        name: commit-hashes.txt
+        path: commit-hashes.txt
+    - name: Setup python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+    - name: Get last artifact URL from last run
+      id: get_last_id
+      run: |
+        pip install requests
+        VAL=$(python utils/get.py)
+        echo "${VAL}"
+        echo "${VAL}" >> $GITHUB_OUTPUT
+    - name: Download artifact from last run if exists
+      if: ${{ fromJSON(steps.get_last_id.outputs.INFO).exists }}
+      continue-on-error: true
+      uses: actions/download-artifact@v4
+      with:
+        name: commit-hashes.txt
+        path: previous-run
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        run-id: ${{ fromJSON(steps.get_last_id.outputs.INFO).id }}
+    - name: Check if test run is needed
+      id: check_run_needed
+      run: |
+        ls -l previous-run/
+        if [ ! -f previous-run/commit-hashes.txt ]; then
+          echo "No previous run hashes, need to re-run"
+          echo 'INFO={"rerun": true}' >> $GITHUB_OUTPUT
+        elif cmp -s commit-hashes.txt previous-run/commit-hashes.txt; then
+          echo "Previous run hash same as this one, no need to re-run"
+          echo 'INFO={"rerun": false}' >> $GITHUB_OUTPUT
+        else
+          echo "Previous run hash different, need to re-run"
+          echo 'INFO={"rerun": true}' >> $GITHUB_OUTPUT
+        fi
+    - name: Run tests
+      if: ${{ fromJSON(steps.check_run_needed.outputs.INFO).rerun }}
+      run: |
+        echo Running tests

get.py

@@ -0,0 +1,33 @@
+import requests
+import json
+
+
+def previous_run_id() -> str | None:
+    req = requests.get(
+        "https://api.github.com/repos/rapidsai/dask-upstream-testing/actions/artifacts",
+        headers={
+            "Accept": "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+        params={"name": "commit-hashes.txt", "page": 1, "per_page": 1},
+    )
+    if req.status_code != 200:
+        return None
+    artifacts = req.json()["artifacts"]
+    try:
+        (artifact,) = artifacts
+        run_id = artifact["workflow_run"]["id"]
+        return run_id
+    except ValueError:
+        # Didn't get exactly one artifact, assume we must rebuild
+        return None
+
+
+if __name__ == "__main__":
+    run_id = previous_run_id()
+    if run_id is not None:
+        info = json.dumps({"id": run_id, "exists": True})
+        print(f"INFO={info}")
+    else:
+        info = json.dumps({"exists": False})
+        print(f"INFO={info}")