[DEVOPS-234] CI/CD: Reuse builds for feature branches in Github Actions workflows

.github/workflows/valgrind.yml

@@ -18,34 +18,102 @@ on:
         required: false
         default: false
 
+env:
+  PYTHON_TAG: cp38
+
 jobs:
+  # TODO: handle case where we run valgrind on a "central" branch
+  look-for-wheel-in-jfrog:
+    if: ${{ github.ref_name != 'dev' && github.ref_name != 'stage' && github.ref_name != 'master' }}
+    outputs:
+      num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }}
+      # We want this environment variable to be available when calling a reusable workflow
+      # so we convert it to a job output so the reusable workflow can access it
+      python-tag: ${{ env.PYTHON_TAG }}
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: jfrog/setup-jfrog-cli@v4
+      env:
+        JF_URL: ${{ secrets.JFROG_PLATFORM_URL }}
+        JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }}
+
+    - name: Get shortened commit hash of this workflow run
+      # versioningit commit sha is always 8 chars long it seems
+      run: echo SHORT_GITHUB_SHA=$(echo ${{ github.sha }} | cut -c1-8) >> $GITHUB_ENV
+
+    - name: Look for wheel built with default settings in JFrog
+      # AQL has the option to exclude patterns from search results
+      # but it doesn't allow regex, so we can't filter out any type of label in a wheel name
+      # Example: we want to filter out "unoptimized" and "dsym" but in case we add more labels, we want to use regex
+      # to handle those new labels without updating the regex.
+      run: echo $(jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" | jq '.[].path' | grep --invert-match "${{ env.SHORT_GITHUB_SHA }}\.") > file_matches.txt
+      shell: bash
+
+    - name: Count artifacts
+      id: count_num_artifacts_found
+      run: echo num_artifacts=$(echo file_matches.txt | wc -l) >> $GITHUB_OUTPUT
+
+    - name: Multiple artifacts found, not sure which one to use. Fail out
+      if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }}
+      run: exit 1
+
+    - name: Found the exact artifact in JFrog, downloading...
+      if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }}
+      run: jf rt download --flat --fail-no-op "$(cat file_matches.txt)"
+      working-directory: .github/workflows
+
+    - name: Pass to valgrind job
+      if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }}
+      uses: actions/upload-artifact@v4
+      with:
+        # Artifact name doesn't matter. Valgrind job downloads all artifacts to get the one wheel
+        if-no-files-found: error
+        path: './.github/workflows/*.whl'
+
   build-manylinux-wheel:
+    needs: look-for-wheel-in-jfrog
+    # If previous job was skipped, we are running on a central branch
+    # Always build wheels from a central branch for now
+    # Or if we are running on a feature branch and could not find any wheels in JFrog, build and upload it to JFrog
+    if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'skipped' || (needs.look-for-wheel-in-jfrog.result == 'success' && needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0)) }}
     uses: ./.github/workflows/build-wheels.yml
     with:
-      python-tags: '["cp38"]'
+      python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]'
       platform-tag: manylinux_x86_64
       sha-to-build-and-test: ${{ github.sha }}
     secrets: inherit
 
+  upload-built-wheel-to-jfrog:
+    needs: build-manylinux-wheel
+    uses: ./.github/workflows/upload-to-jfrog.yml
+    with:
+      jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }}
+    secrets: inherit
+
   valgrind:
     env:
       MASSIF_REPORT_FILE_NAME: massif.out
     needs: build-manylinux-wheel
+    if: ${{ !cancelled() && (needs.build-manylinux-wheel.result == 'success' || needs.build-manylinux-wheel.result == 'skipped') }}
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
       with:
         submodules: recursive
         fetch-depth: 0
 
+    - name: Convert Python tag to Python version
+      run: echo PYTHON_VERSION=$(echo ${{ env.PYTHON_TAG }} | sed -e "s/cp3/cp3./" -e "s/cp//") >> $GITHUB_ENV
+      shell: bash
+
     - uses: actions/setup-python@v2
       with:
-        python-version: '3.8'
+        python-version: '${{ env.PYTHON_VERSION }}'
         architecture: 'x64'
 
     - uses: actions/download-artifact@v4
       with:
-        name: cp38-manylinux_x86_64.build
+        merge-multiple: true
 
     - name: Install client
       run: pip install ./*.whl
@@ -72,6 +140,18 @@ jobs:
     - run: PYTHONMALLOC=malloc valgrind --error-exitcode=1 ${{ env.VALGRIND_ARGS }} python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }}
       working-directory: test
 
+    # TODO: upload report as artifact
     - run: ms_print ./${{ env.MASSIF_REPORT_FILE_NAME }}
       if: ${{ !cancelled() && inputs.massif }}
       working-directory: test
+
+  # See reason for deleting artifacts in dev-workflow-p2.yml
+  delete-artifacts:
+    needs: [
+      # These jobs must have downloaded the artifact from Github before we can delete it
+      upload-built-wheel-to-jfrog,
+      valgrind
+    ]
+    # Workflow run must clean up after itself even if cancelled
+    if: ${{ always() }}
+    uses: ./.github/workflows/delete-artifacts.yml