From 27af7d64f55a70bf8aa0863b3a7aee4b433d52f5 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Tue, 23 Jan 2024 14:12:40 -0800
Subject: [PATCH] Add retry to C++ packaging step. (#1527)

* Add retry to C++ packaging step.

* Change log message slightly.

* Fix indent.

* Remove bad variable.

* Temporarily attempt retry on non-scheduled, for testing.

* Revert "Temporarily attempt retry on non-scheduled, for testing."

This reverts commit b9cfcf9a521bbd2929cd6a80bc1b43201b1f3e58.
---
 .github/workflows/cpp-packaging.yml | 31 +++++++++++++++++++++++++++++
 scripts/gha/retry_test_failures.py  | 17 ++++++++++++++--
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml
index 91bc5d0112..94224b94f1 100644
--- a/.github/workflows/cpp-packaging.yml
+++ b/.github/workflows/cpp-packaging.yml
@@ -893,3 +893,34 @@ jobs:
             -s 10 \
             -A ${verbose_flag}
         fi
+
+
+  attempt_retry:
+    name: "attempt-retry"
+    needs: [trigger_integration_tests]
+    runs-on: ubuntu-20.04
+    if: ${{ failure() && !cancelled() && github.event_name == 'schedule' }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Install python deps
+        run: pip install -r scripts/gha/python_requirements.txt
+      # The default token can't run workflows, so get an alternate token.
+      - name: Generate token for GitHub API
+        uses: tibdex/github-app-token@v1
+        id: generate-token
+        with:
+          app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
+          private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
+      - name: Retry failed tests
+        run: |
+          echo "::warning ::Attempting to retry failed jobs"
+          python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} \
+          -w retry-test-failures.yml \
+          -p run_id ${{ github.run_id }} \
+          -s 10 \
+          -A
diff --git a/scripts/gha/retry_test_failures.py b/scripts/gha/retry_test_failures.py
index bb88cbe873..91a7e1ed68 100644
--- a/scripts/gha/retry_test_failures.py
+++ b/scripts/gha/retry_test_failures.py
@@ -107,13 +107,26 @@ def main(argv):
         # Retry all build failures that don't match a compiler error.
         if not re.search(r'.*/.*:[0-9]+:[0-9]+: error:', job_logs):
           should_rerun_jobs = True
+        # Also retry build jobs that timed out
+        if re.search(r'timed? ?out|network error|maximum execution time',
+                     job_logs, re.IGNORECASE):
+          should_rerun_jobs = True
+      elif job['name'].startswith('download-'):
+        # If a download step failed, automatically retry.
+        should_rerun_jobs = True
+      elif job['name'].startswith('package-'):
+        # Retry packaging jobs that timed out
+        if re.search(r'timed? ?out|network error|maximum execution time',
+                     job_logs, re.IGNORECASE):
+          should_rerun_jobs = True
       elif job['name'].startswith('test-'):
         if '-android' in job['name'] or '-ios' in job['name']:
           # Mobile tests should always be retried.
           should_rerun_jobs = True
         else:
-          # Desktop tests should only retry on a network error.
-          if re.search(r'timed out|network error', job_logs, re.IGNORECASE):
+          # Desktop tests should only retry on a network error or timeout.
+          if re.search(r'timed? ?out|network error|maximum execution time',
+                       job_logs, re.IGNORECASE):
             should_rerun_jobs = True
 
   if should_rerun_jobs: