Implement a Github Action Workflow Log Processor Workflow

Author: ykoer

.github/workflows/fetch_and_log.yaml

@@ -9,7 +9,7 @@ jobs:
   fetch-workflows:
     runs-on: ubuntu-latest
     outputs:
-      workflow_ids: ${{ steps.fetch-ids.outputs.workflow_ids }}
+      workflow_ids: ${{ steps.fetch-unprocessed-run-ids.outputs.workflow_ids }}
 
     steps:
       - name: Get Last Processed Run ID
@@ -24,54 +24,60 @@ jobs:
       - name: Fetch Unprocessed Workflow Runs
         id: fetch-unprocessed-run-ids
         run: |
-          HOURS=1
-          # WORKFLOW_IDS=$(gh run list --repo $GITHUB_REPOSITORY --limit 100 --json databaseId,status,createdAt --jq '[.[] | select(.createdAt > (now - (1 * 3600))) | .databaseId] | @json')
-          # LAST_RUN_ID=$(echo $WORKFLOW_IDS | jq '.[-1]')
+          # This script ensures that only completed GitHub workflow runs are processed while avoiding workflows that are still in progress.
+          # Steps:
+          # 1. Fetch the latest workflow runs from GitHub, sorted by databaseId.
+          # 2. Identify the oldest incomplete workflow (first one still running).
+          # 3. Return an array of workflow IDs up to (but not including) the oldest incomplete workflow ID.
+          WORKFLOW_DATA=$(gh run list --repo $GITHUB_REPOSITORY --limit 100 --json databaseId,createdAt,displayTitle,status --jq "[.[] | select(.databaseId > $LAST_PROCESSED_RUN_ID and .displayTitle != \"Fetch and Log Workflow Runs\")] | sort_by(.databaseId)")
 
-          WORKFLOW_IDS=$(gh run list --repo $GITHUB_REPOSITORY --limit 100 --json databaseId,createdAt --jq "[.[] | select(.databaseId > $LAST_PROCESSED_RUN_ID) | .databaseId] | @json")
-          LAST_RUN_ID=$(echo $WORKFLOW_IDS | jq '.[-1]')  # Store the highest run ID
-          
-          echo "LAST_RUN_ID=$LAST_RUN_ID" >> $GITHUB_ENV
+          OLDEST_INCOMPLETE_ID=$(echo "$WORKFLOW_DATA" | jq '[.[] | select(.status != "completed") | .databaseId] | min // empty')
+
+          if [[ -n "$OLDEST_INCOMPLETE_ID" ]]; then
+            WORKFLOW_IDS=$(echo "$WORKFLOW_DATA" | jq "[.[] | select(.status == \"completed\" and .databaseId < $OLDEST_INCOMPLETE_ID)] | [.[].databaseId] | @json")
+          else
+            WORKFLOW_IDS=$(echo "$WORKFLOW_DATA" | jq "[.[] | select(.status == \"completed\")] | [.[].databaseId] | @json")
+          fi
+
+          echo "Workflow runs to process: $WORKFLOW_IDS"
           echo "workflow_ids=$WORKFLOW_IDS" >> $GITHUB_OUTPUT
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        
+  process-workflows:
+    needs: fetch-workflows
+    runs-on: ubuntu-latest
+    if: ${{ needs.fetch-workflows.outputs.workflow_ids != '[]' && needs.fetch-workflows.outputs.workflow_ids != '' }}
+    strategy:
+      matrix:
+        run_id: ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids || '[]') }}
+      max-parallel: 1
+    steps:
+      - name: Workflow Run to Process
+        run:
+          echo ${{ matrix.run_id }}
+     
+    #   - name: Send Workflow Logs to Splunk
+    #     uses: ykoer/github-workflow-splunk-logger@dev
+    #     with:
+    #       github_token: ${{ secrets.GITHUB_TOKEN }}
+    #       splunk_url: ${{ vars.HEC_URL }}
+    #       splunk_token: ${{ secrets.HEC_TOKEN }}
+    #       run_id: ${{ matrix.run_id }}
 
-      - name: Save Last Processed Run ID
+      - name: Save Processed Run ID
         if: success()
         run: |
-          gh secret set LAST_PROCESSED_ID --repo $GITHUB_REPOSITORY --body "$LAST_RUN_ID"
+          gh variable set LAST_PROCESSED_RUN_ID --repo $GITHUB_REPOSITORY --body "${{ matrix.run_id }}"
         env:
           GH_TOKEN: ${{ secrets.GH_PAT }}
-        
-  process-logs:
+
+
+  # Add a fallback job that runs when there are no workflow Run's to process
+  process-workflows-empty:
     needs: fetch-workflows
     runs-on: ubuntu-latest
-    # strategy:
-    #   matrix:
-    #     run_id: ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids || '[]') }}
+    if: ${{ needs.fetch-workflows.outputs.workflow_ids == '[]' || needs.fetch-workflows.outputs.workflow_ids == '' }}
     steps:
-        # - run:
-        #     echo ${{ matrix.run_id }}
-      - run: |
-          echo "processing logs...."
-          echo "-----------------"
-          echo ${{ needs.fetch-workflows.outputs.workflow_ids }}
-          echo "-----------------"
-          echo ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids || '[]') }}
-          echo "-----------------"
-
-#   process-logs:
-#     needs: fetch-workflows
-#     runs-on: ubuntu-latest
-#     strategy:
-#       matrix:
-#         run_id: ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids) }}
-
-#     steps:
-#       - name: Send Workflow Logs to Splunk
-#         uses: ykoer/github-workflow-splunk-logger@dev
-#         with:
-#           github_token: ${{ secrets.GITHUB_TOKEN }}
-#           splunk_url: ${{ vars.HEC_URL }}
-#           splunk_token: ${{ secrets.HEC_TOKEN }}
-#           run_id: ${{ matrix.run_id }}
+      - name: No workflows to process
+        run: echo "No new workflow runs to process"

.github/workflows/log_to_splunk.yaml

@@ -13,7 +13,7 @@ jobs:
         uses: ykoer/github-workflow-splunk-logger@dev
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          splunk_url: ${{ vars.HEC_URL }}
+          splunk_url: ${{ vars.HEC_URL_TEST }}
           splunk_token: ${{ secrets.HEC_TOKEN }}
-          run_id: 13309230902
+          run_id: 13593560361