Scheduled rpmrepo snapshot #120
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scheduled rpmrepo snapshot | |
| on: | |
| schedule: | |
| - cron: '0 0 1,15 * *' # 1st and 15th of every month | |
| workflow_dispatch: | |
| jobs: | |
| update-snapshots: | |
| name: "Scheduled rpmrepo snapshot" | |
| timeout-minutes: 2880 # 2 days timeout | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Clone repository | |
| uses: actions/[email protected] | |
| - name: Count jobs | |
| id: count_jobs | |
| run: echo "size=$(find repo -name '*json' | wc -l)" >>$GITHUB_OUTPUT | |
| - name: Generate suffix | |
| id: generate_suffix | |
| run: echo "suffix=$(date '+%Y%m%d')" >>$GITHUB_OUTPUT | |
| - name: Submit array job | |
| id: submit_job | |
| run: | | |
| echo size ${{ steps.count_jobs.outputs.size }} sha ${GITHUB_SHA} | |
| aws batch submit-job \ | |
| --job-name "snapshot-runner" \ | |
| --job-definition "rpmrepo-batch-snapshot-staging" \ | |
| --job-queue "rpmrepo-batch-staging" \ | |
| --timeout "attemptDurationSeconds=86400" \ | |
| --array-properties "size=${{ steps.count_jobs.outputs.size }}" \ | |
| --parameters "repoCommit=$GITHUB_SHA,repoBranch=main,repoSuffix=${{ steps.generate_suffix.outputs.suffix }},repoTarget=auto" \ | |
| > out.json | |
| echo "job_id=$(jq -r .jobId out.json)" >>$GITHUB_OUTPUT | |
| echo "job_name=$(jq -r .jobName out.json)" >>$GITHUB_OUTPUT | |
| - name: Wait for jobs to start | |
| id: wait_for_jobs_start | |
| run: | | |
| while true; do | |
| SUBMITTED=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status SUBMITTED | jq '.jobSummaryList | length') | |
| PENDING=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status PENDING | jq '.jobSummaryList | length') | |
| RUNNABLE=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status RUNNABLE | jq '.jobSummaryList | length') | |
| STARTING=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status STARTING | jq '.jobSummaryList | length') | |
| RUNNING=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status RUNNING | jq '.jobSummaryList | length') | |
| if [ $((SUBMITTED + PENDING + RUNNABLE + STARTING + RUNNING)) -gt 0 ]; then | |
| break | |
| fi | |
| sleep 30s | |
| done | |
| - name: Wait for jobs to finish | |
| id: wait_for_jobs | |
| run: | | |
| while true; do | |
| SUBMITTED=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status SUBMITTED | jq '.jobSummaryList | length') | |
| PENDING=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status PENDING | jq '.jobSummaryList | length') | |
| RUNNABLE=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status RUNNABLE | jq '.jobSummaryList | length') | |
| STARTING=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status STARTING | jq '.jobSummaryList | length') | |
| RUNNING=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status RUNNING | jq '.jobSummaryList | length') | |
| if [ $((SUBMITTED + PENDING + RUNNABLE + STARTING + RUNNING)) = 0 ]; then | |
| break | |
| fi | |
| echo Submitted $SUBMITTED, Pending $PENDING, Runnable $RUNNABLE, Starting $STARTING, Running $RUNNING | |
| sleep 5m | |
| done | |
| aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status FAILED > failed.json | |
| if [ $(jq '.jobSummaryList | length' failed.json) != 0 ]; then | |
| echo Failed jobs! | |
| for JOBID in $(jq -r '.jobSummaryList[].jobId' failed.json | xargs); do | |
| LOGSTREAM_NAME=$(aws batch describe-jobs --jobs "$JOBID" | jq -r '.jobs[0].container.logStreamName') | |
| LOGS="$(aws logs get-log-events --log-group-name /aws/batch/job --log-stream-name "$LOGSTREAM_NAME" --start-from-head | jq -r '.events[].message')" | |
| echo "Logs for failed job \"$JOBID\":" | |
| echo "$LOGS" | |
| echo | |
| done | |
| fi | |
| SUCCEEDED=$(aws batch list-jobs --array-job-id ${{ steps.submit_job.outputs.job_id }} --job-status SUCCEEDED | jq '.jobSummaryList | length') | |
| echo "succeeded=$SUCCEEDED" >>$GITHUB_OUTPUT | |
| echo "failed=$(jq '.jobSummaryList | length' failed.json)" >>$GITHUB_OUTPUT | |
| - name: Clone osbuild-composer repository | |
| uses: actions/[email protected] | |
| with: | |
| repository: osbuild/osbuild-composer | |
| path: osbuild-composer | |
| fetch-depth: 0 | |
| - name: Update schutzfile and open PR in osbuild-composer repository | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.SCHUTZBOT_GH_TOKEN }} | |
| SUFFIX: ${{ steps.generate_suffix.outputs.suffix }} | |
| JOBS_SUCCEEDED: ${{ steps.wait_for_jobs-outputs.succeeded }} | |
| JOBS_FAILED: ${{ steps.wait_for_jobs.outputs.failed }} | |
| WORKFLOW_RUN: ${{ github.run_id }} | |
| REPO: "osbuild-composer" | |
| run: .github/scripts/snapshot_update_pr.sh | |
| - name: Clone osbuild repository | |
| uses: actions/[email protected] | |
| with: | |
| repository: osbuild/osbuild | |
| path: osbuild | |
| fetch-depth: 0 | |
| - name: Update schutzfile and open PR in osbuild repository | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.SCHUTZBOT_GH_TOKEN }} | |
| SUFFIX: ${{ steps.generate_suffix.outputs.suffix }} | |
| JOBS_SUCCEEDED: ${{ steps.wait_for_jobs-outputs.succeeded }} | |
| JOBS_FAILED: ${{ steps.wait_for_jobs.outputs.failed }} | |
| WORKFLOW_RUN: ${{ github.run_id }} | |
| REPO: "osbuild" | |
| run: .github/scripts/snapshot_update_pr.sh | |
| - name: Clone cloud image val repository | |
| uses: actions/[email protected] | |
| with: | |
| repository: osbuild/cloud-image-val | |
| path: cloud-image-val | |
| fetch-depth: 0 | |
| - name: Update schutzfile and open PR in cloud-image-val repository | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.SCHUTZBOT_GH_TOKEN }} | |
| SUFFIX: ${{ steps.generate_suffix.outputs.suffix }} | |
| JOBS_SUCCEEDED: ${{ steps.wait_for_jobs-outputs.succeeded }} | |
| JOBS_FAILED: ${{ steps.wait_for_jobs.outputs.failed }} | |
| WORKFLOW_RUN: ${{ github.run_id }} | |
| REPO: "cloud-image-val" | |
| run: .github/scripts/snapshot_update_pr.sh | |
| - name: Clone images repository | |
| uses: actions/[email protected] | |
| with: | |
| repository: osbuild/images | |
| path: images | |
| fetch-depth: 0 | |
| - name: Update schutzfile and open PR in images repository | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.SCHUTZBOT_GH_TOKEN }} | |
| SUFFIX: ${{ steps.generate_suffix.outputs.suffix }} | |
| JOBS_SUCCEEDED: ${{ steps.wait_for_jobs-outputs.succeeded }} | |
| JOBS_FAILED: ${{ steps.wait_for_jobs.outputs.failed }} | |
| WORKFLOW_RUN: ${{ github.run_id }} | |
| REPO: "images" | |
| run: .github/scripts/snapshot_update_pr.sh |