From 228d7b8311c4f1bf999b4cef4e693f7b66aa8983 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 10 Jul 2024 15:14:28 -0500 Subject: [PATCH 1/9] draft initial additon to internal release workflow for cron implementation --- .github/workflows/release-internal.yml | 33 ++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index dbc740909..6555e51f9 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -8,7 +8,7 @@ # # When? # -# Manual trigger. +# Manual trigger or cron job every wednesday morning. name: "Release to Cloud" run-name: "Release to Cloud off of ${{ inputs.ref }}" @@ -32,20 +32,45 @@ on: required: true default: false + schedule: + - cron: '0 13 * * 3' # Every Wednesday at 8:00 AM central time + defaults: run: shell: bash jobs: + get-latest-release: + runs-on: ubuntu-latest + outputs: + latest_release: ${{ steps.get_latest_release.outputs.latest_release }} + steps: + - name: Get latest release + id: get_latest_release + run: | + latest_release=$(curl --silent "https://api.github.com/repos/${{ github.repository }}/releases/latest" | jq -r .tag_name) + echo "::set-output name=latest_release::$latest_release" + + check-for-new-commits: + runs-on: ubuntu-latest + needs: get-latest-release + outputs: + new_commits: ${{ steps.check_new_commits.outputs.new_commits }} + steps: + - name: Check for new commits since last release + id: check_new_commits + run: | + last_release_date=$(curl --silent "https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ needs.get-latest-release.outputs.latest_release }}" | jq -r .published_at) + new_commits=$(git rev-list --count --since="$last_release_date" ${{ inputs.ref }}) + echo "::set-output name=new_commits::$new_commits" + invoke-reusable-workflow: + if: needs.check-for-new-commits.output.new_commits != '0' name: "Build and Release Internally" - uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" - with: package_test_command: "${{ inputs.package_test_command }}" dbms_name: "bigquery" ref: "${{ inputs.ref }}" skip_tests: "${{ inputs.skip_tests }}" - secrets: "inherit" From 3f50932395fd7c4ff9daab35dfc3d9b60897627d Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 10 Jul 2024 15:51:57 -0500 Subject: [PATCH 2/9] minor update, to grab from last successful run of workflow not last release --- .github/workflows/release-internal.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 6555e51f9..54f85a212 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -40,27 +40,28 @@ defaults: shell: bash jobs: - get-latest-release: + get-last-successful-run: runs-on: ubuntu-latest outputs: - latest_release: ${{ steps.get_latest_release.outputs.latest_release }} + commit_sha: ${{ steps.get_last_successful_run.outputs.commit_sha }} steps: - - name: Get latest release - id: get_latest_release + - name: Get last successful run details + id: get_last_successful_run run: | - latest_release=$(curl --silent "https://api.github.com/repos/${{ github.repository }}/releases/latest" | jq -r .tag_name) - echo "::set-output name=latest_release::$latest_release" + last_run_url=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/runs?event=workflow_dispatch&status=success&branch=${{ github.ref }}" | jq -r '.workflow_runs[0].url') + commit_sha=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" $last_run_url | jq -r '.head_sha') + echo "::set-output name=commit_sha::$commit_sha" check-for-new-commits: runs-on: ubuntu-latest - needs: get-latest-release + needs: get-last-successful-run outputs: new_commits: ${{ steps.check_new_commits.outputs.new_commits }} steps: - name: Check for new commits since last release id: check_new_commits run: | - last_release_date=$(curl --silent "https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ needs.get-latest-release.outputs.latest_release }}" | jq -r .published_at) + last_release_date=$(curl --silent "https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ needs.get-last-successful-run.outputs.commit_sha }}" | jq -r .published_at) new_commits=$(git rev-list --count --since="$last_release_date" ${{ inputs.ref }}) echo "::set-output name=new_commits::$new_commits" From 2d1c9a4f3dc6500bd91a0370be0a3d2061a10b07 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 10 Jul 2024 16:12:09 -0500 Subject: [PATCH 3/9] try to target specific release-internal.yml file to check for commit vs workflow-dispatch --- .github/workflows/release-internal.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 54f85a212..0f6bd11d4 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -40,21 +40,21 @@ defaults: shell: bash jobs: - get-last-successful-run: + get-last-successful-release-run: runs-on: ubuntu-latest outputs: - commit_sha: ${{ steps.get_last_successful_run.outputs.commit_sha }} + commit_sha: ${{ steps.get_last_successful_release_run.outputs.commit_sha }} steps: - - name: Get last successful run details - id: get_last_successful_run + - name: Get last successful Release to Cloud run details + id: get_last_successful_release_run run: | - last_run_url=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/runs?event=workflow_dispatch&status=success&branch=${{ github.ref }}" | jq -r '.workflow_runs[0].url') + last_run_url=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/runs?event=workflow_dispatch&status=success&branch=${{ github.ref }}&workflow_file=release.internal.yml" | jq -r '.workflow_runs[0].url') commit_sha=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" $last_run_url | jq -r '.head_sha') echo "::set-output name=commit_sha::$commit_sha" check-for-new-commits: runs-on: ubuntu-latest - needs: get-last-successful-run + needs: get-last-successful-release-run outputs: new_commits: ${{ steps.check_new_commits.outputs.new_commits }} steps: From 7c82ab8851c7cc59c1bce0d9565cc973297185e6 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 11 Jul 2024 10:14:14 -0500 Subject: [PATCH 4/9] update workflow --- .github/workflows/release-internal.yml | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 0f6bd11d4..9b772b5ac 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -49,24 +49,34 @@ jobs: id: get_last_successful_release_run run: | last_run_url=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/runs?event=workflow_dispatch&status=success&branch=${{ github.ref }}&workflow_file=release.internal.yml" | jq -r '.workflow_runs[0].url') + echo "Last run URL: $last_run_url" commit_sha=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" $last_run_url | jq -r '.head_sha') + echo "Commit SHA: $commit_sha" echo "::set-output name=commit_sha::$commit_sha" check-for-new-commits: runs-on: ubuntu-latest needs: get-last-successful-release-run outputs: - new_commits: ${{ steps.check_new_commits.outputs.new_commits }} + latest_commit_sha: ${{ steps.check_new_commits.outputs.latest_commit_sha }} + has_new_commits: ${{ steps.check_new_commits.outputs.has_new_commits }} steps: - - name: Check for new commits since last release + - name: Check for new commits since last Release to Cloud run id: check_new_commits run: | - last_release_date=$(curl --silent "https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ needs.get-last-successful-run.outputs.commit_sha }}" | jq -r .published_at) - new_commits=$(git rev-list --count --since="$last_release_date" ${{ inputs.ref }}) - echo "::set-output name=new_commits::$new_commits" + last_commit_sha=${{ needs.get-last-successful-release-run.outputs.commit_sha }} + echo "Last successful commit SHA: $last_commit_sha" + latest_commit_sha=$(git rev-parse ${{ inputs.ref }}) + echo "Latest commit SHA: $latest_commit_sha" + if [ "$last_commit_sha" != "$latest_commit_sha" ]; then + echo "::set-output name=has_new_commits::true" + else + echo "::set-output name=has_new_commits::false" + fi + echo "::set-output name=latest_commit_sha::$latest_commit_sha" invoke-reusable-workflow: - if: needs.check-for-new-commits.output.new_commits != '0' + if: needs.check-for-new-commits.outputs.has_new_commits == 'true' name: "Build and Release Internally" uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" with: @@ -74,4 +84,4 @@ jobs: dbms_name: "bigquery" ref: "${{ inputs.ref }}" skip_tests: "${{ inputs.skip_tests }}" - secrets: "inherit" + secrets: inherit From ad4d43be5aec5e886eea3b059f1122a390a908b8 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 11 Jul 2024 11:43:54 -0500 Subject: [PATCH 5/9] typo in file name --- .github/workflows/release-internal.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 9b772b5ac..f9b08e0f6 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -48,12 +48,11 @@ jobs: - name: Get last successful Release to Cloud run details id: get_last_successful_release_run run: | - last_run_url=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/runs?event=workflow_dispatch&status=success&branch=${{ github.ref }}&workflow_file=release.internal.yml" | jq -r '.workflow_runs[0].url') + last_run_url=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/runs?event=workflow_dispatch&status=success&branch=main&workflow_file=release-internal.yml" | jq -r '.workflow_runs[0].url') echo "Last run URL: $last_run_url" commit_sha=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" $last_run_url | jq -r '.head_sha') echo "Commit SHA: $commit_sha" echo "::set-output name=commit_sha::$commit_sha" - check-for-new-commits: runs-on: ubuntu-latest needs: get-last-successful-release-run From 0c88154f09eae69cba52c47b424c9faee76587d6 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Fri, 19 Jul 2024 14:56:38 -0500 Subject: [PATCH 6/9] update workflow based on feedback --- .github/workflows/release-internal.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index f9b08e0f6..8ac361226 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -9,6 +9,7 @@ # When? # # Manual trigger or cron job every wednesday morning. +# action will check if a new commit was added if not should do nothing name: "Release to Cloud" run-name: "Release to Cloud off of ${{ inputs.ref }}" @@ -52,7 +53,8 @@ jobs: echo "Last run URL: $last_run_url" commit_sha=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" $last_run_url | jq -r '.head_sha') echo "Commit SHA: $commit_sha" - echo "::set-output name=commit_sha::$commit_sha" + echo "commit_sha=$commit_sha" >> "$GITHUB_OUTPUT" + check-for-new-commits: runs-on: ubuntu-latest needs: get-last-successful-release-run @@ -60,21 +62,25 @@ jobs: latest_commit_sha: ${{ steps.check_new_commits.outputs.latest_commit_sha }} has_new_commits: ${{ steps.check_new_commits.outputs.has_new_commits }} steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Check for new commits since last Release to Cloud run id: check_new_commits run: | - last_commit_sha=${{ needs.get-last-successful-release-run.outputs.commit_sha }} - echo "Last successful commit SHA: $last_commit_sha" + last_released_commit_sha=${{ needs.get-last-successful-release-run.outputs.commit_sha }} + echo "Last released commit SHA: $last_released_commit_sha" latest_commit_sha=$(git rev-parse ${{ inputs.ref }}) echo "Latest commit SHA: $latest_commit_sha" - if [ "$last_commit_sha" != "$latest_commit_sha" ]; then - echo "::set-output name=has_new_commits::true" + if [ "$last_released_commit_sha" != "$latest_commit_sha" ]; then + echo "has_new_commits=true" >> "$GITHUB_OUTPUT" else - echo "::set-output name=has_new_commits::false" + echo "has_new_commits=false" >> "$GITHUB_OUTPUT" fi - echo "::set-output name=latest_commit_sha::$latest_commit_sha" + echo "latest_commit_sha=$latest_commit_sha" >> "$GITHUB_OUTPUT" invoke-reusable-workflow: + needs: check-for-new-commits if: needs.check-for-new-commits.outputs.has_new_commits == 'true' name: "Build and Release Internally" uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" @@ -83,4 +89,4 @@ jobs: dbms_name: "bigquery" ref: "${{ inputs.ref }}" skip_tests: "${{ inputs.skip_tests }}" - secrets: inherit + secrets: "inherit" From ff103958590f4121d897d80f89b5d697dd981fab Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Fri, 19 Jul 2024 15:48:51 -0500 Subject: [PATCH 7/9] add validation for None value --- .github/workflows/release-internal.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 8ac361226..25241c1f7 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -50,8 +50,16 @@ jobs: id: get_last_successful_release_run run: | last_run_url=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/runs?event=workflow_dispatch&status=success&branch=main&workflow_file=release-internal.yml" | jq -r '.workflow_runs[0].url') + if [ -z "$last_run_url" ]; then + echo "No successful Release to Cloud run found." + exit 1 + fi echo "Last run URL: $last_run_url" - commit_sha=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" $last_run_url | jq -r '.head_sha') + commit_sha=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "$last_run_url" | jq -r '.head_sha') + if [ -z "$commit_sha" ]; then + echo "No commit SHA found for the last successful run." + exit 1 + fi echo "Commit SHA: $commit_sha" echo "commit_sha=$commit_sha" >> "$GITHUB_OUTPUT" From 375509b97420ba8118a70f12590edb281f07f57b Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Mon, 22 Jul 2024 11:13:11 -0500 Subject: [PATCH 8/9] feddback from friday to update workflow and implementent dry_run --- .github/workflows/release-internal.yml | 45 +++++++++++++------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 25241c1f7..049f26e61 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -32,6 +32,11 @@ on: type: boolean required: true default: false + dry_run: + description: "Is this a dry run? (default to false)" + type: boolean + required: true + default: false schedule: - cron: '0 13 * * 3' # Every Wednesday at 8:00 AM central time @@ -41,14 +46,19 @@ defaults: shell: bash jobs: - get-last-successful-release-run: + check-for-new-commits: runs-on: ubuntu-latest outputs: - commit_sha: ${{ steps.get_last_successful_release_run.outputs.commit_sha }} + latest_commit_sha: ${{ steps.check_new_commits.outputs.latest_commit_sha }} + has_new_commits: ${{ steps.check_new_commits.outputs.has_new_commits }} steps: - - name: Get last successful Release to Cloud run details - id: get_last_successful_release_run + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Get last successful Release to Cloud run details and check for new commits + id: check_new_commits run: | + # Get last successful Release to Cloud run details last_run_url=$(curl --silent -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/actions/runs?event=workflow_dispatch&status=success&branch=main&workflow_file=release-internal.yml" | jq -r '.workflow_runs[0].url') if [ -z "$last_run_url" ]; then echo "No successful Release to Cloud run found." @@ -61,35 +71,26 @@ jobs: exit 1 fi echo "Commit SHA: $commit_sha" - echo "commit_sha=$commit_sha" >> "$GITHUB_OUTPUT" - check-for-new-commits: - runs-on: ubuntu-latest - needs: get-last-successful-release-run - outputs: - latest_commit_sha: ${{ steps.check_new_commits.outputs.latest_commit_sha }} - has_new_commits: ${{ steps.check_new_commits.outputs.has_new_commits }} - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Check for new commits since last Release to Cloud run - id: check_new_commits - run: | - last_released_commit_sha=${{ needs.get-last-successful-release-run.outputs.commit_sha }} - echo "Last released commit SHA: $last_released_commit_sha" + # Check for new commits + echo "Last released commit SHA: $commit_sha" latest_commit_sha=$(git rev-parse ${{ inputs.ref }}) echo "Latest commit SHA: $latest_commit_sha" - if [ "$last_released_commit_sha" != "$latest_commit_sha" ]; then + if [ "$commit_sha" != "$latest_commit_sha" ]; then echo "has_new_commits=true" >> "$GITHUB_OUTPUT" else echo "has_new_commits=false" >> "$GITHUB_OUTPUT" fi echo "latest_commit_sha=$latest_commit_sha" >> "$GITHUB_OUTPUT" + # Print the commit SHAs for dry run + if [ "${{ inputs.dry_run }}" == "true" ]; then + echo "::notice title=Dry Run::Last released commit SHA: $commit_sha, Latest commit SHA: $latest_commit_sha" + fi + invoke-reusable-workflow: needs: check-for-new-commits - if: needs.check-for-new-commits.outputs.has_new_commits == 'true' + if: needs.check-for-new-commits.outputs.has_new_commits == 'true' && inputs.dry_run == 'false' name: "Build and Release Internally" uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" with: From 450e9a744f4af559ec4d0fb8e10043d41adceae2 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Mon, 22 Jul 2024 13:58:40 -0500 Subject: [PATCH 9/9] update fetch reach to try and make sure we can check everything --- .github/workflows/release-internal.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 049f26e61..a56e14f7d 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -54,6 +54,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Fetch ref + run: | + git fetch origin ${{ inputs.ref }}:${{ inputs.ref }} - name: Get last successful Release to Cloud run details and check for new commits id: check_new_commits