From 295eea2b50567efdf7dae7b5976d5d2e225726e3 Mon Sep 17 00:00:00 2001 From: GPK Date: Sat, 16 Nov 2024 23:53:37 +0000 Subject: [PATCH] Add cherry picker (#44102) * adding cherry picker to support backport pr's * update checkout_no_credentails logic to ignore for backport step --- .github/workflows/backport-branch-finder.yml | 77 ++++++++++++ .github/workflows/backport-cli.yml | 116 ++++++++++++++++++ dev/backport/.cherry_picker.toml | 23 ++++ dev/backport/update_backport_status.py | 91 ++++++++++++++ .../ci/pre_commit/checkout_no_credentials.py | 7 ++ 5 files changed, 314 insertions(+) create mode 100644 .github/workflows/backport-branch-finder.yml create mode 100644 .github/workflows/backport-cli.yml create mode 100644 dev/backport/.cherry_picker.toml create mode 100644 dev/backport/update_backport_status.py diff --git a/.github/workflows/backport-branch-finder.yml b/.github/workflows/backport-branch-finder.yml new file mode 100644 index 0000000000000..c52941792c506 --- /dev/null +++ b/.github/workflows/backport-branch-finder.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Backport branch finder +on: # yamllint disable-line rule:truthy + push: + branches: + - main + +jobs: + get-pr-info: + name: "Get PR information" + runs-on: ubuntu-latest + outputs: + branches: ${{ steps.pr-info.outputs.branches }} + commit-sha: ${{ github.sha }} + steps: + - name: Get commit SHA + id: get-sha + run: echo "COMMIT_SHA=${GITHUB_SHA}" >> $GITHUB_ENV + + - name: Find PR information + id: pr-info + uses: actions/github-script@v7 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + script: | + const { data: pullRequest } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: process.env.GITHUB_SHA + }); + if (pullRequest.length > 0) { + const pr = pullRequest[0]; + const backportBranches = pr.labels + .filter(label => label.name.startsWith('backport-to-')) + .map(label => label.name.replace('backport-to-', '')); + + console.log(`Commit ${process.env.GITHUB_SHA} is associated with PR ${pr.number}`); + console.log(`Backport branches: ${backportBranches}`); + core.setOutput('branches', JSON.stringify(backportBranches)); + } else { + console.log('No pull request found for this commit.'); + core.setOutput('branches', '[]'); + } + + trigger-backport: + name: "Trigger Backport" + uses: ./.github/workflows/backport-cli.yml + needs: get-pr-info + if: ${{ needs.get-pr-info.outputs.branches != '[]' }} + strategy: + matrix: + branch: ${{ fromJSON(needs.get-pr-info.outputs.branches) }} + fail-fast: false + permissions: + contents: write + pull-requests: write + with: + target-branch: ${{ matrix.branch }} + commit-sha: ${{ needs.get-pr-info.outputs.commit-sha }} diff --git a/.github/workflows/backport-cli.yml b/.github/workflows/backport-cli.yml new file mode 100644 index 0000000000000..f521dcb7d8368 --- /dev/null +++ b/.github/workflows/backport-cli.yml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Backport Commit +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + commit-sha: + description: "Commit sha to backport." + required: true + type: string + target-branch: + description: "Target branch to backport." + required: true + type: string + + workflow_call: + inputs: + commit-sha: + description: "Commit sha to backport." + required: true + type: string + target-branch: + description: "Target branch to backport." + required: true + type: string + +permissions: + contents: write + pull-requests: write +jobs: + backport: + runs-on: ubuntu-latest + + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + id: checkout-for-backport + uses: actions/checkout@v4 + with: + persist-credentials: true + fetch-depth: 0 + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install cherry-picker==2.4.0 requests==2.32.3 + + - name: Run backport script + id: execute-backport + env: + GH_AUTH: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global user.email "name@example.com" + git config --global user.name "Your Name" + set +e + { + echo 'cherry_picker_output<> "${GITHUB_OUTPUT}" + continue-on-error: true + + - name: Parse backport output + id: parse-backport-output + run: | + set +e + echo "${{ steps.execute-backport.outputs.cherry_picker_output }}" + + url=$(echo "${{ steps.execute-backport.outputs.cherry_picker_output }}" | \ + grep -o 'Backport PR created at https://[^ ]*' | \ + awk '{print $5}') + + url=${url:-"EMPTY"} + if [ "$url" == "EMPTY" ]; then + # If the backport failed, abort the workflow + cherry_picker --abort + fi + echo "backport-url=$url" >> "${GITHUB_OUTPUT}" + continue-on-error: true + + - name: Update Status + id: backport-status + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + COMMIT_INFO_URL="https://api.github.com/repos/${{ github.repository }}/commits/" + COMMIT_INFO_URL="${COMMIT_INFO_URL}${{ inputs.commit-sha }}/pulls" + + PR_NUMBER=$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ github.repository }}/commits/${{ inputs.commit-sha }}/pulls \ + --jq '.[0].number') + + python ./dev/backport/update_backport_status.py \ + ${{ steps.parse-backport-output.outputs.backport-url }} \ + ${{ inputs.commit-sha }} ${{ inputs.target-branch }} \ + "$PR_NUMBER" diff --git a/dev/backport/.cherry_picker.toml b/dev/backport/.cherry_picker.toml new file mode 100644 index 0000000000000..677644159c78a --- /dev/null +++ b/dev/backport/.cherry_picker.toml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +team = "apache" +repo = "airflow" +"check_sha"= "a85d94e6cdcd09efe93c3acee0b4ce5c9508bc23" +fix_commit_msg = false +default_branch = "main" +require_version_in_branch_name=false diff --git a/dev/backport/update_backport_status.py b/dev/backport/update_backport_status.py new file mode 100644 index 0000000000000..e6e3ce064ce98 --- /dev/null +++ b/dev/backport/update_backport_status.py @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import sys + +import requests + + +def get_success_comment(branch: str, pr_url: str, pr_number: str): + shield_url = f"https://img.shields.io/badge/PR-{pr_number}-blue" + comment = f"""### Backport successfully created: {branch}\n\n + + + + + + + + + + +
StatusBranchResult
✅{branch}PR Link
""" + return comment + + +def get_failure_comment(branch: str, commit_sha_url: str, commit_sha: str): + commit_shield_url = f"https://img.shields.io/badge/Commit-{commit_sha[:7]}-red" + comment = f"""### Backport failed to create: {branch}. View the failure log Run details \n\n + + + + + + + + + + +
StatusBranchResult
❌{branch}Commit Link
""" + return comment + + +def add_comments(backport_url: str, target_branch: str, commit_sha: str, source_pr_number: str): + if backport_url.strip() != "EMPTY": + pr_number = backport_url.split("/")[-1] + comment = get_success_comment(branch=target_branch, pr_url=backport_url, pr_number=pr_number) + else: + commit_sha_url = f"https://github.com/{os.getenv('REPOSITORY')}/commit/{commit_sha}" + comment = get_failure_comment( + branch=target_branch, commit_sha_url=commit_sha_url, commit_sha=commit_sha + ) + + token = os.getenv("GH_TOKEN") + comment_url = f"https://api.github.com/repos/{os.getenv('REPOSITORY')}/issues/{source_pr_number}/comments" + + headers = {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"} + + data = {"body": comment} + + try: + response = requests.post(comment_url, headers=headers, json=data) + if not (response.status_code == 201): + raise requests.HTTPError(response=response) + except requests.HTTPError as e: + print(e) + print(f"Error: Failed to add comments to pr {source_pr_number}") + sys.exit(e.response.status_code) + + +if __name__ == "__main__": + bp_url = sys.argv[1] + commit = sys.argv[2] + tg_branch = sys.argv[3] + source_pr = sys.argv[4] + add_comments(backport_url=bp_url, target_branch=tg_branch, commit_sha=commit, source_pr_number=source_pr) diff --git a/scripts/ci/pre_commit/checkout_no_credentials.py b/scripts/ci/pre_commit/checkout_no_credentials.py index 02e8f0a20f77a..02a720eda6d3a 100755 --- a/scripts/ci/pre_commit/checkout_no_credentials.py +++ b/scripts/ci/pre_commit/checkout_no_credentials.py @@ -57,6 +57,13 @@ def check_file(the_file: Path) -> int: # build. This is ok for security, because we are pushing it only in the `main` branch # of the repository and only for unprotected constraints branch continue + if step.get("id") == "checkout-for-backport": + # This is a special case - we are ok with persisting credentials in backport + # step, because we need them to push backport branch back to the repository in + # backport checkout-for-backport step and create pr for cherry-picker. This is ok for + # security, because cherry picker pushing it only in the `main` branch of the repository + # and only for unprotected backport branch + continue persist_credentials = with_clause.get("persist-credentials") if persist_credentials is None: console.print(