From ed4d20dc5c10e43f6e34044bbc7b685616422dc8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 1 Oct 2024 22:33:15 +0100 Subject: [PATCH] Add a automatic PR-revert utility for failing buildbots Signed-off-by: Pablo Galindo --- buildbothammer/README.md | 63 +++ buildbothammer/pyproject.toml | 16 + buildbothammer/src/buildbothammer/__init__.py | 381 ++++++++++++++++++ buildbothammer/src/buildbothammer/__main__.py | 6 + 4 files changed, 466 insertions(+) create mode 100644 buildbothammer/README.md create mode 100644 buildbothammer/pyproject.toml create mode 100644 buildbothammer/src/buildbothammer/__init__.py create mode 100644 buildbothammer/src/buildbothammer/__main__.py diff --git a/buildbothammer/README.md b/buildbothammer/README.md new file mode 100644 index 00000000..68427915 --- /dev/null +++ b/buildbothammer/README.md @@ -0,0 +1,63 @@ +# BuildbotHammer + +This app automatically creates revert Pull Requests for failing builds in the +CPython project. It monitors Buildbot for failing builds, identifies the +commit that caused the failure, and creates a revert PR on GitHub. It also sends +notifications to Discord. + +## Prerequisites + +- Python 3.11+ +- Git +- A GitHub account with fork of the CPython repository +- A Discord server with webhook set up (for notifications) + +## Installation + +1. Clone this repository: + ``` + git clone https://github.com/python/buildmasterconf.git + cd buildmasterconf/buildbot-hammer + ``` + +2. Install the required Python packages: + ``` + pip install -e . + ``` + +3. Set up environment variables: + ``` + export GITHUB_TOKEN="your-github-personal-access-token" + export DISCORD_WEBHOOK_URL="your-discord-webhook-url" + ``` + +4. Update the script with your GitHub username: + Open the script and replace `FORK_OWNER` variable with your GitHub username. + +5. (Optional) Update the `REPO_CLONE_DIR` path if you want to use a different location for the local CPython clone. + +## Usage + +Run the script with: + +``` +python -m buildbothammer +``` + +The script will: +1. Check Buildbot for failing builds +2. For each failing build, it will: + - Identify the commit that caused the failure + - Create a new branch in your fork + - Revert the problematic commit + - Create a Pull Request to the main CPython repository + - Send a notification to the configured Discord channel + +## Configuration + +- `BUILDBOT_API`: The Buildbot API endpoint (default: "http://buildbot.python.org/api/v2") +- `BUILDBOT_URL`: The Buildbot URL for generating links (default: "http://buildbot.python.org/#/") +- `REPO_OWNER`: The owner of the main repository (default: "python") +- `REPO_NAME`: The name of the repository (default: "cpython") +- `FORK_OWNER`: Your GitHub username (default: "$REPO_OWNER") +- `REPO_CLONE_DIR`: The directory for the local clone of the repository \ No newline at end of file diff --git a/buildbothammer/pyproject.toml b/buildbothammer/pyproject.toml new file mode 100644 index 00000000..31f7a62b --- /dev/null +++ b/buildbothammer/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "buildbothammer" +version = "0.1.0" +description = "Automatic revert failing CPython PRs" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "requests>=2.32.3", + "pygithub>=2.4.0", + "filelock>=3.16.1", + "aiohttp", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/buildbothammer/src/buildbothammer/__init__.py b/buildbothammer/src/buildbothammer/__init__.py new file mode 100644 index 00000000..b301bc68 --- /dev/null +++ b/buildbothammer/src/buildbothammer/__init__.py @@ -0,0 +1,381 @@ +import aiohttp +import asyncio +import json +import os +import logging +from pathlib import Path +import subprocess +from filelock import FileLock +from github import Github, GithubException +from github.InputGitTreeElement import InputGitTreeElement +from github.GithubException import GithubException, UnknownObjectException + +# Set up logging +logging.basicConfig( + level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + +# Buildbot API endpoint +BUILDBOT_API = "http://buildbot.python.org/api/v2" +BUILDBOT_URL = "http://buildbot.python.org/#/" + +DISCORD_WEBHOOK_URL = os.environ.get("DISCORD_WEBHOOK_URL") +if not DISCORD_WEBHOOK_URL: + logger.warning( + "DISCORD_WEBHOOK_URL environment variable is not set. Discord notifications will be disabled." + ) + +# GitHub API token (replace with your own) +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") +if not GITHUB_TOKEN: + raise EnvironmentError("GITHUB_TOKEN environment variable is not set") + +N_BUILDS = 200 + +# GitHub repository details +REPO_OWNER = "python" +REPO_NAME = "cpython" +FORK_OWNER = os.getenv("FORK_OWNER", REPO_OWNER) +REPO_CLONE_DIR = Path("/tmp/cpython") # Replace with actual path +LOCK_FILE = REPO_CLONE_DIR.parent / f"{REPO_NAME}.lock" +MIN_CONSECUTIVE_FAILURES = 3 + + +class BuildbotError(Exception): + """Exception raised for errors in the Buildbot API.""" + + pass + + +class GitHubError(Exception): + """Exception raised for errors in GitHub operations.""" + + pass + + +def generate_build_status_graph(builds): + status_map = { + 0: "🟩", # Success + 2: "🟥", # Failure + None: "⬜", # Not completed + 1: "🟧", # Warnings or unstable + } + return "".join(status_map.get(build["results"], "⬜") for build in builds) + + +async def send_discord_notification(session, message): + if not DISCORD_WEBHOOK_URL: + logger.warning("Discord notification not sent: DISCORD_WEBHOOK_URL is not set") + return + + payload = {"content": message} + + try: + async with session.post(DISCORD_WEBHOOK_URL, json=payload) as response: + if response.status == 204: + logger.info("Successfully sent Discord notification") + else: + logger.error( + f"Failed to send Discord notification. Status: {response.status}" + ) + except Exception as e: + logger.error(f"Error sending Discord notification: {str(e)}") + + +async def fetch(session, url): + try: + async with session.get(url) as response: + response.raise_for_status() + content_type = response.headers.get("Content-Type", "") + if "application/json" in content_type: + return await response.json() + elif "text/plain" in content_type: + text = await response.text() + return json.loads(text) + else: + raise ValueError(f"Unexpected content type: {content_type}") + except aiohttp.ClientResponseError as e: + raise BuildbotError(f"HTTP error {e.status} while fetching {url}: {e.message}") + except json.JSONDecodeError: + raise BuildbotError(f"Failed to decode JSON from {url}") + + +async def get_builder_builds(session, builder, limit=N_BUILDS): + builds_url = f"{BUILDBOT_API}/builders/{builder['builderid']}/builds?limit={limit}&&order=-complete_at" + builds = await fetch(session, builds_url) + status_graph = generate_build_status_graph(builds["builds"]) + return builder, builds["builds"], status_graph + + +def is_failing_builder(builds): + failing_streak = 0 + first_failing_build = None + for build in builds: + if not build["complete"]: + continue + if build["results"] == 2: # 2 indicates a failed build + failing_streak += 1 + first_failing_build = build + continue + elif build["results"] == 0: # 0 indicates a successful build + if failing_streak >= MIN_CONSECUTIVE_FAILURES: + return True, first_failing_build + return False, None + failing_streak = 0 + return False, None + + +async def get_failing_builders(session, limit=N_BUILDS): + logger.info("Fetching failing builders") + builders_url = f"{BUILDBOT_API}/builders" + builders = await fetch(session, builders_url) + builders = builders["builders"] + + relevant_builders = [ + b for b in builders if "3.x" in b["tags"] and "stable" in b["tags"] + ] + + builder_builds = await asyncio.gather( + *[get_builder_builds(session, builder, limit) for builder in relevant_builders], + return_exceptions=True, + ) + + failing_builders = [] + all_builders_status = [] + for result in builder_builds: + if isinstance(result, Exception): + logger.error(f"Error fetching builder builds: {result}") + continue + builder, builds, status_graph = result + is_failing, last_failing_build = is_failing_builder(builds) + if last_failing_build: + status_graph = list(status_graph) + status_graph[builds.index(last_failing_build)] = "🟪" + status_graph = "".join(status_graph) + print(f"{builder['name']}\n{status_graph}\n") + all_builders_status.append((builder["name"], status_graph)) + if is_failing: + logger.info( + f"Found failing builder: {builder['name']} with last failing build {last_failing_build['buildid']}" + ) + failing_builders.append((builder, last_failing_build)) + else: + logger.debug(f"Builder {builder['name']} is not failing") + + logger.info(f"Total failing builders found: {len(failing_builders)}") + return failing_builders, all_builders_status + + +async def get_change_request(session, build): + logger.info(f"Fetching change request for build: {build['buildid']}") + changes_url = f"{BUILDBOT_API}/builds/{build['buildid']}/changes" + changes = await fetch(session, changes_url) + + if len(changes["changes"]) == 1: + logger.debug(f"Found change request for build {build['buildid']}") + return changes["changes"][0] + else: + logger.debug(f"No single change request found for build {build['buildid']}") + return None + + +def run_command(command, cwd=None, check=True, capture_output=False): + logger.debug(f"Running command: {' '.join(command)}") + subprocess.run( + command, cwd=cwd, capture_output=capture_output, text=True, check=check + ) + + +def ensure_repo_clone(): + if not REPO_CLONE_DIR.exists(): + logger.info(f"Cloning repository to {REPO_CLONE_DIR}") + clone_url = f"https://{GITHUB_TOKEN}@github.com/{REPO_OWNER}/{REPO_NAME}.git" + run_command(["git", "clone", clone_url, str(REPO_CLONE_DIR)]) + else: + logger.info("Updating existing repository clone") + run_command(["git", "fetch", "--all"], cwd=str(REPO_CLONE_DIR)) + run_command(["git", "reset", "--hard", "origin/main"], cwd=str(REPO_CLONE_DIR)) + + +def create_revert_pr(commit_sha, builder, failing_build): + logger.info(f"Creating revert PR for commit: {commit_sha}") + g = Github(GITHUB_TOKEN) + + try: + # Get the main repository and your fork + main_repo = g.get_repo(f"{REPO_OWNER}/{REPO_NAME}") + + with FileLock(LOCK_FILE): + ensure_repo_clone() + + # Set up Git configuration + run_command( + ["git", "config", "user.name", "Your Name"], cwd=str(REPO_CLONE_DIR) + ) + run_command( + ["git", "config", "user.email", "your.email@example.com"], + cwd=str(REPO_CLONE_DIR), + ) + + # Create and checkout a new branch + branch_name = f"revert-{commit_sha[:7]}" + run_command(["git", "checkout", "-b", branch_name], cwd=str(REPO_CLONE_DIR)) + logger.info(f"Created and checked out new branch: {branch_name}") + + # Perform the revert + run_command( + ["git", "revert", "--no-edit", commit_sha], cwd=str(REPO_CLONE_DIR) + ) + logger.info(f"Successfully reverted commit {commit_sha}") + + # Push the changes + run_command( + [ + "git", + "push", + "-f", + f"https://{GITHUB_TOKEN}@github.com/{FORK_OWNER}/{REPO_NAME}.git", + branch_name, + ], + cwd=str(REPO_CLONE_DIR), + ) + logger.info("Pushed changes to fork") + + # Get the commit message of the reverted commit + commit_to_revert = main_repo.get_commit(commit_sha) + original_commit_message = commit_to_revert.commit.message.split("\n")[ + 0 + ] # Get first line of commit message + + # Get the commit message and author of the reverted commit + commit_to_revert = main_repo.get_commit(commit_sha) + original_commit_message = commit_to_revert.commit.message.split("\n")[ + 0 + ] # Get first line of commit message + author = ( + commit_to_revert.author.login if commit_to_revert.author else "Unknown" + ) + commit_date = commit_to_revert.commit.author.date.strftime( + "%Y-%m-%d %H:%M:%S UTC" + ) + + # Create a more detailed PR description + pr_description = f""" +🔄 **Automatic Revert** + +This PR automatically reverts commit {commit_sha} due to a failing build. + +📊 **Build Information:** +- Builder: {builder['name']} +- Build Number: {failing_build['number']} +- Build URL: {BUILDBOT_URL}/builders/{failing_build['builderid']}/builds/{failing_build['buildid']} + +💻 **Reverted Commit Details:** +- SHA: `{commit_sha}` +- Author: {author} +- Date: {commit_date} +- Message: "{original_commit_message}" + +🛠 **Next Steps:** +1. Investigate the cause of the build failure. +2. If the revert is necessary, merge this PR. +3. If the revert is not necessary, close this PR and fix the original issue. + +⚠️ Please review this revert carefully before merging. + +cc @{author} - Your attention may be needed on this revert. +""" + + Create the pull request + pr = main_repo.create_pull( + title=f"🔄 Revert: {original_commit_message}", + body=pr_description, + head=f"{FORK_OWNER}:{branch_name}", + base="main", + ) + + logger.info(f"Created revert PR: {pr.html_url}") + + # Prepare Discord notification message + discord_message = f""" +🚨 **Automatic Revert PR Created** + +A build failure has triggered an automatic revert. Details: + +🔗 **PR Link:** {pr.html_url} +🏗 **Failed Build:** {BUILDBOT_URL}/builders/{failing_build['builderid']}/builds/{failing_build['buildid']} +🔄 **Reverted Commit:** `{commit_sha}` +✍️ **Original Author:** {author} +📅 **Commit Date:** {commit_date} +💬 **Commit Message:** "{original_commit_message}" + +Please review and take appropriate action! +""" + + return pr.html_url, discord_message + except GithubException as e: + raise GitHubError(f"GitHub API error: {e.status}, {e.data}") + except Exception as e: + raise GitHubError(f"Unexpected error while creating revert PR: {str(e)}") + finally: + # Always return to main branch and clean up + run_command( + ["git", "revert", "--abort"], + cwd=str(REPO_CLONE_DIR), + check=False, + capture_output=True, + ) + run_command(["git", "checkout", "main"], cwd=str(REPO_CLONE_DIR)) + run_command(["git", "branch", "-D", branch_name], cwd=str(REPO_CLONE_DIR)) + + +async def process_builder(session, builder, first_failing_build): + try: + # Get the change request for the first failing build + change_request = await get_change_request(session, first_failing_build) + + if change_request and "sourcestamp" in change_request: + commit_sha = change_request["sourcestamp"]["revision"] + pr_url, discord_message = create_revert_pr( + commit_sha, builder, first_failing_build + ) + if pr_url: + logger.info(f"Created revert PR for commit {commit_sha}: {pr_url}") + await send_discord_notification(session, discord_message) + else: + logger.error(f"Failed to create revert PR for commit {commit_sha}") + else: + logger.warning( + f"No suitable change request found for builder: {builder['name']}" + ) + except (BuildbotError, GitHubError) as e: + logger.error(f"Error processing builder {builder['name']}: {str(e)}") + raise + + +async def main(): + logger.info("Starting the Async Buildbot and GitHub Revert Script") + + async with aiohttp.ClientSession() as session: + try: + failing_builders, all_builders_status = await get_failing_builders(session) + + results = await asyncio.gather( + *[ + process_builder(session, builder, first_failing_build) + for builder, first_failing_build in failing_builders + ], + return_exceptions=True, + ) + for result in results: + if isinstance(result, Exception): + logger.error(f"Error in processing a builder: {result}") + except Exception as e: + logger.error(f"An error occurred in the main execution: {str(e)}") + finally: + logger.info("Script execution completed") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/buildbothammer/src/buildbothammer/__main__.py b/buildbothammer/src/buildbothammer/__main__.py new file mode 100644 index 00000000..870d3aa8 --- /dev/null +++ b/buildbothammer/src/buildbothammer/__main__.py @@ -0,0 +1,6 @@ +import asyncio + +from . import main + +if __name__ == "__main__": + asyncio.run(main())