diff --git a/.github/repo_policies/BOT_APPROVED_FILES b/.github/repo_policies/BOT_APPROVED_FILES index 3f2a737c3f2..8a7dca89a8e 100644 --- a/.github/repo_policies/BOT_APPROVED_FILES +++ b/.github/repo_policies/BOT_APPROVED_FILES @@ -9,3 +9,4 @@ ic-os/setupos/context/docker-base.dev ic-os/setupos/context/docker-base.prod mainnet-canisters.json testnet/mainnet_revisions.json +mainnet-canisters.json diff --git a/.github/workflows/update-mainnet-revisions.yaml b/.github/workflows/update-mainnet-revisions.yaml index d96a9d95857..d847894404e 100644 --- a/.github/workflows/update-mainnet-revisions.yaml +++ b/.github/workflows/update-mainnet-revisions.yaml @@ -5,6 +5,10 @@ on: - cron: "10 * * * *" workflow_dispatch: +defaults: + run: + shell: bash + jobs: update-ic-versions-file: runs-on: ubuntu-latest @@ -28,5 +32,41 @@ jobs: run: | set -eEuxo pipefail - time python ci/src/mainnet_revisions/mainnet_revisions.py - shell: bash + time python ci/src/mainnet_revisions/mainnet_revisions.py subnets + + update-nervous-system-wasms: + runs-on: + labels: dind-small + container: + image: ghcr.io/dfinity/ic-build@sha256:4fd13b47285e783c3a6f35aadd9559d097c0de162a1cf221ead66ab1598d5d45 + options: >- + -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info + steps: + - name: Create GitHub App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.PR_AUTOMATION_BOT_PUBLIC_APP_ID }} + private-key: ${{ secrets.PR_AUTOMATION_BOT_PUBLIC_PRIVATE_KEY }} + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + token: ${{ steps.app-token.outputs.token }} + + - name: Setup gh cli + uses: ksivamuthu/actions-setup-gh-cli@v3 + with: + version: 2.53.0 + + - name: Update Mainnet canisters file + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -eEuxo pipefail + + # Leftover from previous step of setting up gh cli + rm gh_tar + + time python ci/src/mainnet_revisions/mainnet_revisions.py canisters diff --git a/ci/src/mainnet_revisions/mainnet_revisions.py b/ci/src/mainnet_revisions/mainnet_revisions.py index 1c28cef7498..24e68a69d04 100644 --- a/ci/src/mainnet_revisions/mainnet_revisions.py +++ b/ci/src/mainnet_revisions/mainnet_revisions.py @@ -2,20 +2,105 @@ import argparse import json import logging +import os import pathlib import subprocess -import sys import urllib.request +from enum import Enum +from typing import List -# from pylib.ic_deployment import IcDeployment - -SAVED_VERSIONS_PATH = "testnet/mainnet_revisions.json" +SAVED_VERSIONS_SUBNETS_PATH = "testnet/mainnet_revisions.json" nns_subnet_id = "tdb26-jop6k-aogll-7ltgs-eruif-6kk7m-qpktf-gdiqx-mxtrf-vb5e6-eqe" app_subnet_id = "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe" PUBLIC_DASHBOARD_API = "https://ic-api.internetcomputer.org" +SAVED_VERSIONS_CANISTERS_PATH = "mainnet-canisters.json" + + +class Command(Enum): + SUBNETS = 1 + CANISTERS = 2 + + +def sync_main_branch_and_checkout_branch( + repo_root: pathlib.Path, main_branch: str, branch_to_checkout: str, logger: logging.Logger +): + if not repo_root.exists(): + raise Exception("Expected dir %s to exist", repo_root.name) + + subprocess.call(["git", "fetch", "--depth=1", "--no-tags", "origin", f"{main_branch}:{main_branch}"], cwd=repo_root) + + result = subprocess.run(["git", "status", "--porcelain"], stdout=subprocess.PIPE, text=True, check=True) + if result.stdout.strip(): + raise Exception("Found uncommited work! Commit and then proceed. Uncommited work:\n%s", result.stdout.strip()) + + if subprocess.call(["git", "checkout", branch_to_checkout], cwd=repo_root) == 0: + # The branch already exists, update the existing MR + logger.info("Found an already existing target branch") + else: + subprocess.check_call(["git", "checkout", "-b", branch_to_checkout], cwd=repo_root) + subprocess.check_call(["git", "reset", "--hard", f"origin/{main_branch}"], cwd=repo_root) + + +def commit_and_create_pr( + repo: str, + repo_root: pathlib.Path, + branch: str, + check_for_updates_in_paths: List[str], + logger: logging.Logger, + commit_message: str, + description: str, +): + git_modified_files = subprocess.check_output(["git", "ls-files", "--modified", "--others"], cwd=repo_root).decode( + "utf8" + ) + + paths_to_add = [path for path in check_for_updates_in_paths if path in git_modified_files] + + if len(paths_to_add) > 0: + logger.info("Creating/updating a MR that updates the saved NNS subnet revision") + cmd = ["git", "add"] + paths_to_add + logger.info("Running command '%s'", " ".join(cmd)) + subprocess.check_call(cmd, cwd=repo_root) + cmd = [ + "git", + "-c", + "user.name=CI Automation", + "-c", + "user.email=infra+github-automation@dfinity.org", + "commit", + "-m", + commit_message, + ] + paths_to_add + logger.info("Running command '%s'", " ".join(cmd)) + subprocess.check_call( + cmd, + cwd=repo_root, + ) + subprocess.check_call(["git", "push", "origin", branch, "-f"], cwd=repo_root) + + if not subprocess.check_output( + ["gh", "pr", "list", "--head", branch, "--repo", repo], + cwd=repo_root, + ).decode("utf8"): + subprocess.check_call( + [ + "gh", + "pr", + "create", + "--head", + branch, + "--repo", + repo, + "--body", + description, + "--title", + commit_message, + ], + cwd=repo_root, + ) -def get_saved_versions(repo_root: pathlib.Path): +def get_saved_versions(repo_root: pathlib.Path, file_path: pathlib.Path): """ Return a dict with all saved versions. @@ -32,33 +117,33 @@ def get_saved_versions(repo_root: pathlib.Path): } } """ - saved_versions_path = repo_root / SAVED_VERSIONS_PATH - if saved_versions_path.exists(): - with open(saved_versions_path, "r", encoding="utf-8") as f: + full_path = repo_root / file_path + if full_path.exists(): + with open(full_path, "r", encoding="utf-8") as f: return json.load(f) else: return {} -def update_saved_subnet_version(subnet: str, version: str, repo_root: pathlib.Path): +def update_saved_subnet_version(subnet: str, version: str, repo_root: pathlib.Path, file_path: pathlib.Path): """Update the version that we last saw on a particular IC subnet.""" - saved_versions = get_saved_versions(repo_root=repo_root) + saved_versions = get_saved_versions(repo_root=repo_root, file_path=file_path) subnet_versions = saved_versions.get("subnets", {}) subnet_versions[subnet] = version saved_versions["subnets"] = subnet_versions - with open(repo_root / SAVED_VERSIONS_PATH, "w", encoding="utf-8") as f: + with open(repo_root / SAVED_VERSIONS_SUBNETS_PATH, "w", encoding="utf-8") as f: json.dump(saved_versions, f, indent=2) -def get_saved_nns_subnet_version(repo_root: pathlib.Path): +def get_saved_nns_subnet_version(repo_root: pathlib.Path, file_path: pathlib.Path): """Get the last known version running on the NNS subnet.""" - saved_versions = get_saved_versions(repo_root=repo_root) + saved_versions = get_saved_versions(repo_root=repo_root, file_path=file_path) return saved_versions.get("subnets", {}).get(nns_subnet_id, "") -def get_saved_app_subnet_version(repo_root: pathlib.Path): +def get_saved_app_subnet_version(repo_root: pathlib.Path, file_path: pathlib.Path): """Get the last known version running on an App subnet.""" - saved_versions = get_saved_versions(repo_root=repo_root) + saved_versions = get_saved_versions(repo_root=repo_root, file_path=file_path) return saved_versions.get("subnets", {}).get(app_subnet_id, "") @@ -75,102 +160,113 @@ def get_subnet_replica_version(subnet_id: str) -> str: return latest_replica_version -def get_repo_root() -> str: - return subprocess.run(["git", "rev-parse", "--show-toplevel"], text=True, stdout=subprocess.PIPE).stdout.strip() +def update_mainnet_revisions_subnets_file(repo_root: pathlib.Path, logger: logging.Logger, file_path: pathlib.Path): + current_nns_version = get_subnet_replica_version(nns_subnet_id) + logger.info("Current NNS subnet (%s) revision: %s", nns_subnet_id, current_nns_version) + current_app_subnet_version = get_subnet_replica_version(app_subnet_id) + logger.info("Current App subnet (%s) revision: %s", app_subnet_id, current_app_subnet_version) + + update_saved_subnet_version( + subnet=nns_subnet_id, version=current_nns_version, repo_root=repo_root, file_path=file_path + ) + update_saved_subnet_version( + subnet=app_subnet_id, version=current_app_subnet_version, repo_root=repo_root, file_path=file_path + ) + +def update_mainnet_revisions_canisters_file(repo_root: pathlib.Path, logger: logging.Logger): + cmd = [ + "bazel", + "run", + "--config=ci", + ] + if os.environ.get("CI"): + cmd.append("--repository_cache=/cache/bazel") + cmd.append("//rs/nervous_system/tools/sync-with-released-nervous-system-wasms") -def main(): - """Do the main work.""" + logger.info("Running command: %s", " ".join(cmd)) + subprocess.check_call(cmd, cwd=repo_root) - class HelpfulParser(argparse.ArgumentParser): - """An argparse parser that prints usage on any error.""" - def error(self, message): - sys.stderr.write("error: %s\n" % message) - self.print_help() - sys.exit(2) +def get_logger(level) -> logging.Logger: + FORMAT = "[%(asctime)s] %(levelname)-8s %(message)s" + logging.basicConfig(format=FORMAT, level=level) + return logging.getLogger("logger") - parser = HelpfulParser() +def get_repo_root() -> pathlib.Path: + return pathlib.Path( + subprocess.run(["git", "rev-parse", "--show-toplevel"], text=True, stdout=subprocess.PIPE).stdout.strip() + ) + + +def get_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="GitCiHelper", description="Tool for automating git operations for CI") parser.add_argument("--verbose", "-v", action="store_true", help="Verbose mode") - args = parser.parse_args() + subparsers = parser.add_subparsers(title="subcommands", description="valid commands", help="sub-command help") - if args.verbose: - logging.basicConfig(level=logging.DEBUG) - else: - logging.basicConfig(level=logging.INFO) + parser_subnets = subparsers.add_parser("subnets", help=f"Update {SAVED_VERSIONS_SUBNETS_PATH} file") + parser_subnets.set_defaults(command=Command.SUBNETS) - current_nns_version = get_subnet_replica_version(nns_subnet_id) - logging.info("Current NNS subnet (%s) revision: %s", nns_subnet_id, current_nns_version) - current_app_subnet_version = get_subnet_replica_version(app_subnet_id) - logging.info("Current App subnet (%s) revision: %s", app_subnet_id, current_app_subnet_version) + parser_canisters = subparsers.add_parser("canisters", help=f"Update {SAVED_VERSIONS_CANISTERS_PATH} file") + parser_canisters.set_defaults(command=Command.CANISTERS) - repo = "dfinity/ic" + return parser - repo_root = pathlib.Path(get_repo_root()) - if not repo_root.parent.exists(): - raise Exception("Expected dir %s to exist", repo_root.name) +def main(): + """Do the main work.""" - branch = "ic-mainnet-revisions" - subprocess.call(["git", "fetch", "origin", "master:master"], cwd=repo_root) + parser = get_parser() + args = parser.parse_args() + logger = get_logger(logging.DEBUG if args.verbose else logging.INFO) - result = subprocess.run(["git", "status", "--porcelain"], stdout=subprocess.PIPE, text=True, check=True) - if result.stdout.strip(): - logging.error("Found uncommited work! Commit and then proceed.") - exit(2) + repo = "dfinity/ic" + repo_root = get_repo_root() + main_branch = "master" - if subprocess.call(["git", "checkout", branch], cwd=repo_root) == 0: - # The branch already exists, update the existing MR - logging.info("Found an already existing target branch") - else: - subprocess.check_call(["git", "checkout", "-b", branch], cwd=repo_root) - subprocess.check_call(["git", "reset", "--hard", "origin/master"], cwd=repo_root) + if not hasattr(args, "command"): + parser.print_help() + exit(1) - update_saved_subnet_version(subnet=nns_subnet_id, version=current_nns_version, repo_root=pathlib.Path(repo_root)) - update_saved_subnet_version( - subnet=app_subnet_id, version=current_app_subnet_version, repo_root=pathlib.Path(repo_root) - ) - git_modified_files = subprocess.check_output(["git", "ls-files", "--modified", "--others"], cwd=repo_root).decode( - "utf8" - ) - if SAVED_VERSIONS_PATH in git_modified_files: - logging.info("Creating/updating a MR that updates the saved NNS subnet revision") - subprocess.check_call(["git", "add", SAVED_VERSIONS_PATH], cwd=repo_root) - subprocess.check_call( - [ - "git", - "-c", - "user.name=CI Automation", - "-c", - "user.email=infra+github-automation@dfinity.org", - "commit", - "-m", - "chore: Update Mainnet IC revisions file", - SAVED_VERSIONS_PATH, - ], - cwd=repo_root, - ) - subprocess.check_call(["git", "push", "origin", branch, "-f"], cwd=repo_root) + pr_description = """{description} - if not subprocess.check_output( - ["gh", "pr", "list", "--head", branch, "--repo", repo], - cwd=repo_root, - ).decode("utf8"): - subprocess.check_call( - [ - "gh", - "pr", - "create", - "--head", - branch, - "--repo", - repo, - "--fill", - ], - cwd=repo_root, - ) +This PR is created automatically using [`mainnet_revisions.py`](https://github.com/dfinity/ic/blob/master/ci/src/mainnet_revisions/mainnet_revisions.py) + """ + + if args.command == Command.SUBNETS: + branch = "ic-mainnet-revisions" + sync_main_branch_and_checkout_branch(repo_root, main_branch, branch, logger) + update_mainnet_revisions_subnets_file(repo_root, logger, pathlib.Path(SAVED_VERSIONS_SUBNETS_PATH)) + commit_and_create_pr( + repo, + repo_root, + branch, + [SAVED_VERSIONS_SUBNETS_PATH], + logger, + "chore: Update Mainnet IC revisions subnets file", + pr_description.format( + description="Update mainnet revisions file to include the latest version released on the mainnet." + ), + ) + elif args.command == Command.CANISTERS: + branch = "ic-nervous-system-wasms" + sync_main_branch_and_checkout_branch(repo_root, main_branch, branch, logger) + update_mainnet_revisions_canisters_file(repo_root, logger) + commit_and_create_pr( + repo, + repo_root, + branch, + [SAVED_VERSIONS_CANISTERS_PATH], + logger, + "chore: Update Mainnet IC revisions canisters file", + pr_description.format( + description="Update mainnet system canisters revisions file to include the latest WASM version released on the mainnet." + ), + ) + else: + raise Exception("This shouldn't happen") if __name__ == "__main__":