diff --git a/update-shell.nix b/update-shell.nix new file mode 100644 index 0000000..ae73cd4 --- /dev/null +++ b/update-shell.nix @@ -0,0 +1,15 @@ +{ pkgs ? import ./pkgs.nix {} }: +pkgs.mkShell { + name = "update-py-shell"; + packages = with pkgs; [ + python3 + cacert + git + nix + nix-update + ]; + + meta = { + description = "Shell for update.py"; + }; +} diff --git a/update.py b/update.py new file mode 100755 index 0000000..c112c14 --- /dev/null +++ b/update.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +# updates the revision hash for each upstream package. +# for each updated package, this checks the derivation can be built +# then commits its results. + +# usage in nix-shell: +# nix-shell ./update-shell.nix --pure --run './update.py ...' + +import os +import json +import logging +import argparse +import subprocess +import urllib.request + +from typing import cast, Literal +from dataclasses import dataclass, field, fields + +log = logging.getLogger() + +@dataclass +class Package: + attr: str + repo: str # github owner/repository + branch: str = '' + then: list[str] = field(default_factory=list) # users of this package, used for testing dependent builds. + + def repo_api(self) -> str: + return f"https://api.github.com/repos/{self.repo}" + + def commits_api(self) -> str: + assert self.branch + return f"https://api.github.com/repos/{self.repo}/commits/{self.branch}" + + def compare_api(self, base: str) -> str: + return f"https://api.github.com/repos/{self.repo}/compare/{base}...{self.branch}" + + def commits_atom(self) -> str: + return f"https://github.com/{self.repo}/commits/{self.branch}.atom" + + def compare_link(self, base: str) -> str: + return self.compare_permalink(base, self.branch) + + def compare_permalink(self, base: str, target: str) -> str: + return f"https://github.com/{self.repo}/compare/{base}...{target}" + + def repo_git(self) -> str: + return f'https://github.com/{self.repo}.git' + + def fetch_default_branch(self) -> str: + out = run(['git', 'ls-remote', '--symref', self.repo_git(), 'HEAD'], + stdout=subprocess.PIPE).stdout.decode('utf-8') + return out.split('\t')[0].split('/')[-1] + + def fetch_commits_behind(self, base: str) -> int: + return ( + curl_raw(self.compare_link(base) + '.patch') + .replace('\r', '') + .count('\n---\n') + ) + + def fetch_latest_commit(self) -> str: + out = curl_raw(self.commits_atom()) + marker = f'https://github.com/{self.repo}/commit/' + left = out.find(marker, out.find('')) + return out[left + len(marker):][:40] + +@dataclass +class Args: + mode: Literal['check', 'upgrade'] + file: str + rest: list[str] + +PACKAGES: list[Package] = [ + Package('asli', 'UQ-PAC/aslp', then=['aslp']), + Package('bap-asli-plugin', 'UQ-PAC/bap-asli-plugin', then=['bap-aslp']), + Package('basil', 'UQ-PAC/bil-to-boogie-translator'), + Package('bap-uq-pac', 'UQ-PAC/bap', 'aarch64-pull-request-2'), +] + + +def run(args: list[str], **kwargs) -> subprocess.CompletedProcess: + log.debug('subprocess: %s', str(args)) + return subprocess.run(args, **kwargs) + + +def curl_raw(url) -> str: + req = urllib.request.Request(url) + token = os.getenv('GITHUB_TOKEN') + if token: + req.add_header('Authorization', 'Bearer ' + token) + log.debug('request: %s authenticated=%s', req.get_full_url(), bool(token)) + with urllib.request.urlopen(req) as f: + return f.read().decode('utf-8') + +def curl(url: str) -> dict: + return json.loads(curl_raw(url)) + +def arg_path_exists(p: str) -> str: + if not os.path.exists(p): + try: open(p) + except FileNotFoundError as e: + raise argparse.ArgumentTypeError(e) + return p + + +def upgrade(p: Package, args: Args): + + if args.mode == 'upgrade': + + run(['nix-update', '-f', args.file, p.attr, '--version', f'branch={p.branch}'] + + args.rest) + for p2 in p.then: + print(f'testing downstream build of {p2}...') + run(['nix-build', args.file, '-A', p2, '--no-out-link']) + + elif args.mode == 'check': + + current = run(['nix-instantiate', '--eval', args.file, '-A', f'{p.attr}.src.rev'], + stdout=subprocess.PIPE).stdout.decode('ascii').strip('"\n') + + total_commits = p.fetch_commits_behind(current) + latest = p.fetch_latest_commit() + permalink = p.compare_permalink(current, latest) + + print() + print('compare link:', p.compare_link(current)) + if total_commits != 0: + print( + f"::warning title=package outdated: {p.attr}::" + f"{p.attr} differs by {total_commits} non-merge commits from {p.branch} ({permalink})" + ) + else: + print( + f"::notice title=package up to date: {p.attr}::" + f"{p.attr} differs by {total_commits} non-merge commits from {p.branch} ({permalink})" + ) + print() + + else: + assert False + + +if __name__ == "__main__": + logging.basicConfig(format='[%(levelname)s:%(name)s@%(filename)s:%(funcName)s:%(lineno)d] %(message)s') + log.setLevel(logging.DEBUG) + + attrs = [x.attr for x in PACKAGES] + p = argparse.ArgumentParser(description=f'updates pac-nix packages. supported packages: {", ".join(attrs)}.') + p.add_argument('mode', choices=['check', 'upgrade', 'do-upgrade'], + help='action to perform. do-upgrade is upgrade but also builds, tests, and commits the changes.') + p.add_argument('--file', '-f', default='default.nix', type=arg_path_exists, + help='use the given file as a package set.') + p.add_argument('--attr', '-A', action='append', choices=attrs, default=[], metavar='PACKAGE', dest='attrs', + help='only act on the given packages.') + p.add_argument('rest', nargs='*', metavar='-- NIX-UPDATE OPTIONS', + help='arguments to forward to nix-update.') + + args = p.parse_intermixed_args() + log.debug('args=%s', str(args)) + + if not args.attrs: + args.attrs = attrs + + log.info('we will %s the following packages: %s', args.mode.upper(), str(args.attrs)) + PACKAGES = [p for p in PACKAGES if p.attr in args.attrs] + + if args.mode == 'do-upgrade': + args.mode = 'upgrade' + args.rest = ['--build', '--test', '--commit'] + args.rest + + for f in fields(Args): + assert f.name in args, f.name + + for p in PACKAGES: + if not p.branch: + p.branch = p.fetch_default_branch() + log.debug('inferred %s branch to be %s', p.repo, repr(p.branch)) + + for p in PACKAGES: + upgrade(p, cast(Args, args)) + diff --git a/update.sh b/update.sh deleted file mode 100755 index f268388..0000000 --- a/update.sh +++ /dev/null @@ -1,107 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash --packages nix-update curl jq cacert git nix vim --pure --keep GITHUB_TOKEN --keep NIX_PATH - -# updates the revision hash for each upstream package. -# for each updated package, this checks the derivation can be built -# then commits its results. - -set -e - -case "$1" in - check) - MODE=check - ;; - upgrade) - MODE=upgrade - ;; - *) - echo "$0: missing required argument" >&2 - echo "Usage: $0 [ check | upgrade ] [ NIX-UPDATE OPTION ]... " >&2 - exit 1 - ;; -esac - -shift -ARGS=("$@") - -do-upgrade() { - [[ $MODE == upgrade ]] -} - -PKGS=. # uses default.nix in current directory -TMP=$(mktemp) - -PID=$$ -curl() { - TMP=$(mktemp) - if ! [[ -z "$GITHUB_TOKEN" ]]; then - command curl --fail-with-body -s --header "Authorization: Bearer $GITHUB_TOKEN" "$@" >$TMP - else - command curl --fail-with-body -s "$@" >$TMP - fi - EXIT=$? - if [[ $EXIT != 0 ]]; then - echo "::error title=curl failure ($EXIT)::curl $@" >&2 - cat $TMP >&2 - kill -ABRT $PID - else - cat $TMP - fi -} - -update-github() { - attr="$1" # pkgs.nix package name to update - repo="$2" # username/repository of upstream repository - branch="$3" # branch (if unset, uses repo's default branch) - - REPO_API="https://api.github.com/repos/$repo" - - if [[ -z "$branch" ]]; then - branch=$(curl "$REPO_API" | jq -r .default_branch) - fi - - COMMITS_API="https://api.github.com/repos/$repo/commits/$branch" - latest=$(curl "$COMMITS_API" | jq -r .sha) - - if do-upgrade; then - nix-update -f "$PKGS" $attr --version branch=$branch --commit "${ARGS[@]}" - else - current=$(nix-instantiate --eval $PKGS -A $attr.src.rev | jq -r) - COMPARE="https://api.github.com/repos/$repo/compare/$current...$branch" - compare=$(curl "$COMPARE") - - result=$(echo "$compare" | jq '{ html_url, status, ahead_by, behind_by, total_commits }') - echo "$compare" | jq '{ html_url, status, ahead_by, behind_by, total_commits }' - - if [[ $(echo "$compare" | jq .total_commits) != 0 ]]; then - echo "::warning title=Package outdated: $attr::$attr differs by $(echo "$compare" | jq .total_commits) commits from $branch ($(echo "$compare" | jq .permalink_url -r))" - else - echo "::notice title=Package current: $attr::$attr differs by $(echo "$compare" | jq .total_commits) commits from $branch ($(echo "$compare" | jq .permalink_url -r))" - fi - fi -} - -test-build() { - pkg="$1" - if do-upgrade; then - nix-build $PKGS -A $pkg --no-out-link - fi -} - -if do-upgrade; then - echo "Performing upgrade..." -else - echo "Checking for upstream updates..." -fi - -update-github asli UQ-PAC/aslp -test-build aslp - -update-github bap-asli-plugin UQ-PAC/bap-asli-plugin -test-build bap-aslp - -update-github basil UQ-PAC/bil-to-boogie-translator - -update-github bap-uq-pac UQ-PAC/bap aarch64-pull-request-2 - -rm -fv ./result