From 4a4f1fdf8b26f71eb2bd7fc7b77d52ceffe1513a Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 08:38:00 -0500 Subject: [PATCH 01/11] Set up pre-commit --- .github/workflows/template/mkworkflows.py | 9 ++++-- .../containers/buildenv-git-annex/Dockerfile | 2 +- .github/workflows/tools/daily-status.py | 3 +- .../workflows/tools/download-latest-artifact | 2 +- .pre-commit-config.yaml | 29 +++++++++++++++++++ clients/testannex.py | 7 ++--- setup.cfg | 16 ++++++++++ 7 files changed, 56 insertions(+), 12 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 setup.cfg diff --git a/.github/workflows/template/mkworkflows.py b/.github/workflows/template/mkworkflows.py index 7a0046de13..2264e98968 100644 --- a/.github/workflows/template/mkworkflows.py +++ b/.github/workflows/template/mkworkflows.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -from pathlib import Path +from pathlib import Path import sys import jinja2 import yaml + def jinja_render(template, context): """ Custom renderer, in which we first replace GitHub Actions' ``${{`` with a @@ -16,11 +17,12 @@ def jinja_render(template, context): ) rendered = jinja2.Template( template.replace("${{", PLACEHOLDER), - trim_blocks = True, - lstrip_blocks = True, + trim_blocks=True, + lstrip_blocks=True, ).render(context) return rendered.replace(PLACEHOLDER, "${{") + def main(): specs_file, template_file, workflows_dir = map(Path, sys.argv[1:]) with specs_file.open() as fp: @@ -31,5 +33,6 @@ def main(): filename = jinja_render(template_file.with_suffix("").name, sp) (workflows_dir / filename).write_text(workflow + "\n") + if __name__ == "__main__": main() diff --git a/.github/workflows/tools/containers/buildenv-git-annex/Dockerfile b/.github/workflows/tools/containers/buildenv-git-annex/Dockerfile index 8452604e83..21ed6e91bc 100644 --- a/.github/workflows/tools/containers/buildenv-git-annex/Dockerfile +++ b/.github/workflows/tools/containers/buildenv-git-annex/Dockerfile @@ -8,7 +8,7 @@ RUN echo 'Acquire::http::Dl-Limit "1000";' >| /etc/apt/apt.conf.d/20snapshots \ && echo 'Acquire::https::Dl-Limit "1000";' >> /etc/apt/apt.conf.d/20snapshots \ && echo 'Acquire::Retries "5";' >> /etc/apt/apt.conf.d/20snapshots -# Notes: +# Notes: # - in APT for NeuroDebian we have #deb-src for debian-devel, so we need to change # that too. # - APT specification switched away from .list to .sources format in bookworm. diff --git a/.github/workflows/tools/daily-status.py b/.github/workflows/tools/daily-status.py index bac16e54b8..b27997ae65 100644 --- a/.github/workflows/tools/daily-status.py +++ b/.github/workflows/tools/daily-status.py @@ -19,9 +19,8 @@ import re import sys from tempfile import TemporaryFile -from zipfile import Path as ZipPath from xml.sax.saxutils import escape - +from zipfile import Path as ZipPath from dateutil.parser import isoparse from github import Auth, Github import requests diff --git a/.github/workflows/tools/download-latest-artifact b/.github/workflows/tools/download-latest-artifact index 724bfcb0bd..7fd80defe0 100755 --- a/.github/workflows/tools/download-latest-artifact +++ b/.github/workflows/tools/download-latest-artifact @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Based on https://raw.githubusercontent.com/RedHatInsights/policies-ui-frontend/master/.github/scripts/download-latest-openapi.sh -# Apache 2.0 license +# Apache 2.0 license set -eu : "${TARGET_REPO:=datalad/git-annex}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..91022d58e1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-json + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + + - repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear + - flake8-builtins + - flake8-unused-arguments diff --git a/clients/testannex.py b/clients/testannex.py index ff473dcc5c..0c081e09d2 100644 --- a/clients/testannex.py +++ b/clients/testannex.py @@ -11,7 +11,6 @@ import sys from tempfile import NamedTemporaryFile, TemporaryDirectory from typing import Any, Dict - import click from click_loglevel import LogLevel from pydantic import BaseModel, TypeAdapter @@ -73,9 +72,7 @@ def parse_clients(path: Path | None = None) -> dict[str, Client]: class GitRepo: path: Path - def run( - self, *args: str | Path, **kwargs: Any - ) -> subprocess.CompletedProcess: + def run(self, *args: str | Path, **kwargs: Any) -> subprocess.CompletedProcess: kwargs.setdefault("cwd", self.path) return runcmd("git", *args, **kwargs) @@ -250,7 +247,7 @@ def main(clientid: str, jobdir: Path, log_level: int) -> None: jobrepo.run("branch", "-D", branch) # It appears that `git push origin result-*` doesn't create a branch # under origin/ when in a "single branch" clone - #jobrepo.run("branch", "-D", "-r", f"origin/{branch}") + # jobrepo.run("branch", "-D", "-r", f"origin/{branch}") jobrepo.run("gc") if failed_jobs: diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..38a8a0b4c0 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,16 @@ +[flake8] +doctests = True +#max-doc-length = 100 +#max-line-length = 80 +unused-arguments-ignore-stub-functions = True +extend-select = B901,B902 +ignore = A003,B005,E203,E262,E266,E501,U101,W503 + +[isort] +atomic = True +force_sort_within_sections = True +honor_noqa = True +lines_between_sections = 0 +profile = black +reverse_relative = True +sort_relative_in_force_sorted_sections = True From 58175a019f5b06e518d1950e4651baa08c26370a Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 08:44:44 -0500 Subject: [PATCH 02/11] Type-check testannex.py --- .gitignore | 1 + clients/.gitignore | 1 - clients/testannex.py | 2 +- noxfile.py | 8 ++++++++ setup.cfg | 16 ++++++++++++++++ 5 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 noxfile.py diff --git a/.gitignore b/.gitignore index f18e91466c..6e42fba6ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea +__pycache__/ venv/ diff --git a/clients/.gitignore b/clients/.gitignore index 91b8175330..63f1fef0e5 100644 --- a/clients/.gitignore +++ b/clients/.gitignore @@ -1,2 +1 @@ *.lock -__pycache__/ diff --git a/clients/testannex.py b/clients/testannex.py index 0c081e09d2..5c3677ed46 100644 --- a/clients/testannex.py +++ b/clients/testannex.py @@ -28,7 +28,7 @@ class Test: shell: str body: str - def run(self, result_dir: Path, **kwargs) -> bool: + def run(self, result_dir: Path, **kwargs: Any) -> bool: log.info("Running test %r", self.name) with NamedTemporaryFile("w+") as script: print(self.body, file=script, flush=True) diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000000..cc65e586d2 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,8 @@ +import nox + + +@nox.session +def typing_testannex(session): + session.install("-r", "clients/requirements.txt") + session.install("mypy") + session.run("mypy", "clients/testannex.py") diff --git a/setup.cfg b/setup.cfg index 38a8a0b4c0..686fdc64d8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,3 +14,19 @@ lines_between_sections = 0 profile = black reverse_relative = True sort_relative_in_force_sorted_sections = True + +[mypy] +allow_incomplete_defs = False +allow_untyped_defs = False +ignore_missing_imports = False +# : +no_implicit_optional = True +implicit_reexport = False +local_partial_types = True +pretty = True +show_error_codes = True +show_traceback = True +strict_equality = True +warn_redundant_casts = True +warn_return_any = True +warn_unreachable = True From 4986cac947a9b2e6d6b9aec224c9e0611e961585 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 09:01:42 -0500 Subject: [PATCH 03/11] Type-check daily-status.py --- .github/workflows/tools/daily-status.py | 6 +++--- noxfile.py | 28 ++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tools/daily-status.py b/.github/workflows/tools/daily-status.py index b27997ae65..535db25306 100644 --- a/.github/workflows/tools/daily-status.py +++ b/.github/workflows/tools/daily-status.py @@ -52,7 +52,7 @@ def from_conclusion(cls, concl: str) -> Outcome: elif concl == "failure": return cls.FAIL elif concl == "timed_out": - return cls.ERRROR + return cls.ERROR elif concl in {"neutral", "action_required", "cancelled", "skipped", "stale"}: return cls.INCOMPLETE else: @@ -88,7 +88,7 @@ class DailyStatus: appveyor_builds: list[AppveyorBuild] def get_subject_body(self) -> tuple[str, str]: - qtys = Counter() + qtys: Counter[Outcome] = Counter() body = "
    \n

  • GitHub:

    \n
      \n" if self.github_runs: for wfstatus in self.github_runs: @@ -295,7 +295,7 @@ def main() -> None: ) ) - client_statuses = [] + client_statuses: list[ClientStatus | ResultProcessError] = [] for run in gh.get_repo(CLIENTS_REPO).get_workflow(CLIENTS_WORKFLOW).get_runs(): if run.status != "completed": continue diff --git a/noxfile.py b/noxfile.py index cc65e586d2..5413782f90 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,8 +1,34 @@ +import os.path import nox @nox.session -def typing_testannex(session): +def typing_testannex(session: nox.Session) -> None: session.install("-r", "clients/requirements.txt") session.install("mypy") session.run("mypy", "clients/testannex.py") + + +@nox.session +def typing_daily_status(session: nox.Session) -> None: + path = ".github/workflows/tools/daily-status.py" + install_requires(session, path) + session.install("mypy", "types-python-dateutil", "types-requests") + session.run("mypy", path) + + +def install_requires(session: nox.Session, path: str) -> None: + tmpdir = session.create_tmp() + reqfile = os.path.join(tmpdir, "requirements.txt") + session.install("pip-run") + with open(reqfile, "w", encoding="utf-8") as fp: + session.run( + "python", + "-m", + "pip_run.read-deps", + "--separator", + "newline", + path, + stdout=fp, + ) + session.install("-r", reqfile) From d74185443e1a484cc7bcc3579cbd471ceaad0132 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 09:12:26 -0500 Subject: [PATCH 04/11] Type-check dispatch-build --- .github/workflows/tools/dispatch-build | 48 +++++++++++++++++++------- noxfile.py | 10 ++++++ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tools/dispatch-build b/.github/workflows/tools/dispatch-build index ef8204a80f..00c8995fc7 100755 --- a/.github/workflows/tools/dispatch-build +++ b/.github/workflows/tools/dispatch-build @@ -1,31 +1,48 @@ #!/usr/bin/env python3 -__requires__ = ["click ~= 8.0", "PyGithub == 2.*"] +from __future__ import annotations +from collections.abc import Sequence import os import subprocess import click from github import Auth, Github +from github.Workflow import Workflow + +__python_requires__ = ">= 3.8" +__requires__ = ["click ~= 8.0", "PyGithub == 2.*"] REPO = "datalad/git-annex" ALL_OS_TYPES = ("macos", "ubuntu", "windows") class BuildDispatcher: - def __init__(self, token): + def __init__(self, token: str) -> None: self.gh = Github(auth=Auth.Token(token)) self.repo = self.gh.get_repo(REPO) - def get_os_workflows(self, ostypes): + def get_os_workflows(self, ostypes: Sequence[str]) -> list[Workflow]: return [self.repo.get_workflow(f"build-{o}.yaml") for o in ostypes] - def build_pr(self, pr, ostypes=ALL_OS_TYPES, workflow_ref="master"): + def build_pr( + self, + pr: int, + ostypes: Sequence[str] = ALL_OS_TYPES, + workflow_ref: str = "master", + ) -> None: for w in self.get_os_workflows(ostypes): w.create_dispatch(ref=workflow_ref, inputs={"pr": str(pr)}) - def build_commitish(self, commitish, ostypes=ALL_OS_TYPES, workflow_ref="master"): + def build_commitish( + self, + commitish: str, + ostypes: Sequence[str] = ALL_OS_TYPES, + workflow_ref: str = "master", + ) -> None: for w in self.get_os_workflows(ostypes): w.create_dispatch(ref=workflow_ref, inputs={"commitish": commitish}) - def build_latest(self, ostypes=ALL_OS_TYPES, workflow_ref="master"): + def build_latest( + self, ostypes: Sequence[str] = ALL_OS_TYPES, workflow_ref: str = "master" + ) -> None: for w in self.get_os_workflows(ostypes): w.create_dispatch(ref=workflow_ref, inputs={}) @@ -48,7 +65,9 @@ class BuildDispatcher: show_default=True, ) @click.argument("ref", required=False) -def main(ostypes, pr, ref, workflow_ref): +def main( + ostypes: tuple[str, ...], pr: bool, ref: str | None, workflow_ref: str +) -> None: """ Trigger builds of datalad/git-annex. @@ -91,16 +110,19 @@ def main(ostypes, pr, ref, workflow_ref): token = os.environ.get("GITHUB_TOKEN") if token is None: token = subprocess.check_output( - ["git", "config", "hub.oauthtoken"], universal_newlines=True + ["git", "config", "hub.oauthtoken"], text=True ).strip() dispatcher = BuildDispatcher(token) if pr: - try: - int(ref) - except ValueError: + if ref is None: raise click.UsageError("--pr requires a PR number") - dispatcher.build_pr(pr, ostypes, workflow_ref=workflow_ref) - elif ref: + else: + try: + pr_num = int(ref) + except ValueError: + raise click.UsageError("--pr requires a PR number") + dispatcher.build_pr(pr_num, ostypes, workflow_ref=workflow_ref) + elif ref is not None: dispatcher.build_commitish(ref, ostypes, workflow_ref=workflow_ref) else: dispatcher.build_latest(ostypes, workflow_ref=workflow_ref) diff --git a/noxfile.py b/noxfile.py index 5413782f90..b45c0c6fe2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -17,6 +17,16 @@ def typing_daily_status(session: nox.Session) -> None: session.run("mypy", path) +@nox.session +def typing_dispatch_build(session: nox.Session) -> None: + path = ".github/workflows/tools/dispatch-build" + install_requires(session, path) + # PyGithub uses python-dateutil and requests, so apparently their typing + # stubs have to be installed in order for mypy to analyze PyGithub + session.install("mypy", "types-python-dateutil", "types-requests") + session.run("mypy", path) + + def install_requires(session: nox.Session, path: str) -> None: tmpdir = session.create_tmp() reqfile = os.path.join(tmpdir, "requirements.txt") From 5c3f4a73f51b32cd31c2e214084677354e52248c Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 09:14:08 -0500 Subject: [PATCH 05/11] Add type-checking workflow --- .github/workflows/typing.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/typing.yml diff --git a/.github/workflows/typing.yml b/.github/workflows/typing.yml new file mode 100644 index 0000000000..31b8470d33 --- /dev/null +++ b/.github/workflows/typing.yml @@ -0,0 +1,27 @@ +name: Type-check + +on: + push: + branches: + - master + pull_request: + +jobs: + typing: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '^3.8' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade nox + + - name: Run type checker + run: nox From de692d2dd7117b80e6298c932275796cef46f199 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 09:19:30 -0500 Subject: [PATCH 06/11] Make the build workflows only check out .github/ and patches/ from master --- .github/workflows/build-macos.yaml | 6 +++--- .github/workflows/build-ubuntu.yaml | 6 +++--- .github/workflows/build-windows.yaml | 6 +++--- .../template/build-{{ostype}}.yaml.j2 | 18 +++++++++--------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-macos.yaml b/.github/workflows/build-macos.yaml index e8f543220d..d948605ba5 100644 --- a/.github/workflows/build-macos.yaml +++ b/.github/workflows/build-macos.yaml @@ -60,10 +60,10 @@ jobs: run: | # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for # git-annex's version detection to use the correct git commit) with - # the contents of master — or whatever ref is being used as the - # workflow source — also available. + # the .github/ and patches/ trees from master — or whatever ref is + # being used as the workflow source — also available. git checkout "$BUILD_COMMIT" - git checkout "$GITHUB_SHA" -- . + git checkout "$GITHUB_SHA" -- .github patches - name: Get build version id: build-version diff --git a/.github/workflows/build-ubuntu.yaml b/.github/workflows/build-ubuntu.yaml index 806f622498..d15113e29b 100644 --- a/.github/workflows/build-ubuntu.yaml +++ b/.github/workflows/build-ubuntu.yaml @@ -79,10 +79,10 @@ jobs: run: | # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for # git-annex's version detection to use the correct git commit) with - # the contents of master — or whatever ref is being used as the - # workflow source — also available. + # the .github/ and patches/ trees from master — or whatever ref is + # being used as the workflow source — also available. git checkout "$BUILD_COMMIT" - git checkout "$GITHUB_SHA" -- . + git checkout "$GITHUB_SHA" -- .github patches - name: Get build version id: build-version diff --git a/.github/workflows/build-windows.yaml b/.github/workflows/build-windows.yaml index 04f7e5ef19..3f1621b474 100644 --- a/.github/workflows/build-windows.yaml +++ b/.github/workflows/build-windows.yaml @@ -66,14 +66,14 @@ jobs: run: | # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for # git-annex's version detection to use the correct git commit) with - # the contents of master — or whatever ref is being used as the - # workflow source — also available. + # the .github/ and patches/ trees from master — or whatever ref is + # being used as the workflow source — also available. git reset --soft "$BUILD_COMMIT" # Avoid checking out unnecessary files with paths that are invalid on # Windows. git ls-tree --name-only HEAD | grep -v '^doc$' | xargs git checkout HEAD git checkout HEAD doc/license ':(glob)doc/*.mdwn' ':(glob)doc/logo*' - git checkout "$GITHUB_SHA" -- . + git checkout "$GITHUB_SHA" -- .github patches - name: Get build version id: build-version diff --git a/.github/workflows/template/build-{{ostype}}.yaml.j2 b/.github/workflows/template/build-{{ostype}}.yaml.j2 index e8b13aaf6a..44fbde5176 100644 --- a/.github/workflows/template/build-{{ostype}}.yaml.j2 +++ b/.github/workflows/template/build-{{ostype}}.yaml.j2 @@ -33,7 +33,7 @@ jobs: build-version: ${{ steps.build-version.outputs.version }} steps: - name: Checkout this repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -96,8 +96,8 @@ jobs: run: | # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for # git-annex's version detection to use the correct git commit) with - # the contents of master — or whatever ref is being used as the - # workflow source — also available. + # the .github/ and patches/ trees from master — or whatever ref is + # being used as the workflow source — also available. {% if ostype == "windows" %} git reset --soft "$BUILD_COMMIT" # Avoid checking out unnecessary files with paths that are invalid on @@ -107,7 +107,7 @@ jobs: {% else %} git checkout "$BUILD_COMMIT" {% endif %} - git checkout "$GITHUB_SHA" -- . + git checkout "$GITHUB_SHA" -- .github patches - name: Get build version id: build-version @@ -277,7 +277,7 @@ jobs: {% if ostype == "ubuntu" %} - name: Clone datalad/git-annex-ci-client-jobs if: contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: datalad/git-annex-ci-client-jobs fetch-depth: 1 @@ -384,7 +384,7 @@ jobs: {% endif %} steps: - name: Checkout this repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create pending PR status if: github.event.inputs.pr != '' @@ -539,7 +539,7 @@ jobs: needs: build-package steps: - name: Checkout this repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create pending PR status if: github.event.inputs.pr != '' @@ -600,7 +600,7 @@ jobs: fail-fast: false steps: - name: Checkout this repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create pending PR status if: github.event.inputs.pr != '' @@ -732,7 +732,7 @@ jobs: # needed for ssh certs under ubuntu and tox.ini everywhere - name: Checkout datalad - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: datalad/datalad path: datalad From beeb6b143bfd2445fdd054652ca91abf28d0af29 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 11:36:27 -0500 Subject: [PATCH 07/11] Use a separate mypy cache for each nox environment --- noxfile.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index b45c0c6fe2..72a2dd4758 100644 --- a/noxfile.py +++ b/noxfile.py @@ -6,7 +6,12 @@ def typing_testannex(session: nox.Session) -> None: session.install("-r", "clients/requirements.txt") session.install("mypy") - session.run("mypy", "clients/testannex.py") + session.run( + "mypy", + "--cache-dir", + str(session.cache_dir / "mypy_cache" / session.name), + "clients/testannex.py", + ) @nox.session @@ -14,7 +19,12 @@ def typing_daily_status(session: nox.Session) -> None: path = ".github/workflows/tools/daily-status.py" install_requires(session, path) session.install("mypy", "types-python-dateutil", "types-requests") - session.run("mypy", path) + session.run( + "mypy", + "--cache-dir", + str(session.cache_dir / "mypy_cache" / session.name), + path, + ) @nox.session @@ -24,7 +34,12 @@ def typing_dispatch_build(session: nox.Session) -> None: # PyGithub uses python-dateutil and requests, so apparently their typing # stubs have to be installed in order for mypy to analyze PyGithub session.install("mypy", "types-python-dateutil", "types-requests") - session.run("mypy", path) + session.run( + "mypy", + "--cache-dir", + str(session.cache_dir / "mypy_cache" / session.name), + path, + ) def install_requires(session: nox.Session, path: str) -> None: From aef63c7aa4faa731a5d1967ad4c990da78457526 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 10:48:32 -0500 Subject: [PATCH 08/11] daily-status.py: Replace PyGithub with ghreq & pydantic --- .github/workflows/tools/daily-status.py | 134 +++++++++++++----------- 1 file changed, 72 insertions(+), 62 deletions(-) diff --git a/.github/workflows/tools/daily-status.py b/.github/workflows/tools/daily-status.py index 535db25306..aa5d12d767 100644 --- a/.github/workflows/tools/daily-status.py +++ b/.github/workflows/tools/daily-status.py @@ -3,8 +3,10 @@ __python_requires__ = ">= 3.8" __requires__ = [ + "ghreq ~= 0.2", + "ghtoken ~= 0.1", + "pydantic ~= 2.0", "python-dateutil ~= 2.7", - "PyGithub ~= 2.0", "requests ~= 2.20", "ruamel.yaml ~= 0.15", ] @@ -14,7 +16,6 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from enum import Enum -import os from pathlib import Path import re import sys @@ -22,7 +23,9 @@ from xml.sax.saxutils import escape from zipfile import Path as ZipPath from dateutil.parser import isoparse -from github import Auth, Github +from ghreq import Client +from ghtoken import get_ghtoken +from pydantic import BaseModel import requests from ruamel.yaml import YAML @@ -46,7 +49,7 @@ class Outcome(Enum): INCOMPLETE = "INCOMPLETE" @classmethod - def from_conclusion(cls, concl: str) -> Outcome: + def from_conclusion(cls, concl: str | None) -> Outcome: if concl == "success": return cls.PASS elif concl == "failure": @@ -146,6 +149,40 @@ def get_subject_body(self) -> tuple[str, str]: return (subject, body) +class WorkflowRun(BaseModel): + name: str | None = None + head_branch: str | None = None + run_number: int + event: str + status: str | None = None + conclusion: str | None = None + created_at: datetime + artifacts_url: str + jobs_url: str + html_url: str + check_suite_id: int + + +class WorkflowJob(BaseModel): + name: str + html_url: str + started_at: datetime + conclusion: str + + @property + def outcome(self) -> Outcome: + return Outcome.from_conclusion(self.conclusion) + + def as_html(self) -> str: + return f'{self.outcome.as_html()} {escape(self.name)} {self.started_at}' + + +class Artifact(BaseModel): + id: int + created_at: datetime + archive_download_url: str + + @dataclass class WorkflowStatus: file: str @@ -154,7 +191,7 @@ class WorkflowStatus: url: str timestamp: datetime outcome: Outcome - jobs: list[JobStatus] + jobs: list[WorkflowJob] def get_summary(self) -> Counter[Outcome]: return Counter(j.outcome for j in self.jobs) @@ -166,17 +203,6 @@ def as_html(self) -> str: return s + "
    \n" -@dataclass -class JobStatus: - name: str - url: str - timestamp: datetime - outcome: Outcome - - def as_html(self) -> str: - return f'{self.outcome.as_html()} {escape(self.name)} {self.timestamp}' - - @dataclass class ClientStatus: client_id: str @@ -248,78 +274,71 @@ def as_html(self) -> str: def main() -> None: outfile = sys.argv[1] - token = os.environ["GITHUB_TOKEN"] - gh = Github(auth=Auth.Token(token)) + token = get_ghtoken() cutoff = datetime.now(timezone.utc) - WINDOW with CLIENT_INFO_FILE.open() as fp: client_info = YAML(typ="safe").load(fp) all_clients = set(client_info.keys()) - with requests.Session() as s: - s.headers["Authorization"] = f"bearer {token}" - + with Client(token=token) as client: github_statuses = [] - wfrepo = gh.get_repo(WORKFLOW_REPO) + wfrepo = client / "repos" / WORKFLOW_REPO for wffilename in WORKFLOWS: - wf = wfrepo.get_workflow(wffilename) - for run in wf.get_runs(): + wfep = wfrepo / "actions" / "workflows" / wffilename + wf = wfep.get() + for data in (wfep / "runs").paginate(): + run = WorkflowRun.model_validate(data) if run.status != "completed" or run.event not in ( "schedule", "workflow_dispatch", ): continue - dt = ensure_aware(run.created_at) - if dt <= cutoff: + if run.created_at <= cutoff: break - r = s.get(run.jobs_url) - r.raise_for_status() - job_statuses = [ - JobStatus( - name=j["name"], - url=j["html_url"], - timestamp=isoparse(j["started_at"]), - outcome=Outcome.from_conclusion(j["conclusion"]), - ) - for j in r.json()["jobs"] + jobs = [ + WorkflowJob.model_validate(jdata) + for jdata in client.paginate(run.jobs_url) ] github_statuses.append( WorkflowStatus( file=wffilename, - name=wf.name, + name=wf["name"], build_id=run.run_number, url=run.html_url, - timestamp=dt, + timestamp=run.created_at, outcome=Outcome.from_conclusion(run.conclusion), - jobs=job_statuses, + jobs=jobs, ) ) client_statuses: list[ClientStatus | ResultProcessError] = [] - for run in gh.get_repo(CLIENTS_REPO).get_workflow(CLIENTS_WORKFLOW).get_runs(): + for data in client.paginate( + f"/repos/{CLIENTS_REPO}/actions/workflows/{CLIENTS_WORKFLOW}/runs" + ): + run = WorkflowRun.model_validate(data) if run.status != "completed": continue - dt = ensure_aware(run.created_at) - if dt <= cutoff: + if run.created_at <= cutoff: break + assert run.head_branch is not None m = re.fullmatch(r"result-(.+)-(\d+)", run.head_branch) assert m if Outcome.from_conclusion(run.conclusion) is Outcome.PASS: - r = s.get(run.artifacts_url) - r.raise_for_status() - (artifact,) = r.json()["artifacts"] + (artdata,) = client.paginate(run.artifacts_url) + artifact = Artifact.model_validate(artdata) client_statuses.append( ClientStatus( client_id=m[1], build_id=int(m[2]), - timestamp=dt, + timestamp=run.created_at, artifact_url=( f"https://github.com/{CLIENTS_REPO}/suites" - f"/{run.raw_data['check_suite_id']}/artifacts" - f"/{artifact['id']}" + f"/{run.check_suite_id}/artifacts" + f"/{artifact.id}" ), tests=get_client_test_outcomes( - s, artifact["archive_download_url"] + client, artifact.archive_download_url ), ) ) @@ -328,7 +347,7 @@ def main() -> None: ResultProcessError( client_id=m[1], build_id=int(m[2]), - timestamp=dt, + timestamp=run.created_at, url=run.html_url, ) ) @@ -377,22 +396,13 @@ def main() -> None: print(body, file=fp) -def ensure_aware(dt: datetime) -> datetime: - # Pygithub returns naïve datetimes for timestamps with a "Z" suffix. Until - # that's fixed , we need to - # make such datetimes timezone-aware manually. - return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt - - -def get_client_test_outcomes( - s: requests.Session, artifact_url: str -) -> dict[str, Outcome]: +def get_client_test_outcomes(client: Client, artifact_url: str) -> dict[str, Outcome]: tests = {} with TemporaryFile() as fp: - with s.get(artifact_url, stream=True) as r: - r.raise_for_status() + with client.get(artifact_url, stream=True) as r: for chunk in r.iter_content(chunk_size=8192): fp.write(chunk) + fp.flush() fp.seek(0) for p in ZipPath(fp).iterdir(): if p.name.endswith(".rc"): From 6f1f61ba84f6386cd3702e75d6268a3465261e04 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 11:11:32 -0500 Subject: [PATCH 09/11] daily-status.py: Use pydantic to parse Appveyor API responses --- .github/workflows/tools/daily-status.py | 100 ++++++++++++------------ noxfile.py | 2 +- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/.github/workflows/tools/daily-status.py b/.github/workflows/tools/daily-status.py index aa5d12d767..226b02a93b 100644 --- a/.github/workflows/tools/daily-status.py +++ b/.github/workflows/tools/daily-status.py @@ -6,7 +6,6 @@ "ghreq ~= 0.2", "ghtoken ~= 0.1", "pydantic ~= 2.0", - "python-dateutil ~= 2.7", "requests ~= 2.20", "ruamel.yaml ~= 0.15", ] @@ -20,12 +19,12 @@ import re import sys from tempfile import TemporaryFile +from typing import List from xml.sax.saxutils import escape from zipfile import Path as ZipPath -from dateutil.parser import isoparse from ghreq import Client from ghtoken import get_ghtoken -from pydantic import BaseModel +from pydantic import BaseModel, Field import requests from ruamel.yaml import YAML @@ -235,41 +234,56 @@ def as_html(self) -> str: return f'{Outcome.ERROR.as_html()} processing results for {escape(self.client_id)} #{self.build_id} [logs] {self.timestamp}' -@dataclass -class AppveyorBuild: - id: int +class AppveyorJob(BaseModel): + jobId: str + name: str + status: str + + @property + def outcome(self) -> Outcome: + return Outcome.from_appveyor_status(self.status) + + def url(self, build_id: int) -> str: + return f"https://ci.appveyor.com/project/{APPVEYOR_PROJECT}/builds/{build_id}/job/{self.jobId}" + + def as_html(self, build_id: int) -> str: + return f'{self.outcome.as_html()} {escape(self.name)}' + + +class AppveyorBuild(BaseModel): + buildId: int + finished: datetime | None + started: datetime version: str - timestamp: datetime - outcome: Outcome - jobs: list[AppveyorJob] + status: str + jobs: List[AppveyorJob] + + @property + def outcome(self) -> Outcome: + return Outcome.from_appveyor_status(self.status) @property def url(self) -> str: - return f"https://ci.appveyor.com/project/{APPVEYOR_PROJECT}/builds/{self.id}" + return ( + f"https://ci.appveyor.com/project/{APPVEYOR_PROJECT}/builds/{self.buildId}" + ) def get_summary(self) -> Counter[Outcome]: return Counter(j.outcome for j in self.jobs) def as_html(self) -> str: - s = f'

    {self.outcome.as_html()} {self.version} {self.timestamp}

    \n
      \n' + s = f'

      {self.outcome.as_html()} {self.version} {self.started}

      \n
        \n' for j in self.jobs: - s += "
      • " + j.as_html() + "
        • \n" + s += "
      • " + j.as_html(self.buildId) + "
        • \n" return s + "
      \n" -@dataclass -class AppveyorJob: - build_id: int - id: str - name: str - outcome: Outcome +class AppveyorProject(BaseModel): + build: AppveyorBuild - @property - def url(self) -> str: - return f"https://ci.appveyor.com/project/{APPVEYOR_PROJECT}/builds/{self.build_id}/job/{self.id}" - def as_html(self) -> str: - return f'{self.outcome.as_html()} {escape(self.name)}' +class AppveyorHistory(BaseModel): + builds: List[AppveyorBuild] = Field(default_factory=list) def main() -> None: @@ -355,34 +369,20 @@ def main() -> None: appveyor_builds = [] with requests.Session() as s: for build in get_appveyor_builds(s): - if build.get("finished") is None: + if build.finished is None: continue - finished = isoparse(build["finished"]) - if finished <= cutoff: + if build.finished <= cutoff: break + # Appveyor's build history endpoint omits job information, so we + # need to refetch each build via its individual endpoint to get the + # jobs. r = s.get( f"https://ci.appveyor.com/api/projects/{APPVEYOR_PROJECT}" - f"/build/{build['version']}" + f"/build/{build.version}" ) r.raise_for_status() - data = r.json() - appveyor_builds.append( - AppveyorBuild( - id=build["buildId"], - version=build["version"], - outcome=Outcome.from_appveyor_status(build["status"]), - timestamp=isoparse(build["started"]), - jobs=[ - AppveyorJob( - build_id=build["buildId"], - id=job["jobId"], - name=job["name"], - outcome=Outcome.from_appveyor_status(job["status"]), - ) - for job in data["build"]["jobs"] - ], - ) - ) + project = AppveyorProject.model_validate(r.json()) + appveyor_builds.append(project.build) status = DailyStatus( github_runs=github_statuses, @@ -412,7 +412,7 @@ def get_client_test_outcomes(client: Client, artifact_url: str) -> dict[str, Out return tests -def get_appveyor_builds(s: requests.Session) -> Iterator[dict]: +def get_appveyor_builds(s: requests.Session) -> Iterator[AppveyorBuild]: params = {"recordsNumber": 20} while True: r = s.get( @@ -420,10 +420,10 @@ def get_appveyor_builds(s: requests.Session) -> Iterator[dict]: params=params, ) r.raise_for_status() - data = r.json() - if builds := data.get("builds"): - yield from builds - params["startBuildId"] = builds[-1]["buildId"] + history = AppveyorHistory.model_validate(r.json()) + if history.builds: + yield from history.builds + params["startBuildId"] = history.builds[-1].buildId else: break diff --git a/noxfile.py b/noxfile.py index 72a2dd4758..b8dd63bc51 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,7 +18,7 @@ def typing_testannex(session: nox.Session) -> None: def typing_daily_status(session: nox.Session) -> None: path = ".github/workflows/tools/daily-status.py" install_requires(session, path) - session.install("mypy", "types-python-dateutil", "types-requests") + session.install("mypy", "types-requests") session.run( "mypy", "--cache-dir", From a7bbadf7c5956109e59e7bc0d7aa8206fdb08d01 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 11:27:32 -0500 Subject: [PATCH 10/11] dispatch-build: Replace PyGithub with ghreq --- .github/workflows/tools/dispatch-build | 74 ++++++++++++-------------- noxfile.py | 4 +- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/.github/workflows/tools/dispatch-build b/.github/workflows/tools/dispatch-build index 00c8995fc7..18d0946070 100755 --- a/.github/workflows/tools/dispatch-build +++ b/.github/workflows/tools/dispatch-build @@ -1,26 +1,20 @@ #!/usr/bin/env python3 from __future__ import annotations from collections.abc import Sequence -import os -import subprocess import click -from github import Auth, Github -from github.Workflow import Workflow +from ghreq import Client +from ghtoken import get_ghtoken __python_requires__ = ">= 3.8" -__requires__ = ["click ~= 8.0", "PyGithub == 2.*"] +__requires__ = ["click ~= 8.0", "ghreq ~= 0.2", "ghtoken ~= 0.1"] REPO = "datalad/git-annex" ALL_OS_TYPES = ("macos", "ubuntu", "windows") -class BuildDispatcher: - def __init__(self, token: str) -> None: - self.gh = Github(auth=Auth.Token(token)) - self.repo = self.gh.get_repo(REPO) - - def get_os_workflows(self, ostypes: Sequence[str]) -> list[Workflow]: - return [self.repo.get_workflow(f"build-{o}.yaml") for o in ostypes] +class BuildDispatcher(Client): + def dispatch(self, workflow: str, payload: dict) -> None: + self.post(f"/repos/{REPO}/actions/workflows/{workflow}/dispatches", payload) def build_pr( self, @@ -28,8 +22,10 @@ class BuildDispatcher: ostypes: Sequence[str] = ALL_OS_TYPES, workflow_ref: str = "master", ) -> None: - for w in self.get_os_workflows(ostypes): - w.create_dispatch(ref=workflow_ref, inputs={"pr": str(pr)}) + for o in ostypes: + self.dispatch( + f"build-{o}.yaml", {"ref": workflow_ref, "inputs": {"pr": str(pr)}} + ) def build_commitish( self, @@ -37,14 +33,17 @@ class BuildDispatcher: ostypes: Sequence[str] = ALL_OS_TYPES, workflow_ref: str = "master", ) -> None: - for w in self.get_os_workflows(ostypes): - w.create_dispatch(ref=workflow_ref, inputs={"commitish": commitish}) + for o in ostypes: + self.dispatch( + f"build-{o}.yaml", + {"ref": workflow_ref, "inputs": {"commitish": commitish}}, + ) def build_latest( self, ostypes: Sequence[str] = ALL_OS_TYPES, workflow_ref: str = "master" ) -> None: - for w in self.get_os_workflows(ostypes): - w.create_dispatch(ref=workflow_ref, inputs={}) + for o in ostypes: + self.dispatch(f"build-{o}.yaml", {"ref": workflow_ref, "inputs": {}}) @click.command() @@ -103,29 +102,26 @@ def main( # Build a commit hash: python3 dispatch-build.py [] 65131af - This script requires a GitHub OAuth token in order to run. The token can - be specified via either the `GITHUB_TOKEN` environment variable or, if that - is not set, via the Git `hub.oauthtoken` config option. + This script requires a GitHub access token with appropriate permissions in + order to run. Specify the token via the `GH_TOKEN` or `GITHUB_TOKEN` + environment variable (possibly in an `.env` file), by storing a token with + the `gh` or `hub` command, or by setting the `hub.oauthtoken` Git config + option in your `~/.gitconfig` file. """ - token = os.environ.get("GITHUB_TOKEN") - if token is None: - token = subprocess.check_output( - ["git", "config", "hub.oauthtoken"], text=True - ).strip() - dispatcher = BuildDispatcher(token) - if pr: - if ref is None: - raise click.UsageError("--pr requires a PR number") - else: - try: - pr_num = int(ref) - except ValueError: + with BuildDispatcher(token=get_ghtoken()) as client: + if pr: + if ref is None: raise click.UsageError("--pr requires a PR number") - dispatcher.build_pr(pr_num, ostypes, workflow_ref=workflow_ref) - elif ref is not None: - dispatcher.build_commitish(ref, ostypes, workflow_ref=workflow_ref) - else: - dispatcher.build_latest(ostypes, workflow_ref=workflow_ref) + else: + try: + pr_num = int(ref) + except ValueError: + raise click.UsageError("--pr requires a PR number") + client.build_pr(pr_num, ostypes, workflow_ref=workflow_ref) + elif ref is not None: + client.build_commitish(ref, ostypes, workflow_ref=workflow_ref) + else: + client.build_latest(ostypes, workflow_ref=workflow_ref) if __name__ == "__main__": diff --git a/noxfile.py b/noxfile.py index b8dd63bc51..65bfe9f8e9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -31,9 +31,7 @@ def typing_daily_status(session: nox.Session) -> None: def typing_dispatch_build(session: nox.Session) -> None: path = ".github/workflows/tools/dispatch-build" install_requires(session, path) - # PyGithub uses python-dateutil and requests, so apparently their typing - # stubs have to be installed in order for mypy to analyze PyGithub - session.install("mypy", "types-python-dateutil", "types-requests") + session.install("mypy") session.run( "mypy", "--cache-dir", From 7e28bc4e7ddb2df794a10c593ff6a78f315f5fed Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 14 Nov 2023 11:47:53 -0500 Subject: [PATCH 11/11] daily-status.py and dispatch-build: Set User-Agent header --- .github/workflows/tools/daily-status.py | 8 ++++++-- .github/workflows/tools/dispatch-build | 7 +++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tools/daily-status.py b/.github/workflows/tools/daily-status.py index 226b02a93b..f7cfa5fef3 100644 --- a/.github/workflows/tools/daily-status.py +++ b/.github/workflows/tools/daily-status.py @@ -22,7 +22,7 @@ from typing import List from xml.sax.saxutils import escape from zipfile import Path as ZipPath -from ghreq import Client +from ghreq import Client, make_user_agent from ghtoken import get_ghtoken from pydantic import BaseModel, Field import requests @@ -289,13 +289,16 @@ class AppveyorHistory(BaseModel): def main() -> None: outfile = sys.argv[1] token = get_ghtoken() + user_agent = make_user_agent( + "daily-status.py", url="https://github.com/datalad/git-annex" + ) cutoff = datetime.now(timezone.utc) - WINDOW with CLIENT_INFO_FILE.open() as fp: client_info = YAML(typ="safe").load(fp) all_clients = set(client_info.keys()) - with Client(token=token) as client: + with Client(token=token, user_agent=user_agent) as client: github_statuses = [] wfrepo = client / "repos" / WORKFLOW_REPO for wffilename in WORKFLOWS: @@ -368,6 +371,7 @@ def main() -> None: appveyor_builds = [] with requests.Session() as s: + s.headers["User-Agent"] = user_agent for build in get_appveyor_builds(s): if build.finished is None: continue diff --git a/.github/workflows/tools/dispatch-build b/.github/workflows/tools/dispatch-build index 18d0946070..eaf63b77e0 100755 --- a/.github/workflows/tools/dispatch-build +++ b/.github/workflows/tools/dispatch-build @@ -2,7 +2,7 @@ from __future__ import annotations from collections.abc import Sequence import click -from ghreq import Client +from ghreq import Client, make_user_agent from ghtoken import get_ghtoken __python_requires__ = ">= 3.8" @@ -108,7 +108,10 @@ def main( the `gh` or `hub` command, or by setting the `hub.oauthtoken` Git config option in your `~/.gitconfig` file. """ - with BuildDispatcher(token=get_ghtoken()) as client: + user_agent = make_user_agent( + "dispatch-build", url="https://github.com/datalad/git-annex" + ) + with BuildDispatcher(token=get_ghtoken(), user_agent=user_agent) as client: if pr: if ref is None: raise click.UsageError("--pr requires a PR number")