diff --git a/.github/workflows/build-macos.yaml b/.github/workflows/build-macos.yaml index e8f543220d..d948605ba5 100644 --- a/.github/workflows/build-macos.yaml +++ b/.github/workflows/build-macos.yaml @@ -60,10 +60,10 @@ jobs: run: | # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for # git-annex's version detection to use the correct git commit) with - # the contents of master — or whatever ref is being used as the - # workflow source — also available. + # the .github/ and patches/ trees from master — or whatever ref is + # being used as the workflow source — also available. git checkout "$BUILD_COMMIT" - git checkout "$GITHUB_SHA" -- . + git checkout "$GITHUB_SHA" -- .github patches - name: Get build version id: build-version diff --git a/.github/workflows/build-ubuntu.yaml b/.github/workflows/build-ubuntu.yaml index 806f622498..d15113e29b 100644 --- a/.github/workflows/build-ubuntu.yaml +++ b/.github/workflows/build-ubuntu.yaml @@ -79,10 +79,10 @@ jobs: run: | # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for # git-annex's version detection to use the correct git commit) with - # the contents of master — or whatever ref is being used as the - # workflow source — also available. + # the .github/ and patches/ trees from master — or whatever ref is + # being used as the workflow source — also available. git checkout "$BUILD_COMMIT" - git checkout "$GITHUB_SHA" -- . + git checkout "$GITHUB_SHA" -- .github patches - name: Get build version id: build-version diff --git a/.github/workflows/build-windows.yaml b/.github/workflows/build-windows.yaml index 04f7e5ef19..3f1621b474 100644 --- a/.github/workflows/build-windows.yaml +++ b/.github/workflows/build-windows.yaml @@ -66,14 +66,14 @@ jobs: run: | # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for # git-annex's version detection to use the correct git commit) with - # the contents of master — or whatever ref is being used as the - # workflow source — also available. + # the .github/ and patches/ trees from master — or whatever ref is + # being used as the workflow source — also available. git reset --soft "$BUILD_COMMIT" # Avoid checking out unnecessary files with paths that are invalid on # Windows. git ls-tree --name-only HEAD | grep -v '^doc$' | xargs git checkout HEAD git checkout HEAD doc/license ':(glob)doc/*.mdwn' ':(glob)doc/logo*' - git checkout "$GITHUB_SHA" -- . + git checkout "$GITHUB_SHA" -- .github patches - name: Get build version id: build-version diff --git a/.github/workflows/template/build-{{ostype}}.yaml.j2 b/.github/workflows/template/build-{{ostype}}.yaml.j2 index e8b13aaf6a..44fbde5176 100644 --- a/.github/workflows/template/build-{{ostype}}.yaml.j2 +++ b/.github/workflows/template/build-{{ostype}}.yaml.j2 @@ -33,7 +33,7 @@ jobs: build-version: ${{ steps.build-version.outputs.version }} steps: - name: Checkout this repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -96,8 +96,8 @@ jobs: run: | # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for # git-annex's version detection to use the correct git commit) with - # the contents of master — or whatever ref is being used as the - # workflow source — also available. + # the .github/ and patches/ trees from master — or whatever ref is + # being used as the workflow source — also available. {% if ostype == "windows" %} git reset --soft "$BUILD_COMMIT" # Avoid checking out unnecessary files with paths that are invalid on @@ -107,7 +107,7 @@ jobs: {% else %} git checkout "$BUILD_COMMIT" {% endif %} - git checkout "$GITHUB_SHA" -- . + git checkout "$GITHUB_SHA" -- .github patches - name: Get build version id: build-version @@ -277,7 +277,7 @@ jobs: {% if ostype == "ubuntu" %} - name: Clone datalad/git-annex-ci-client-jobs if: contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: datalad/git-annex-ci-client-jobs fetch-depth: 1 @@ -384,7 +384,7 @@ jobs: {% endif %} steps: - name: Checkout this repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create pending PR status if: github.event.inputs.pr != '' @@ -539,7 +539,7 @@ jobs: needs: build-package steps: - name: Checkout this repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create pending PR status if: github.event.inputs.pr != '' @@ -600,7 +600,7 @@ jobs: fail-fast: false steps: - name: Checkout this repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create pending PR status if: github.event.inputs.pr != '' @@ -732,7 +732,7 @@ jobs: # needed for ssh certs under ubuntu and tox.ini everywhere - name: Checkout datalad - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: datalad/datalad path: datalad diff --git a/.github/workflows/template/mkworkflows.py b/.github/workflows/template/mkworkflows.py index 7a0046de13..2264e98968 100644 --- a/.github/workflows/template/mkworkflows.py +++ b/.github/workflows/template/mkworkflows.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -from pathlib import Path +from pathlib import Path import sys import jinja2 import yaml + def jinja_render(template, context): """ Custom renderer, in which we first replace GitHub Actions' ``${{`` with a @@ -16,11 +17,12 @@ def jinja_render(template, context): ) rendered = jinja2.Template( template.replace("${{", PLACEHOLDER), - trim_blocks = True, - lstrip_blocks = True, + trim_blocks=True, + lstrip_blocks=True, ).render(context) return rendered.replace(PLACEHOLDER, "${{") + def main(): specs_file, template_file, workflows_dir = map(Path, sys.argv[1:]) with specs_file.open() as fp: @@ -31,5 +33,6 @@ def main(): filename = jinja_render(template_file.with_suffix("").name, sp) (workflows_dir / filename).write_text(workflow + "\n") + if __name__ == "__main__": main() diff --git a/.github/workflows/tools/containers/buildenv-git-annex/Dockerfile b/.github/workflows/tools/containers/buildenv-git-annex/Dockerfile index 8452604e83..21ed6e91bc 100644 --- a/.github/workflows/tools/containers/buildenv-git-annex/Dockerfile +++ b/.github/workflows/tools/containers/buildenv-git-annex/Dockerfile @@ -8,7 +8,7 @@ RUN echo 'Acquire::http::Dl-Limit "1000";' >| /etc/apt/apt.conf.d/20snapshots \ && echo 'Acquire::https::Dl-Limit "1000";' >> /etc/apt/apt.conf.d/20snapshots \ && echo 'Acquire::Retries "5";' >> /etc/apt/apt.conf.d/20snapshots -# Notes: +# Notes: # - in APT for NeuroDebian we have #deb-src for debian-devel, so we need to change # that too. # - APT specification switched away from .list to .sources format in bookworm. diff --git a/.github/workflows/tools/daily-status.py b/.github/workflows/tools/daily-status.py index bac16e54b8..f7cfa5fef3 100644 --- a/.github/workflows/tools/daily-status.py +++ b/.github/workflows/tools/daily-status.py @@ -3,8 +3,9 @@ __python_requires__ = ">= 3.8" __requires__ = [ - "python-dateutil ~= 2.7", - "PyGithub ~= 2.0", + "ghreq ~= 0.2", + "ghtoken ~= 0.1", + "pydantic ~= 2.0", "requests ~= 2.20", "ruamel.yaml ~= 0.15", ] @@ -14,16 +15,16 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from enum import Enum -import os from pathlib import Path import re import sys from tempfile import TemporaryFile -from zipfile import Path as ZipPath +from typing import List from xml.sax.saxutils import escape - -from dateutil.parser import isoparse -from github import Auth, Github +from zipfile import Path as ZipPath +from ghreq import Client, make_user_agent +from ghtoken import get_ghtoken +from pydantic import BaseModel, Field import requests from ruamel.yaml import YAML @@ -47,13 +48,13 @@ class Outcome(Enum): INCOMPLETE = "INCOMPLETE" @classmethod - def from_conclusion(cls, concl: str) -> Outcome: + def from_conclusion(cls, concl: str | None) -> Outcome: if concl == "success": return cls.PASS elif concl == "failure": return cls.FAIL elif concl == "timed_out": - return cls.ERRROR + return cls.ERROR elif concl in {"neutral", "action_required", "cancelled", "skipped", "stale"}: return cls.INCOMPLETE else: @@ -89,7 +90,7 @@ class DailyStatus: appveyor_builds: list[AppveyorBuild] def get_subject_body(self) -> tuple[str, str]: - qtys = Counter() + qtys: Counter[Outcome] = Counter() body = "
GitHub:
\n{self.outcome.as_html()} {self.version} {self.timestamp}
\n{self.outcome.as_html()} {self.version} {self.started}
\n