diff --git a/.github/workflows/antsibull-build.yml b/.github/workflows/antsibull-build.yml index d3addec7..32364217 100644 --- a/.github/workflows/antsibull-build.yml +++ b/.github/workflows/antsibull-build.yml @@ -31,24 +31,28 @@ jobs: antsibull_changelog_ref: 0.24.0 antsibull_core_ref: stable-2 antsibull_docutils_ref: 1.0.0 # isn't used by antsibull-changelog 0.24.0 + antsibull_fileutils_ref: main - name: Ansible 9 options: '-e antsibull_ansible_version=9.99.0' python: '3.11' antsibull_changelog_ref: main antsibull_core_ref: stable-2 antsibull_docutils_ref: main + antsibull_fileutils_ref: main - name: Ansible 10 options: '-e antsibull_ansible_version=10.99.0' python: '3.12' antsibull_changelog_ref: main antsibull_core_ref: main antsibull_docutils_ref: main + antsibull_fileutils_ref: main - name: Ansible 11 options: '-e antsibull_ansible_version=11.99.0' python: '3.12' antsibull_changelog_ref: main antsibull_core_ref: main antsibull_docutils_ref: main + antsibull_fileutils_ref: main steps: - name: Check out antsibull @@ -77,6 +81,13 @@ jobs: path: antsibull-docutils ref: ${{ matrix.antsibull_docutils_ref }} + - name: Check out dependent project antsibull-fileutils + uses: actions/checkout@v4 + with: + repository: ansible-community/antsibull-fileutils + path: antsibull-fileutils + ref: ${{ matrix.antsibull_fileutils_ref }} + - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: diff --git a/.github/workflows/nox.yml b/.github/workflows/nox.yml index 5c4632ff..c0d22069 100644 --- a/.github/workflows/nox.yml +++ b/.github/workflows/nox.yml @@ -74,6 +74,11 @@ jobs: with: repository: ansible-community/antsibull-docutils path: antsibull-docutils + - name: Check out dependent project antsibull-fileutils + uses: actions/checkout@v4 + with: + repository: ansible-community/antsibull-fileutils + path: antsibull-fileutils - name: Install extra packages if: "matrix.packages != ''" run: | diff --git a/README.md b/README.md index 1ad8d0ea..16221ffc 100644 --- a/README.md +++ b/README.md @@ -48,19 +48,20 @@ and install the requirements needed to run the tests there. --- -antsibull depends on the sister antsibull-core, antsibull-changelog, and -antsibull-docutils projects. +antsibull depends on the sister antsibull-core, antsibull-changelog, +antsibull-docutils, and antsibull-fileutils projects. By default, `nox` will install development versions of these projects from Github. -If you're hacking on antsibull-core or antsibull-changelog alongside antsibull, +If you're hacking on antsibull-core, antsibull-changelog, antsibull-docutils and/or +antsibull-fileutils alongside antsibull, nox will automatically install the projects from `../antsibull-core`, -`../antsibull-changelog`, and `../antsibull-docutils` when running tests -if those paths exist. +`../antsibull-changelog`, `../antsibull-docutils`, and `../antsibull-fileutils` +when running tests if those paths exist. You can change this behavior through the `OTHER_ANTSIBULL_MODE` env var: - `OTHER_ANTSIBULL_MODE=auto` — the default behavior described above - `OTHER_ANTSIBULL_MODE=local` — install the projects from `../antsibull-core`, - `../antsibull-changelog`, and `../antsibull-docutils`. + `../antsibull-changelog`, `../antsibull-docutils`, and `../antsibull-fileutils`. Fail if those paths don't exist. - `OTHER_ANTSIBULL_MODE=git` — install the projects from the Github main branch - `OTHER_ANTSIBULL_MODE=pypi` — install the latest version from PyPI diff --git a/changelogs/fragments/619-antsibull-fileutils.yml b/changelogs/fragments/619-antsibull-fileutils.yml new file mode 100644 index 00000000..19f76a3c --- /dev/null +++ b/changelogs/fragments/619-antsibull-fileutils.yml @@ -0,0 +1,6 @@ +minor_changes: + - "Add dependency on antsibull-fileutils. Some functionality from antsibull-core is moving there, so we can use it from there directly + (https://github.com/ansible-community/antsibull/pull/619)." +bugfixes: + - "Add explicit dependency on PyYAML, since we directly use it + (https://github.com/ansible-community/antsibull/pull/619)." diff --git a/noxfile.py b/noxfile.py index af77fbf4..0c3d7f21 100644 --- a/noxfile.py +++ b/noxfile.py @@ -43,7 +43,12 @@ def other_antsibull( if mode is None: mode = DEFAULT_MODE to_install: list[str | Path] = [] - args = ("antsibull-core", "antsibull-changelog", "antsibull-docutils") + args = ( + "antsibull-core", + "antsibull-changelog", + "antsibull-docutils", + "antsibull-fileutils", + ) for project in args: path = Path("../", project) path_exists = path.is_dir() diff --git a/pyproject.toml b/pyproject.toml index fc0404ec..30a84e18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ requires-python = ">=3.9" dependencies = [ "antsibull-changelog >= 0.24.0", "antsibull-core >= 2.0.0, < 4.0.0", + "antsibull-fileutils >= 1.0.0, < 2.0.0", "asyncio-pool", "build", "jinja2", @@ -36,6 +37,7 @@ dependencies = [ "aiofiles", "aiohttp >= 3.0.0", "twiggy", + "pyyaml", # We rely on deprecated features to maintain compat btw. pydantic v1 and v2 "pydantic < 3", # pydantic already pulls it in, but we use it for TypedDict diff --git a/src/antsibull/build_ansible_commands.py b/src/antsibull/build_ansible_commands.py index 147fa523..46ce69cd 100644 --- a/src/antsibull/build_ansible_commands.py +++ b/src/antsibull/build_ansible_commands.py @@ -25,7 +25,7 @@ from antsibull_core.galaxy import CollectionDownloader, GalaxyContext from antsibull_core.logging import log from antsibull_core.subprocess_util import async_log_run, log_run -from antsibull_core.yaml import store_yaml_file, store_yaml_stream +from antsibull_fileutils.yaml import store_yaml_file, store_yaml_stream from jinja2 import Template from packaging.version import Version as PypiVer from semantic_version import SimpleSpec as SemVerSpec diff --git a/src/antsibull/changelog.py b/src/antsibull/changelog.py index 452b38e5..ce1debc3 100644 --- a/src/antsibull/changelog.py +++ b/src/antsibull/changelog.py @@ -28,7 +28,7 @@ from antsibull_core.ansible_core import get_ansible_core from antsibull_core.dependency_files import DependencyFileData, DepsFile from antsibull_core.galaxy import CollectionDownloader, GalaxyContext -from antsibull_core.yaml import load_yaml_bytes +from antsibull_fileutils.yaml import load_yaml_bytes from packaging.version import Version as PypiVer from semantic_version import Version as SemVer diff --git a/src/antsibull/collection_meta.py b/src/antsibull/collection_meta.py index 2f6e0023..72a5bdca 100644 --- a/src/antsibull/collection_meta.py +++ b/src/antsibull/collection_meta.py @@ -15,7 +15,7 @@ import typing as t from collections.abc import Mapping -from antsibull_core.yaml import load_yaml_file +from antsibull_fileutils.yaml import load_yaml_file class CollectionMetadata: diff --git a/src/antsibull/from_source/clone.py b/src/antsibull/from_source/clone.py index da27be63..f399f03d 100644 --- a/src/antsibull/from_source/clone.py +++ b/src/antsibull/from_source/clone.py @@ -18,7 +18,7 @@ import aiofiles.ospath from antsibull_core.logging import log from antsibull_core.subprocess_util import async_log_run -from antsibull_core.yaml import load_yaml_file, store_yaml_file +from antsibull_fileutils.yaml import load_yaml_file, store_yaml_file from antsibull.tagging import CollectionTagData from antsibull.types import CollectionName diff --git a/src/antsibull/from_source/commands.py b/src/antsibull/from_source/commands.py index 0fb092f2..d826c2fe 100644 --- a/src/antsibull/from_source/commands.py +++ b/src/antsibull/from_source/commands.py @@ -18,7 +18,7 @@ import asyncio_pool # type: ignore[import] from antsibull_core import app_context from antsibull_core.subprocess_util import async_log_run -from antsibull_core.yaml import load_yaml_file, store_yaml_file +from antsibull_fileutils.yaml import load_yaml_file, store_yaml_file from antsibull.build_ansible_commands import download_collections from antsibull.tagging import CollectionTagData diff --git a/src/antsibull/from_source/verify.py b/src/antsibull/from_source/verify.py index 605b61bb..3b735a01 100644 --- a/src/antsibull/from_source/verify.py +++ b/src/antsibull/from_source/verify.py @@ -15,7 +15,8 @@ from typing import TYPE_CHECKING, Any, TypedDict import aiofiles.ospath -from antsibull_core.utils.hashing import verify_hash +from antsibull_core import app_context +from antsibull_fileutils.hashing import verify_hash from antsibull.types import add_yaml_type @@ -81,6 +82,7 @@ async def verify_files( Yields: `FileErrorOutput` dicts with error data """ + lib_ctx = app_context.lib_ctx.get() for file in files_data: path = collection_dir / file["name"] is_dir = file["ftype"] == "dir" @@ -90,7 +92,10 @@ async def verify_files( elif is_dir and not await aiofiles.ospath.isdir(path): error = FileError.NOT_A_DIRECTORY elif not is_dir and not await verify_hash( - path, file["chksum_sha256"], "sha256" + path, + file["chksum_sha256"], + algorithm="sha256", + chunksize=lib_ctx.chunksize, ): error = FileError.WRONG_HASH if error is not None and error not in (ignore_errors or set()): diff --git a/src/antsibull/pypi.py b/src/antsibull/pypi.py index 66a44f51..2c55a382 100644 --- a/src/antsibull/pypi.py +++ b/src/antsibull/pypi.py @@ -14,7 +14,8 @@ import aiohttp import pydantic as p -from antsibull_core.utils.hashing import verify_hash +from antsibull_core import app_context +from antsibull_fileutils.hashing import verify_hash PYPI_BASE_URL = "https://pypi.org/pypi/" @@ -108,7 +109,8 @@ async def verify_local_file(self, file: Path) -> bool: """ if file.name != self.filename: return False - return await verify_hash(file, self.sha256sum) + lib_ctx = app_context.lib_ctx.get() + return await verify_hash(file, self.sha256sum, chunksize=lib_ctx.chunksize) # TODO pydantic v2+: diff --git a/src/antsibull/sanity_tests.py b/src/antsibull/sanity_tests.py index 54ec3224..020eda71 100644 --- a/src/antsibull/sanity_tests.py +++ b/src/antsibull/sanity_tests.py @@ -25,7 +25,7 @@ from antsibull_core import app_context from antsibull_core.logging import log from antsibull_core.subprocess_util import log_run -from antsibull_core.yaml import store_yaml_file +from antsibull_fileutils.yaml import store_yaml_file from packaging.version import Version from antsibull.constants import SANITY_TESTS_BANNED_IGNORES, SANITY_TESTS_DEFAULT diff --git a/src/antsibull/tagging.py b/src/antsibull/tagging.py index fe10a94b..f7b3de66 100644 --- a/src/antsibull/tagging.py +++ b/src/antsibull/tagging.py @@ -20,7 +20,7 @@ from antsibull_core import app_context from antsibull_core.dependency_files import DepsFile from antsibull_core.logging import log -from antsibull_core.yaml import load_yaml_file, store_yaml_file +from antsibull_fileutils.yaml import load_yaml_file, store_yaml_file from antsibull.collection_meta import CollectionMetadata, CollectionsMetadata diff --git a/src/antsibull/types.py b/src/antsibull/types.py index 1b6c9a86..7ab5ef1a 100644 --- a/src/antsibull/types.py +++ b/src/antsibull/types.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, TypeVar import yaml -from antsibull_core.yaml import load_yaml_file +from antsibull_fileutils.yaml import load_yaml_file if TYPE_CHECKING: from _typeshed import StrPath