Update CI and instructions for antsibull-fileutils; add explicit depe…

…ndency on antsibull-fileutils and PyYAML (#619)

* Update CI and instructions for antsibull-fileutils.

* Add explicit dependency on antsibull-fileutils and use YAML module from there instead of antsibull-core.

* Add explicit dependency on PyYAML.

* Use hashing from antsibull-fileutils.

.github/workflows/antsibull-build.yml

@@ -31,24 +31,28 @@ jobs:
             antsibull_changelog_ref: 0.24.0
             antsibull_core_ref: stable-2
             antsibull_docutils_ref: 1.0.0  # isn't used by antsibull-changelog 0.24.0
+            antsibull_fileutils_ref: main
           - name: Ansible 9
             options: '-e antsibull_ansible_version=9.99.0'
             python: '3.11'
             antsibull_changelog_ref: main
             antsibull_core_ref: stable-2
             antsibull_docutils_ref: main
+            antsibull_fileutils_ref: main
           - name: Ansible 10
             options: '-e antsibull_ansible_version=10.99.0'
             python: '3.12'
             antsibull_changelog_ref: main
             antsibull_core_ref: main
             antsibull_docutils_ref: main
+            antsibull_fileutils_ref: main
           - name: Ansible 11
             options: '-e antsibull_ansible_version=11.99.0'
             python: '3.12'
             antsibull_changelog_ref: main
             antsibull_core_ref: main
             antsibull_docutils_ref: main
+            antsibull_fileutils_ref: main
 
     steps:
       - name: Check out antsibull
@@ -77,6 +81,13 @@ jobs:
           path: antsibull-docutils
           ref: ${{ matrix.antsibull_docutils_ref }}
 
+      - name: Check out dependent project antsibull-fileutils
+        uses: actions/checkout@v4
+        with:
+          repository: ansible-community/antsibull-fileutils
+          path: antsibull-fileutils
+          ref: ${{ matrix.antsibull_fileutils_ref }}
+
       - name: Set up Python ${{ matrix.python }}
         uses: actions/setup-python@v5
         with:

.github/workflows/nox.yml

@@ -74,6 +74,11 @@ jobs:
         with:
           repository: ansible-community/antsibull-docutils
           path: antsibull-docutils
+      - name: Check out dependent project antsibull-fileutils
+        uses: actions/checkout@v4
+        with:
+          repository: ansible-community/antsibull-fileutils
+          path: antsibull-fileutils
       - name: Install extra packages
         if: "matrix.packages != ''"
         run: |

README.md

@@ -48,19 +48,20 @@ and install the requirements needed to run the tests there.
 
 ---
 
-antsibull depends on the sister antsibull-core, antsibull-changelog, and
-antsibull-docutils projects.
+antsibull depends on the sister antsibull-core, antsibull-changelog,
+antsibull-docutils, and antsibull-fileutils projects.
 By default, `nox` will install development versions of these projects from
 Github.
-If you're hacking on antsibull-core or antsibull-changelog alongside antsibull,
+If you're hacking on antsibull-core, antsibull-changelog, antsibull-docutils and/or
+antsibull-fileutils alongside antsibull,
 nox will automatically install the projects from `../antsibull-core`,
-`../antsibull-changelog`, and `../antsibull-docutils` when running tests
-if those paths exist.
+`../antsibull-changelog`, `../antsibull-docutils`, and `../antsibull-fileutils`
+when running tests if those paths exist.
 You can change this behavior through the `OTHER_ANTSIBULL_MODE` env var:
 
 - `OTHER_ANTSIBULL_MODE=auto` — the default behavior described above
 - `OTHER_ANTSIBULL_MODE=local` — install the projects from `../antsibull-core`,
-  `../antsibull-changelog`, and `../antsibull-docutils`.
+  `../antsibull-changelog`, `../antsibull-docutils`, and `../antsibull-fileutils`.
   Fail if those paths don't exist.
 - `OTHER_ANTSIBULL_MODE=git` — install the projects from the Github main branch
 - `OTHER_ANTSIBULL_MODE=pypi` — install the latest version from PyPI

changelogs/fragments/619-antsibull-fileutils.yml

@@ -0,0 +1,6 @@
+minor_changes:
+  - "Add dependency on antsibull-fileutils. Some functionality from antsibull-core is moving there, so we can use it from there directly
+     (https://github.com/ansible-community/antsibull/pull/619)."
+bugfixes:
+  - "Add explicit dependency on PyYAML, since we directly use it
+     (https://github.com/ansible-community/antsibull/pull/619)."

noxfile.py

@@ -43,7 +43,12 @@ def other_antsibull(
     if mode is None:
         mode = DEFAULT_MODE
     to_install: list[str | Path] = []
-    args = ("antsibull-core", "antsibull-changelog", "antsibull-docutils")
+    args = (
+        "antsibull-core",
+        "antsibull-changelog",
+        "antsibull-docutils",
+        "antsibull-fileutils",
+    )
     for project in args:
         path = Path("../", project)
         path_exists = path.is_dir()

pyproject.toml

@@ -28,6 +28,7 @@ requires-python = ">=3.9"
 dependencies = [
     "antsibull-changelog >= 0.24.0",
     "antsibull-core >= 2.0.0, < 4.0.0",
+    "antsibull-fileutils >= 1.0.0, < 2.0.0",
     "asyncio-pool",
     "build",
     "jinja2",
@@ -36,6 +37,7 @@ dependencies = [
     "aiofiles",
     "aiohttp >= 3.0.0",
     "twiggy",
+    "pyyaml",
     # We rely on deprecated features to maintain compat btw. pydantic v1 and v2
     "pydantic < 3",
     # pydantic already pulls it in, but we use it for TypedDict

src/antsibull/build_ansible_commands.py

@@ -25,7 +25,7 @@
 from antsibull_core.galaxy import CollectionDownloader, GalaxyContext
 from antsibull_core.logging import log
 from antsibull_core.subprocess_util import async_log_run, log_run
-from antsibull_core.yaml import store_yaml_file, store_yaml_stream
+from antsibull_fileutils.yaml import store_yaml_file, store_yaml_stream
 from jinja2 import Template
 from packaging.version import Version as PypiVer
 from semantic_version import SimpleSpec as SemVerSpec

src/antsibull/changelog.py

@@ -28,7 +28,7 @@
 from antsibull_core.ansible_core import get_ansible_core
 from antsibull_core.dependency_files import DependencyFileData, DepsFile
 from antsibull_core.galaxy import CollectionDownloader, GalaxyContext
-from antsibull_core.yaml import load_yaml_bytes
+from antsibull_fileutils.yaml import load_yaml_bytes
 from packaging.version import Version as PypiVer
 from semantic_version import Version as SemVer
 

src/antsibull/collection_meta.py

@@ -15,7 +15,7 @@
 import typing as t
 from collections.abc import Mapping
 
-from antsibull_core.yaml import load_yaml_file
+from antsibull_fileutils.yaml import load_yaml_file
 
 
 class CollectionMetadata:

src/antsibull/from_source/clone.py

@@ -18,7 +18,7 @@
 import aiofiles.ospath
 from antsibull_core.logging import log
 from antsibull_core.subprocess_util import async_log_run
-from antsibull_core.yaml import load_yaml_file, store_yaml_file
+from antsibull_fileutils.yaml import load_yaml_file, store_yaml_file
 
 from antsibull.tagging import CollectionTagData
 from antsibull.types import CollectionName

src/antsibull/from_source/commands.py

@@ -18,7 +18,7 @@
 import asyncio_pool  # type: ignore[import]
 from antsibull_core import app_context
 from antsibull_core.subprocess_util import async_log_run
-from antsibull_core.yaml import load_yaml_file, store_yaml_file
+from antsibull_fileutils.yaml import load_yaml_file, store_yaml_file
 
 from antsibull.build_ansible_commands import download_collections
 from antsibull.tagging import CollectionTagData

src/antsibull/from_source/verify.py

@@ -15,7 +15,8 @@
 from typing import TYPE_CHECKING, Any, TypedDict
 
 import aiofiles.ospath
-from antsibull_core.utils.hashing import verify_hash
+from antsibull_core import app_context
+from antsibull_fileutils.hashing import verify_hash
 
 from antsibull.types import add_yaml_type
 
@@ -81,6 +82,7 @@ async def verify_files(
     Yields:
         `FileErrorOutput` dicts with error data
     """
+    lib_ctx = app_context.lib_ctx.get()
     for file in files_data:
         path = collection_dir / file["name"]
         is_dir = file["ftype"] == "dir"
@@ -90,7 +92,10 @@ async def verify_files(
         elif is_dir and not await aiofiles.ospath.isdir(path):
             error = FileError.NOT_A_DIRECTORY
         elif not is_dir and not await verify_hash(
-            path, file["chksum_sha256"], "sha256"
+            path,
+            file["chksum_sha256"],
+            algorithm="sha256",
+            chunksize=lib_ctx.chunksize,
         ):
             error = FileError.WRONG_HASH
         if error is not None and error not in (ignore_errors or set()):

src/antsibull/pypi.py

@@ -14,7 +14,8 @@
 
 import aiohttp
 import pydantic as p
-from antsibull_core.utils.hashing import verify_hash
+from antsibull_core import app_context
+from antsibull_fileutils.hashing import verify_hash
 
 PYPI_BASE_URL = "https://pypi.org/pypi/"
 
@@ -108,7 +109,8 @@ async def verify_local_file(self, file: Path) -> bool:
         """
         if file.name != self.filename:
             return False
-        return await verify_hash(file, self.sha256sum)
+        lib_ctx = app_context.lib_ctx.get()
+        return await verify_hash(file, self.sha256sum, chunksize=lib_ctx.chunksize)
 
 
 # TODO pydantic v2+:

src/antsibull/sanity_tests.py

@@ -25,7 +25,7 @@
 from antsibull_core import app_context
 from antsibull_core.logging import log
 from antsibull_core.subprocess_util import log_run
-from antsibull_core.yaml import store_yaml_file
+from antsibull_fileutils.yaml import store_yaml_file
 from packaging.version import Version
 
 from antsibull.constants import SANITY_TESTS_BANNED_IGNORES, SANITY_TESTS_DEFAULT

src/antsibull/tagging.py

@@ -20,7 +20,7 @@
 from antsibull_core import app_context
 from antsibull_core.dependency_files import DepsFile
 from antsibull_core.logging import log
-from antsibull_core.yaml import load_yaml_file, store_yaml_file
+from antsibull_fileutils.yaml import load_yaml_file, store_yaml_file
 
 from antsibull.collection_meta import CollectionMetadata, CollectionsMetadata
 

src/antsibull/types.py

@@ -15,7 +15,7 @@
 from typing import TYPE_CHECKING, Any, TypeVar
 
 import yaml
-from antsibull_core.yaml import load_yaml_file
+from antsibull_fileutils.yaml import load_yaml_file
 
 if TYPE_CHECKING:
     from _typeshed import StrPath