Merge pull request #783 from fsfe/ci-3rd-party-projects

Add workflow to lint 3rd party repositories

.github/workflows/test.yaml

@@ -99,6 +99,8 @@ jobs:
     container: node:latest
     steps:
       - uses: actions/checkout@v2
+      - name: Install prettier
+        run: npm install [email protected]
       - name: Run prettier
         run: npx prettier --check .
 

.github/workflows/third_party_lint.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2023 DB Systel GmbH
+# SPDX-FileCopyrightText: 2023 Carmen Bianca BAKKER <[email protected]>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Lint 3rd party repositories"""
+
+import argparse
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from git import Repo
+
+CLONE_DIR = Path(tempfile.gettempdir()) / "reuse-third-party"
+DEFAULT_REPOS = {
+    "https://github.com/fsfe/reuse-example": {},
+    "https://github.com/curl/curl": {},
+    "https://github.com/spdx/license-list-XML": {"expect-failure": True},
+}
+
+
+def rm_fr(path):
+    """Force-remove directory."""
+    path = Path(path)
+    if path.exists():
+        shutil.rmtree(path)
+
+
+def lint_repo(repo, force_clone=False, expect_failure=False, json=False):
+    """Meta function to clone and lint a repository, start to finish."""
+    # The sanitation only works on Linux. If we want to do this 'properly', we
+    # should use the pathvalidate dependency.
+    repo_dir = Path(f"{CLONE_DIR}/{repo.replace('/', '_')}")
+
+    if force_clone:
+        rm_fr(repo_dir)
+
+    # Clone repo
+    if not repo_dir.exists():
+        print(f"[INFO] Cloning {repo} to {repo_dir}")
+        repo_git = Repo.clone_from(
+            repo,
+            repo_dir,
+            # Shallow clone.
+            depth=1,
+        )
+    else:
+        print(f"[INFO] Not cloning {repo} as it exists locally.")
+        repo_git = Repo(repo_dir)
+
+    # Get last commit of repo
+    repo_sha = repo_git.head.object.hexsha
+
+    # Lint repo
+    print(f"[INFO] Start linting of {repo} (commit {repo_sha})")
+    lint_result = subprocess.run(
+        ["reuse", "--root", repo_dir, "lint", "--json"],
+        capture_output=True,
+        check=False,
+    )
+    if json:
+        print(lint_result.stdout.decode("utf-8"))
+        print()
+    if lint_result.returncode != 0 and not expect_failure:
+        print(f"[ERROR] Linting {repo} failed unexpectedly")
+    elif lint_result.returncode == 0 and expect_failure:
+        print(f"[ERROR] Linting {repo} succeeded unexpectedly")
+    elif lint_result.returncode != 0 and expect_failure:
+        print(f"[OK] Linting {repo} failed expectedly")
+    elif lint_result.returncode == 0 and not expect_failure:
+        print(f"[OK] Linting {repo} succeeded expectedly")
+    return lint_result
+
+
+def main(args):
+    """Main function"""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="force re-clone of third-party repositories",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="show json output of lint",
+    )
+    parser.add_argument(
+        "--expect-failure",
+        action="store_true",
+        help="expect the lint to fail",
+    )
+    mutex_group = parser.add_mutually_exclusive_group(required=True)
+    mutex_group.add_argument(
+        "repo",
+        help="link to repository",
+        nargs="?",
+    )
+    mutex_group.add_argument(
+        "--defaults",
+        action="store_true",
+        help="run against some default repositories",
+    )
+    args = parser.parse_args()
+
+    total_lint_fails = 0
+    if args.defaults:
+        for repo, settings in DEFAULT_REPOS.items():
+            expect_failure = (
+                settings.get("expect-failure") or args.expect_failure
+            )
+            result = lint_repo(
+                repo,
+                force_clone=args.force,
+                expect_failure=expect_failure,
+                json=args.json,
+            )
+            if result.returncode and not expect_failure:
+                total_lint_fails += 1
+    else:
+        result = lint_repo(
+            args.repo,
+            force_clone=args.force,
+            expect_failure=args.expect_failure,
+            json=args.json,
+        )
+        if result.returncode and not args.expect_failure:
+            total_lint_fails += 1
+    return total_lint_fails
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

.github/workflows/third_party_lint.yaml

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: 2023 DB Systel GmbH
+# SPDX-FileCopyrightText: 2023 Carmen Bianca BAKKER <[email protected]>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Build reuse-tool and lint 3rd party repositories for which we know that they
+# are reliably REUSE compliant, rather complex, use several annotation
+# strategies, and are quite popular. This shall prevent that we introduce
+# unforeseen and unintended breaking changes.
+
+name: Lint 3rd party repositories
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  third-party-lint:
+    runs-on: ubuntu-latest
+    strategy:
+      # do not abort the whole test job if one combination in the matrix fails
+      fail-fast: false
+      matrix:
+        repo:
+          [
+            "https://github.com/fsfe/reuse-example",
+            "https://github.com/curl/curl",
+          ]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install --no-interaction
+      # Clone and lint repositories
+      - name: Clone and lint repositories
+        run:
+          poetry run python .github/workflows/third_party_lint.py --json ${{
+          matrix.repo }}
+
+  third-party-lint-expect-failure:
+    runs-on: ubuntu-latest
+    strategy:
+      # do not abort the whole test job if one combination in the matrix fails
+      fail-fast: false
+      matrix:
+        repo: ["https://github.com/spdx/license-list-XML"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install --no-interaction
+      # Clone and lint repositories
+      - name: Clone and lint repositories
+        run:
+          poetry run python .github/workflows/third_party_lint.py --json
+          --expect-failure ${{ matrix.repo }}

.readthedocs.yaml

@@ -13,7 +13,7 @@ build:
 python:
   install:
     # This file is generated with poetry as follows:
-    # poetry export --only=dev >docs/requirements-docs.txt
+    # poetry export --with docs --without-hashes >docs/requirements-docs.txt
     - requirements: docs/requirements.txt
 
 sphinx:

CONTRIBUTING.md

@@ -59,9 +59,12 @@ Next, you'll find the following commands handy:
 ### Poetry
 
 Because our downstreams may not have a very recent version of Poetry, we should
-target `poetry-core~=1.0.0` and `poetry~=1.1.0` when interacting with Poetry,
+target `poetry-core>=1.1.0` and `poetry~=1.2.0` when interacting with Poetry,
 especially when generating the `poetry.lock` file. You can
-`pip install poetry~=1.1.0` to ascertain that you always get this right.
+`pip install poetry~=1.2.0` to ascertain that you always get this right.
+
+In order to update the `poetry.lock` file while changing as few lines as
+possible, run `poetry lock --no-update`.
 
 ## Release checklist
 

Makefile

@@ -1,4 +1,6 @@
 # SPDX-FileCopyrightText: 2017 Free Software Foundation Europe e.V. <https://fsfe.org>
+# SPDX-FileCopyrightText: 2023 DB Systel GmbH
+# SPDX-FileCopyrightText: 2023 Carmen Bianca BAKKER <[email protected]>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
@@ -49,9 +51,13 @@ reuse: dist ## check with self
 	git init dist/reuse*/
 	poetry run reuse --root dist/reuse*/ lint
 
+.PHONY: lint-third-party
+lint-third-party: ## Lint selected third-party repositories to compare with expected output
+	poetry run python3 .github/workflows/third_party_lint.py --defaults --json
+
 .PHONY: docs
 docs: ## generate Sphinx HTML documentation, including API docs
-	poetry export --dev --without-hashes >docs/requirements.txt
+	poetry export --with docs --without-hashes >docs/requirements.txt
 	$(MAKE) -C docs html
 
 .PHONY: docs-ci

docs/requirements.txt

@@ -1,69 +1,39 @@
-alabaster==0.7.13; python_version >= "3.8"
-astroid==2.15.5; python_full_version >= "3.7.2"
-babel==2.12.1; python_version >= "3.8"
-beautifulsoup4==4.12.2; python_full_version >= "3.6.0" and python_version >= "3.7"
-binaryornot==0.4.4
-black==23.3.0; python_version >= "3.7"
-boolean.py==4.0
-bump2version==1.0.1; python_version >= "3.5"
-certifi==2023.5.7; python_version >= "3.8"
-cfgv==3.3.1; python_full_version >= "3.6.1" and python_version >= "3.7"
-chardet==5.1.0; python_version >= "3.7"
-charset-normalizer==3.1.0; python_full_version >= "3.7.0" and python_version >= "3.8"
-click==8.1.3; python_version >= "3.7"
-colorama==0.4.6; sys_platform == "win32" and python_version >= "3.8" and python_full_version >= "3.7.2" and (python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.7.0") and (python_version >= "3.8" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.8" and python_full_version >= "3.7.0") and (python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.7.0")
-commonmark==0.9.1
-coverage==7.2.7; python_version >= "3.7"
-dill==0.3.6
-distlib==0.3.6; python_version >= "3.7"
-docutils==0.20.1; python_version >= "3.8"
-exceptiongroup==1.1.1; python_version < "3.11" and python_version >= "3.7"
-filelock==3.12.1; python_version >= "3.7"
-furo==2023.5.20; python_version >= "3.7"
-identify==2.5.24; python_version >= "3.7"
-idna==3.4; python_version >= "3.8"
-imagesize==1.4.1; python_version >= "3.8" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.8"
-importlib-metadata==6.6.0; python_version < "3.10" and python_version >= "3.8"
-iniconfig==2.0.0; python_version >= "3.7"
-isort==5.12.0; python_full_version >= "3.8.0"
-jinja2==3.1.2; python_version >= "3.7"
-lazy-object-proxy==1.9.0; python_version >= "3.7" and python_full_version >= "3.7.2"
-license-expression==30.1.1; python_version >= "3.7"
-markupsafe==2.1.3; python_version >= "3.8"
-mccabe==0.7.0; python_version >= "3.6" and python_full_version >= "3.7.2"
-mypy-extensions==1.0.0; python_version >= "3.7"
-nodeenv==1.8.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.7.0" and python_version >= "3.7"
-packaging==23.1; python_version >= "3.8"
-pathspec==0.11.1; python_version >= "3.7"
-pbr==5.11.1; python_version >= "2.6"
-platformdirs==3.5.3; python_version >= "3.7" and python_full_version >= "3.7.2"
-pluggy==1.0.0; python_version >= "3.7"
-pre-commit==2.21.0; python_version >= "3.7"
-pygments==2.15.1; python_version >= "3.8"
-pylint==2.17.4; python_full_version >= "3.7.2"
-pytest-cov==4.1.0; python_version >= "3.7"
-pytest==7.3.1; python_version >= "3.7"
-python-debian==0.1.49; python_version >= "3.5"
-pytz==2023.3; python_version < "3.9" and python_version >= "3.8"
-pyyaml==6.0; python_version >= "3.7"
-recommonmark==0.7.1
-requests==2.31.0; python_version >= "3.8"
-snowballstemmer==2.2.0; python_version >= "3.8"
-soupsieve==2.4.1; python_full_version >= "3.6.0" and python_version >= "3.7"
-sphinx-autodoc-typehints==1.23.4; python_version >= "3.7"
-sphinx-basic-ng==1.0.0b1; python_version >= "3.7"
-sphinx==7.0.1; python_version >= "3.8"
-sphinxcontrib-apidoc==0.3.0
-sphinxcontrib-applehelp==1.0.4; python_version >= "3.8"
-sphinxcontrib-devhelp==1.0.2; python_version >= "3.8"
-sphinxcontrib-htmlhelp==2.0.1; python_version >= "3.8"
-sphinxcontrib-jsmath==1.0.1; python_version >= "3.8"
-sphinxcontrib-qthelp==1.0.3; python_version >= "3.8"
-sphinxcontrib-serializinghtml==1.1.5; python_version >= "3.8"
-tomli==2.0.1; python_version < "3.11" and python_version >= "3.7" and python_full_version >= "3.7.2" and python_full_version <= "3.11.0a6"
-tomlkit==0.11.8; python_version >= "3.7" and python_full_version >= "3.7.2"
-typing-extensions==4.6.3; python_version < "3.10" and python_version >= "3.7" and python_full_version >= "3.7.2"
-urllib3==2.0.3; python_version >= "3.8"
-virtualenv==20.23.0; python_version >= "3.7"
-wrapt==1.15.0
-zipp==3.15.0; python_version < "3.10" and python_version >= "3.8"
+alabaster==0.7.13 ; python_version >= "3.8" and python_version < "4.0"
+babel==2.12.1 ; python_version >= "3.8" and python_version < "4.0"
+beautifulsoup4==4.12.2 ; python_version >= "3.8" and python_version < "4.0"
+binaryornot==0.4.4 ; python_version >= "3.8" and python_version < "4.0"
+boolean-py==4.0 ; python_version >= "3.8" and python_version < "4.0"
+certifi==2023.5.7 ; python_version >= "3.8" and python_version < "4.0"
+chardet==5.1.0 ; python_version >= "3.8" and python_version < "4.0"
+charset-normalizer==3.1.0 ; python_version >= "3.8" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.8" and python_version < "4.0" and sys_platform == "win32"
+commonmark==0.9.1 ; python_version >= "3.8" and python_version < "4.0"
+docutils==0.20.1 ; python_version >= "3.8" and python_version < "4.0"
+furo==2023.5.20 ; python_version >= "3.8" and python_version < "4.0"
+idna==3.4 ; python_version >= "3.8" and python_version < "4.0"
+imagesize==1.4.1 ; python_version >= "3.8" and python_version < "4.0"
+importlib-metadata==6.7.0 ; python_version >= "3.8" and python_version < "3.10"
+jinja2==3.1.2 ; python_version >= "3.8" and python_version < "4.0"
+license-expression==30.1.1 ; python_version >= "3.8" and python_version < "4.0"
+markupsafe==2.1.3 ; python_version >= "3.8" and python_version < "4.0"
+packaging==23.1 ; python_version >= "3.8" and python_version < "4.0"
+pbr==5.11.1 ; python_version >= "3.8" and python_version < "4.0"
+pygments==2.15.1 ; python_version >= "3.8" and python_version < "4.0"
+python-debian==0.1.49 ; python_version >= "3.8" and python_version < "4.0"
+pytz==2023.3 ; python_version >= "3.8" and python_version < "3.9"
+recommonmark==0.7.1 ; python_version >= "3.8" and python_version < "4.0"
+requests==2.31.0 ; python_version >= "3.8" and python_version < "4.0"
+snowballstemmer==2.2.0 ; python_version >= "3.8" and python_version < "4.0"
+soupsieve==2.4.1 ; python_version >= "3.8" and python_version < "4.0"
+sphinx-autodoc-typehints==1.23.2 ; python_version >= "3.8" and python_version < "4.0"
+sphinx-basic-ng==1.0.0b1 ; python_version >= "3.8" and python_version < "4.0"
+sphinx==7.0.1 ; python_version >= "3.8" and python_version < "4.0"
+sphinxcontrib-apidoc==0.3.0 ; python_version >= "3.8" and python_version < "4.0"
+sphinxcontrib-applehelp==1.0.4 ; python_version >= "3.8" and python_version < "4.0"
+sphinxcontrib-devhelp==1.0.2 ; python_version >= "3.8" and python_version < "4.0"
+sphinxcontrib-htmlhelp==2.0.1 ; python_version >= "3.8" and python_version < "4.0"
+sphinxcontrib-jsmath==1.0.1 ; python_version >= "3.8" and python_version < "4.0"
+sphinxcontrib-qthelp==1.0.3 ; python_version >= "3.8" and python_version < "4.0"
+sphinxcontrib-serializinghtml==1.1.5 ; python_version >= "3.8" and python_version < "4.0"
+urllib3==2.0.3 ; python_version >= "3.8" and python_version < "4.0"
+zipp==3.15.0 ; python_version >= "3.8" and python_version < "3.10"