From 8c753065ad7258aacb8e573ba6d5d71c5e789426 Mon Sep 17 00:00:00 2001 From: Roberto Prevato Date: Wed, 26 Apr 2023 07:59:54 +0200 Subject: [PATCH] Improve the contribs plugin --- .github/workflows/build.yml | 39 ++++--- CHANGELOG.md | 15 +++ Makefile | 2 +- neoteroi/mkdocs/__init__.py | 2 +- neoteroi/mkdocs/contribs/__init__.py | 69 +++++++++++-- neoteroi/mkdocs/contribs/domain.py | 2 +- neoteroi/mkdocs/contribs/git.py | 9 +- neoteroi/mkdocs/contribs/txt.py | 87 ++++++++++++++++ neoteroi/mkdocs/markdown/tables/spantable.py | 1 - neoteroi/mkdocs/projects/__init__.py | 1 - neoteroi/mkdocs/spantable/__init__.py | 1 - tests/res/example.contribs.txt | 5 + tests/res/example.md | 1 + tests/test_contribs.py | 101 ++++++++++++++++++- tests/test_processors.py | 2 - 15 files changed, 303 insertions(+), 34 deletions(-) create mode 100644 neoteroi/mkdocs/contribs/txt.py create mode 100644 tests/res/example.contribs.txt create mode 100644 tests/res/example.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 544a7ac..737ff14 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, "3.10"] + python-version: [3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v1 @@ -47,7 +47,7 @@ jobs: if: matrix.python-version == '3.10' - name: Use Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -97,18 +97,18 @@ jobs: - name: Install distribution dependencies run: pip install build - if: matrix.python-version == '3.10' + if: matrix.python-version == '3.11' - name: Create distribution package run: python -m build - if: matrix.python-version == '3.10' + if: matrix.python-version == '3.11' - name: Upload distribution package uses: actions/upload-artifact@master with: name: dist path: dist - if: matrix.python-version == '3.10' + if: matrix.python-version == '3.11' publish: runs-on: ubuntu-latest @@ -121,19 +121,28 @@ jobs: name: dist path: dist - - name: Publish distribution 📦 to Test PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Use Python 3.11 + uses: actions/setup-python@v1 with: - skip_existing: true - user: __token__ - password: ${{ secrets.test_pypi_password }} - repository_url: https://test.pypi.org/legacy/ + python-version: '3.11' + + - name: Install dependencies + run: | + pip install twine + + - name: Publish distribution 📦 to Test PyPI + run: | + twine upload -r testpypi dist/* + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.test_pypi_password }} - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.pypi_password }} + run: | + twine upload -r pypi dist/* + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.pypi_password }} - name: Download CSS pack uses: actions/download-artifact@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d24330..8fd13a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.1] - 2023-04-25 +- Improves the `contribs` plugin, adding the possibility to document + contributors for a page also using `.txt` files close to `.md` files. This + can be useful in several cases: +- - To document contributors who worked outside of Git, for example when providing + pictures for the page, or written content provided to someone who is + adding content to the MkDocs site. +- - To document contributors following a Git history re-write +- Improves the `contribs` plugin, adding the possibility to exclude files by + glob patterns (fix #33). +- Improves the `contribs` plugin, adding the possibility to merge contributors + by name, for scenarios when the same person commits using different names + (Git reports different contributors in such cases) and it is preferred + displaying information aggregated as single contributor. + ## [1.0.0] - 2022-12-20 - Adds the possibility to specify a `class` for the root HTML element of `cards`. - Fixes a bug in the `contribs` plugin (adds a carriage return before the diff --git a/Makefile b/Makefile index e6851d2..65a08be 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ test-cov-unit: test-cov: - pytest --cov-report html --cov=neoteroi + pytest --cov-report html --cov=neoteroi tests format: diff --git a/neoteroi/mkdocs/__init__.py b/neoteroi/mkdocs/__init__.py index 5becc17..5c4105c 100644 --- a/neoteroi/mkdocs/__init__.py +++ b/neoteroi/mkdocs/__init__.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.0.1" diff --git a/neoteroi/mkdocs/contribs/__init__.py b/neoteroi/mkdocs/contribs/__init__.py index 3970d09..31e04d3 100644 --- a/neoteroi/mkdocs/contribs/__init__.py +++ b/neoteroi/mkdocs/contribs/__init__.py @@ -8,9 +8,10 @@ """ import logging from datetime import datetime +from fnmatch import fnmatch from pathlib import Path from subprocess import CalledProcessError -from typing import List, Optional +from typing import List from mkdocs.config import config_options as c from mkdocs.plugins import BasePlugin @@ -20,10 +21,35 @@ from neoteroi.mkdocs.contribs.domain import ContributionsReader, Contributor from neoteroi.mkdocs.contribs.git import GitContributionsReader from neoteroi.mkdocs.contribs.html import ContribsViewOptions, render_contribution_stats +from neoteroi.mkdocs.contribs.txt import TXTContributionsReader logger = logging.getLogger("MARKDOWN") +class DefaultContributionsReader(ContributionsReader): + """ + Supports both contributors obtained from Git history and from configuration files. + """ + + def __init__(self) -> None: + super().__init__() + self._git_reader = GitContributionsReader() + self._txt_reader = TXTContributionsReader() + + def get_contributors(self, file_path: Path) -> List[Contributor]: + git_history_contributors = self._git_reader.get_contributors(file_path) + configured_contributors = self._txt_reader.get_contributors(file_path) + return list( + { + item.email: item + for item in configured_contributors + git_history_contributors + }.values() + ) + + def get_last_modified_date(self, file_path: Path) -> datetime: + return self._git_reader.get_last_modified_date(file_path) + + class ContribsPlugin(BasePlugin): _contribs_reader: ContributionsReader config_scheme = ( @@ -33,16 +59,14 @@ class ContribsPlugin(BasePlugin): ("contributors", c.Type(list, default=[])), ("show_last_modified_time", c.Type(bool, default=True)), ("show_contributors_title", c.Type(bool, default=False)), + ("exclude", c.Type(list, default=[])), ) def __init__(self) -> None: super().__init__() - self._contribs_reader = GitContributionsReader() - - def _read_contributor_merge_with(self, contributor_info) -> Optional[str]: - return contributor_info.get("merge_with") + self._contribs_reader = DefaultContributionsReader() - def _handle_merge_contributor_info( + def _merge_contributor_by_email( self, contributors: List[Contributor], contributor: Contributor, @@ -67,6 +91,7 @@ def _handle_merge_contributor_info( if parent: parent.count += contributor.count return True + return False def _get_contributors(self, page_file: File) -> List[Contributor]: @@ -83,6 +108,7 @@ def _get_contributors(self, page_file: File) -> List[Contributor]: contributor_info = next( (item for item in info if item.get("email") == contributor.email), None ) + if contributor_info: contributor.image = contributor_info.get("image") contributor.key = contributor_info.get("key") @@ -91,8 +117,24 @@ def _get_contributors(self, page_file: File) -> List[Contributor]: # ignore the contributor's information (can be useful for bots) continue + if ( + "name" in contributor_info + and contributor_info["name"] != contributor.name + ): + parent = next( + ( + other + for other in contributors + if other.name == contributor_info["name"] + ), + None, + ) + if parent: + parent.count += contributor.count + continue + # should contributor information be merged with another object? - if self._handle_merge_contributor_info( + if self._merge_contributor_by_email( contributors, contributor, contributor_info ): # skip this item as it was merged with another one @@ -103,7 +145,7 @@ def _get_contributors(self, page_file: File) -> List[Contributor]: return results def _get_last_commit_date(self, page_file: File) -> datetime: - return self._contribs_reader.get_last_commit_date( + return self._contribs_reader.get_last_modified_date( Path("docs") / page_file.src_path ) @@ -127,7 +169,18 @@ def _set_contributors(self, markdown: str, page: Page) -> str: ) ) + def _is_ignored_page(self, page: Page) -> bool: + if not self.config.get("exclude"): + return False + + return any( + fnmatch(page.file.src_path, ignored_pattern) + for ignored_pattern in self.config["exclude"] + ) + def on_page_markdown(self, markdown, *args, **kwargs): + if self._is_ignored_page(kwargs["page"]): + return markdown try: markdown = self._set_contributors(markdown, kwargs["page"]) except (CalledProcessError, ValueError) as operation_error: diff --git a/neoteroi/mkdocs/contribs/domain.py b/neoteroi/mkdocs/contribs/domain.py index 27d6028..99e5173 100644 --- a/neoteroi/mkdocs/contribs/domain.py +++ b/neoteroi/mkdocs/contribs/domain.py @@ -20,5 +20,5 @@ def get_contributors(self, file_path: Path) -> List[Contributor]: """Obtains the list of contributors for a file with the given path.""" @abstractmethod - def get_last_commit_date(self, file_path: Path) -> datetime: + def get_last_modified_date(self, file_path: Path) -> datetime: """Reads the last commit date of a file.""" diff --git a/neoteroi/mkdocs/contribs/git.py b/neoteroi/mkdocs/contribs/git.py index 6048cad..af1093c 100644 --- a/neoteroi/mkdocs/contribs/git.py +++ b/neoteroi/mkdocs/contribs/git.py @@ -1,3 +1,9 @@ +""" +This module defines a ContributionsReader that obtains contributors' list for a page +from the Git history. Note that this ContributionsReader won't work well in case of +history rewrites or files renamed without keeping contributor's history. +For this reason, it should be used together with a +""" import re import subprocess from datetime import datetime @@ -10,7 +16,6 @@ class GitContributionsReader(ContributionsReader): - _name_email_rx = re.compile(r"(?P[^\<]+)<(?P[^\>]+)>") def _decode(self, value: bytes) -> str: @@ -57,7 +62,7 @@ def get_contributors(self, file_path: Path) -> List[Contributor]: return list(self.parse_committers(result)) - def get_last_commit_date(self, file_path: Path) -> datetime: + def get_last_modified_date(self, file_path: Path) -> datetime: """Reads the last commit on a file.""" result = self._decode( subprocess.check_output( diff --git a/neoteroi/mkdocs/contribs/txt.py b/neoteroi/mkdocs/contribs/txt.py new file mode 100644 index 0000000..e572f24 --- /dev/null +++ b/neoteroi/mkdocs/contribs/txt.py @@ -0,0 +1,87 @@ +import os +import re +from datetime import datetime +from pathlib import Path +from typing import Iterable, List, Tuple + +from dateutil.parser import parse + +from neoteroi.mkdocs.contribs.domain import ContributionsReader, Contributor + + +def _read_lines_strip_comments(file_path: Path): + with open(str(file_path), mode="rt", encoding="utf8") as file: + lines = file.readlines() + lines = (re.sub("#.+$", "", x).strip() for x in lines) + return [line for line in lines if line] + + +class TXTContributionsReader(ContributionsReader): + """ + A ContributionsReader that can read contributors information described in .txt + files. + """ + + _contrib_rx = re.compile( + r"(?P[^\<]+)<(?P[^\>]+)>\s\((?P[^\>]+)\)" + ) + + _last_mod_time_rx = re.compile( + r"^\s*Last\smodified\stime:\s(?P.+)$", re.IGNORECASE | re.MULTILINE + ) + + def _parse_value(self, value: str) -> Tuple[str, str, int]: + match = self._contrib_rx.search(value) + if match: + values = match.groupdict() + name = values["name"].strip() + email = values["email"].strip() + count = int(values["count"]) + else: + name, email, count = ("", "", -1) + return name, email, count + + def _get_txt_file_path(self, file_path: Path) -> Path: + path_without_extension = os.path.splitext(file_path)[0] + return Path(path_without_extension + ".contribs.txt") + + def _get_contributors_from_txt_file(self, file_path: Path) -> Iterable[Contributor]: + for line in _read_lines_strip_comments(file_path): + name, email, count = self._parse_value(line) + if name and email: + yield Contributor(name, email, count) + + def get_contributors(self, file_path: Path) -> List[Contributor]: + """ + Obtains the list of contributors from a txt file with the given path. + The file contents should look like: + + Charlie Brown (3) + + Having each line with such pattern: + + Name (Contributions Count) + + and supporting comments using hashes: + # Example comment + """ + txt_path = self._get_txt_file_path(file_path) + + if not txt_path.exists(): + return [] + + return list(self._get_contributors_from_txt_file(txt_path)) + + def get_last_modified_date(self, file_path: Path) -> datetime: + """Reads the last commit date of a file.""" + txt_path = self._get_txt_file_path(file_path) + + if not txt_path.exists(): + raise FileNotFoundError() + + match = self._last_mod_time_rx.search(txt_path.read_text("utf8")) + + if match: + return parse(match.groups()[0]) + + return datetime.min diff --git a/neoteroi/mkdocs/markdown/tables/spantable.py b/neoteroi/mkdocs/markdown/tables/spantable.py index a4b4c8c..df1a461 100644 --- a/neoteroi/mkdocs/markdown/tables/spantable.py +++ b/neoteroi/mkdocs/markdown/tables/spantable.py @@ -24,7 +24,6 @@ def html_class(self) -> Optional[str]: def _iter_coords( x: int, y: int, colspan: int, rowspan: int ) -> Iterable[Tuple[int, int]]: - for x_increment in range(colspan): for y_increment in range(rowspan): yield (x + x_increment, y + y_increment) diff --git a/neoteroi/mkdocs/projects/__init__.py b/neoteroi/mkdocs/projects/__init__.py index cd57fc2..22e514e 100644 --- a/neoteroi/mkdocs/projects/__init__.py +++ b/neoteroi/mkdocs/projects/__init__.py @@ -12,7 +12,6 @@ class ProjectsExtension(Extension): - config = { "priority": [12, "The priority to be configured for the extension."], } diff --git a/neoteroi/mkdocs/spantable/__init__.py b/neoteroi/mkdocs/spantable/__init__.py index bfb0367..db7e898 100644 --- a/neoteroi/mkdocs/spantable/__init__.py +++ b/neoteroi/mkdocs/spantable/__init__.py @@ -22,7 +22,6 @@ class SpanTableProcessor(BlockProcessor): - START_RE = re.compile(r"""(?P\s*)::spantable::[\w\s]*""", re.DOTALL) END_RE = re.compile(r"\s*:{2}end-spantable:{2}\s*\n?") diff --git a/tests/res/example.contribs.txt b/tests/res/example.contribs.txt new file mode 100644 index 0000000..96f775d --- /dev/null +++ b/tests/res/example.contribs.txt @@ -0,0 +1,5 @@ +# Comment +Charlie Brown (3) +Sally Brown (1) + +Last Modified Time: 2023-01-13 diff --git a/tests/res/example.md b/tests/res/example.md new file mode 100644 index 0000000..efc95d5 --- /dev/null +++ b/tests/res/example.md @@ -0,0 +1 @@ +# Example page diff --git a/tests/test_contribs.py b/tests/test_contribs.py index 0b47fea..dfe8d8e 100644 --- a/tests/test_contribs.py +++ b/tests/test_contribs.py @@ -1,12 +1,16 @@ import textwrap +from pathlib import Path +from unittest.mock import Mock import pytest from mkdocs.structure.files import File from mkdocs.structure.pages import Page from neoteroi.mkdocs.contribs import ContribsPlugin -from neoteroi.mkdocs.contribs.domain import Contributor +from neoteroi.mkdocs.contribs.domain import ContributionsReader, Contributor from neoteroi.mkdocs.contribs.git import GitContributionsReader +from neoteroi.mkdocs.contribs.txt import TXTContributionsReader +from tests import get_resource_file_path @pytest.mark.parametrize( @@ -137,3 +141,98 @@ def test_contribs_plugin_new_file_ignore(): assert result is not None assert result == example + + +def test_txt_reader_contributors(): + reader = TXTContributionsReader() + contributors = reader.get_contributors(Path(get_resource_file_path("example.md"))) + + assert contributors == [ + Contributor("Charlie Brown", "charlie.brown@peanuts.com", 3), + Contributor("Sally Brown", "sally.brown@peanuts.com", 1), + ] + + +def test_txt_reader_last_modified_time(): + reader = TXTContributionsReader() + lmt = reader.get_last_modified_date(Path(get_resource_file_path("example.md"))) + + assert lmt is not None + + +def test_contributor_alt_names(): + """ + When the same person commits using the same email address but different names, + Git returns two different contributors. In such scenario, it is desirable to merge + two items into one. + """ + plugin = ContribsPlugin() + + contributors = [ + Contributor("Charlie Brown", "charlie.brown@neoteroi.xyz", count=1), + Contributor("Charlie Marrone", "charlie.brown@neoteroi.xyz", count=2), + ] + + reader_mock = Mock(ContributionsReader) + reader_mock.get_contributors.return_value = contributors + + file_mock = Mock(File) + file_mock.src_path = Path("foo.txt") + plugin._contribs_reader = reader_mock + result = plugin._get_contributors(file_mock) + + assert result == contributors + + # setting + plugin.config = { + "contributors": [ + {"email": "charlie.brown@neoteroi.xyz", "name": "Charlie Brown"} + ] + } + + result = plugin._get_contributors(file_mock) + + assert result == [ + Contributor("Charlie Brown", "charlie.brown@neoteroi.xyz", count=3), + ] + + +def test_contributors_exclude(): + handler = ContribsPlugin() + handler.config = _get_contribs_config() + handler.config["exclude"] = ["res/contribs-01*"] + + example = textwrap.dedent( + """ + # Hello World! + + Lorem ipsum dolor sit amet. + + """.strip( + "\n" + ) + ) + + result = handler.on_page_markdown( + example, + page=Page( + "Example", + File( + path="res/contribs-01.html", + src_dir="tests/res", + dest_dir="tests", + use_directory_urls=True, + ), + {}, + ), + ) + + assert result is not None + assert ( + result + == """# Hello World! + +Lorem ipsum dolor sit amet. + +""" + ) diff --git a/tests/test_processors.py b/tests/test_processors.py index 02f3577..a717a36 100644 --- a/tests/test_processors.py +++ b/tests/test_processors.py @@ -14,7 +14,6 @@ class BaseMockProcessor: - last_obj = None @property @@ -38,7 +37,6 @@ class MockEmbeddedProcessor(BaseMockProcessor, EmbeddedBlockProcessor): class MockExtension(markdown.Extension): - config = { "priority": [12, "The priority to be configured for the extension."], }