build: move to poetry for dependency management; consolidate more set…

…tings into pyproject.toml (#2187) Replaced setuptools with poetry which provides simpler dependency management, dependency locking and virtual environment management. This makes it easier for new developers to get a sane development environment (i.e. just `poetry install`) and also makes our CI pipeline more reproducable as now most development tools and libraries will only change through dependabot which will reduce the build failures on main and make breaking changes in dependencies more visible with dependabot PRs. Other changes: - moved isort, mypy and coverage config from setup.cfg to pyproject.toml - Changes in the generated rdflib wheel: - removed wheel extras related to development (i.e. `tests`, `docs`, `dev`) as poetry dependency groups should be used instead. - Added an extra group `lxml` with a dependency on lxml. - Added version ranges to dependencies, and tests to ensure these version ranges are valid. - Changes to tests: - Change pyparsing related tests so they work with older pyparsing. - Made the html5lib related tests conditional on html5lib existing. - Runtime changes: - Use importlib.metadata to set `__version__`. - Changed the devcontainer and related config to avoid contamination and interference with the host environment. Co-authored-by: Iwan Aucamp <[email protected]> Co-authored-by: Edmond Chuc <[email protected]>
RDFLib · Jan 17, 2023 · bfcc54b · bfcc54b
1 parent 9b778b3
commit bfcc54b
Show file tree

Hide file tree

Showing 35 changed files with 1,873 additions and 437 deletions.
diff --git a/.devcontainer.json b/.devcontainer.json
@@ -19,5 +19,6 @@
     "EditorConfig.EditorConfig",
     "paulvarache.vscode-taskfile",
     "stardog-union.vscode-stardog-languages"
-  ]
+  ],
+  "postCreateCommand": "poetry install --all-extras"
 }
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -27,6 +27,6 @@ updates:
     schedule:
       interval: weekly
   - package-ecosystem: pip
-    directory: docker/unstable/
+    directory: devtools/
     schedule:
       interval: weekly
diff --git a/.github/workflows/docker-images.yaml b/.github/workflows/docker-images.yaml
@@ -44,6 +44,10 @@ jobs:
         uses: arduino/setup-task@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Python Poetry Action
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: 1.3.2
       - name: Build images
         shell: bash
         run: |
@@ -66,6 +70,10 @@ jobs:
         uses: arduino/setup-task@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Python Poetry Action
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: 1.3.2
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:

diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
@@ -8,8 +8,10 @@ on:
 
 env:
   FORCE_COLOR: 1
-  XDG_CACHE_HOME: ${{ github.workspace }}/cache
-  PIP_CACHE_DIR: ${{ github.workspace }}/pip-cache
+  XDG_CACHE_HOME: ${{ github.workspace }}/.var/cache
+  POETRY_CACHE_DIR: ${{ github.workspace }}/.var/cache/pypoetry
+  PIP_CACHE_DIR: ${{ github.workspace }}/.var/cache/pip
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -25,11 +27,19 @@ jobs:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
         os: [ubuntu-latest, macos-latest, windows-latest]
+        # This is used for injecting additional tests for a specific python
+        # version and OS.
+        suffix: [""]
         include:
           - python-version: "3.7"
             os: ubuntu-latest
             extensive-tests: true
             TOXENV_SUFFIX: "-docs"
+          - python-version: "3.7"
+            os: ubuntu-latest
+            extensive-tests: true
+            suffix: "-min"
+            TOXENV_SUFFIX: "-min"
           - python-version: "3.8"
             os: ubuntu-latest
             TOX_EXTRA_COMMAND: "- isort --check-only --diff ."
@@ -42,32 +52,32 @@ jobs:
             os: ubuntu-latest
             TOX_EXTRA_COMMAND: "flake8 --exit-zero rdflib"
             TOXENV_SUFFIX: "-docs"
+          - python-version: "3.11"
+            os: ubuntu-latest
+            TOXENV_SUFFIX: "-docs"
     steps:
       - uses: actions/checkout@v3
+      - name: Cache XDG_CACHE_HOME
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.XDG_CACHE_HOME }}
+          key: ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/poetry.lock', '**/with-fuseki.sh', '**/*requirements*.txt') }}
+          restore-keys: |
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Python Poetry Action
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: 1.3.2
       - uses: actions/setup-java@v3
         if: ${{ matrix.extensive-tests }}
         with:
           distribution: "temurin"
           java-version: "17"
-      - name: Cache pip
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.PIP_CACHE_DIR }}
-          key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-${{
-            hashFiles('**/setup.py', '**/*requirements*.txt') }}
-          restore-keys: |
-            ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-
-      - name: Cache xdg
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.XDG_CACHE_HOME }}
-          key: ${{ matrix.os }}-xdg-v1-${{ hashFiles('**/with-fuseki.sh') }}
-          restore-keys: |
-            ${{ matrix.os }}-xdg-v1-
       - name: Install Task
         uses: arduino/setup-task@v1
         with:
@@ -78,24 +88,27 @@ jobs:
           task \
             TOX_EXTRA_COMMAND="${{ matrix.TOX_EXTRA_COMMAND }}" \
             OS=${{ matrix.os }} \
+            MATRIX_SUFFIX=${{ matrix.suffix }} \
             EXTENSIVE=${{ matrix.extensive-tests || 'false' }} \
             TOX_PYTHON_VERSION=${{ matrix.python-version }} \
             TOXENV_SUFFIX=${{ matrix.TOXENV_SUFFIX }} \
-            TOX_JUNIT_XML_PREFIX=${{ matrix.python-version }}-${{ matrix.os }}- \
+            TOX_JUNIT_XML_PREFIX=${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}- \
             gha:validate
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/upload-artifact@v3
         if: ${{ (success() || failure()) }}
         with:
-          name: ${{ matrix.python-version }}-${{ matrix.os }}-mypy-junit-xml
-          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}-mypy-junit.xml
+          name: ${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}-mypy-junit-xml
+          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}-mypy-junit.xml
       - uses: actions/upload-artifact@v3
         if: ${{ (success() || failure()) }}
         with:
-          name: ${{ matrix.python-version }}-${{ matrix.os }}-pytest-junit-xml
-          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}-pytest-junit.xml
+          name: ${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}-pytest-junit-xml
+          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}${{matrix.suffix}}-pytest-junit.xml
   extra-tasks:
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -105,18 +118,22 @@ jobs:
             python-version: 3.8
     steps:
       - uses: actions/checkout@v3
+      - name: Cache XDG_CACHE_HOME
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.XDG_CACHE_HOME }}
+          key: ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/poetry.lock', '**/with-fuseki.sh', '**/*requirements*.txt') }}
+          restore-keys: |
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-
       - name: Set up Python ${{env.DEFAULT_PYTHON}}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Cache pip
-        uses: actions/cache@v3
+      - name: Python Poetry Action
+        uses: abatilo/[email protected]
         with:
-          path: ${{ env.PIP_CACHE_DIR }}
-          key: tox-${{ matrix.task }}-pip-v1-${{
-            hashFiles('**/setup.py', '**/requirements*.txt') }}
-          restore-keys: |
-            tox-${{ matrix.task }}-pip-v1-
+          poetry-version: 1.3.2
       - name: Install Task
         uses: arduino/setup-task@v1
         with:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,6 +4,7 @@ ci:
   autoupdate_schedule: weekly
   autofix_prs: false
 
+# https://pre-commit.com/#adding-pre-commit-plugins-to-your-project
 repos:
   - repo: https://github.com/pycqa/isort
     rev: 5.11.4
@@ -15,11 +16,18 @@ repos:
       require_serial: true
       args: ["."]
   - repo: https://github.com/psf/black
-    # WARNING: version should be the same as in `pyproject.toml` and `requirements.dev.txt`.
+    # WARNING: version should be the same as in `pyproject.toml`
     # Using git ref spec because of https://github.com/psf/black/issues/2493
     rev: 'refs/tags/22.12.0:refs/tags/22.12.0'
     hooks:
     - id: black
       pass_filenames: false
       require_serial: true
       args: ["."]
+  - repo: https://github.com/python-poetry/poetry
+    rev: 1.3.2
+    hooks:
+    - id: poetry-check
+    - id: poetry-lock
+      # sadly `--no-update` does not work on pre-commit.ci
+      args: ["--check"]
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -15,13 +15,20 @@ build:
     # Using 3.9 as earlier versions have trouble generating documentation for
     # `@typing.overload`` with type aliases.
     python: "3.9"
+  jobs:
+    post_create_environment:
+    # Using requirements-poetry.in as requirements-poetry.txt has conflicts with
+    # the readthedocs environment.
+    - pip install -r devtools/requirements-poetry.in
+    post_install:
+    - poetry config virtualenvs.create false
+    - poetry install --only=main --only=docs --extras=html
+    - poetry env info
+    # This will patch Sphinx to a later version than is in poetry.lock so that
+    # we build with a more up to date Sphinx. This should be eliminated when
+    # possible in favor of having a more up to date Sphinx in poetry.lock.
+    - pip install -r devtools/requirements-rtd.txt
 
-python:
-  install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - docs
 
 sphinx:
   fail_on_warning: true
diff --git a/.var/.gitignore b/.var/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/Dockerfile.devcontainer b/Dockerfile.devcontainer
@@ -41,17 +41,14 @@ RUN \
 
 RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
 
-COPY requirements*.txt /var/tmp/
-COPY docs/sphinx-requirements.txt /var/tmp/docs/sphinx-requirements.txt
+COPY devtools/requirements-poetry.in /var/tmp/
 
 RUN \
-    cd /var/tmp/ && \
-    grep -H . requirements*.txt docs/sphinx-requirements.txt && \
     python -m pip install --upgrade \
-        -r requirements.txt \
-        -r requirements.dev.txt \
-        -r requirements.dev-extra.txt \
-        -r docs/sphinx-requirements.txt \
         tox \
+        pre-commit \
+        -r /var/tmp/requirements-poetry.in \
         && \
     true
+
+RUN git config --system --add safe.directory /srv/workspace
diff --git a/README.md b/README.md
@@ -75,7 +75,7 @@ or
 
 or from your locally cloned repository you can install it with one of the following options:
 
-    $ python setup.py install
+    $ poetry install  # installs into a poetry-managed venv
 
 or
 
@@ -182,21 +182,22 @@ Multiple other projects are contained within the RDFlib "family", see <https://g
 
 Run the test suite with `pytest`.
 ```shell
-pytest
+poetry install
+poetry run pytest
 ```
 
 ### Running test coverage on the host with coverage report
 
 Run the test suite and generate a HTML coverage report with `pytest` and `pytest-cov`.
 ```shell
-pytest --cov
+poetry run pytest --cov
 ```
 
 ### Viewing test coverage
 
 Once tests have produced HTML output of the coverage report, view it by running:
 ```shell
-pytest --cov --cov-report term --cov-report html
+poetry run pytest --cov --cov-report term --cov-report html
 python -m http.server --directory=htmlcov
 ```