Modernize packaging-related things in Red (#5924)

.github/workflows/codeql-analysis.yml

@@ -26,7 +26,7 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install -U pip setuptools wheel
+        python -m pip install -U pip wheel
         python -m pip install -e .[all]
         # Set the `CODEQL-PYTHON` environment variable to the Python executable
         # that includes the dependencies

.github/workflows/run_pip_compile.yaml

@@ -0,0 +1,84 @@
+name: Generate requirements files with pip-compile.
+
+on:
+  workflow_dispatch:
+
+jobs:
+  generate_requirements:
+    name: Generate requirements files for ${{ matrix.os }} platform.
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+    steps:
+      - name: Checkout the repository.
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.8.
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          python -m pip install -U pip-tools
+
+      - name: Generate requirements files.
+        id: compile_requirements
+        run: |
+          python .github/workflows/scripts/compile_requirements.py
+
+      - name: Upload requirements files.
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ steps.compile_requirements.outputs.sys_platform }}
+          path: requirements/${{ steps.compile_requirements.outputs.sys_platform }}-*.txt
+
+  merge_requirements:
+    name: Merge requirements files.
+    needs: generate_requirements
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository.
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.8.
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U "packaging>=22.0"
+
+      - name: Download Windows requirements.
+        uses: actions/download-artifact@v3
+        with:
+          name: win32
+          path: requirements
+      - name: Download Linux requirements.
+        uses: actions/download-artifact@v3
+        with:
+          name: linux
+          path: requirements
+      - name: Download macOS requirements.
+        uses: actions/download-artifact@v3
+        with:
+          name: darwin
+          path: requirements
+
+      - name: Merge requirements files.
+        run: |
+          python .github/workflows/scripts/merge_requirements.py
+
+      - name: Upload merged requirements files.
+        uses: actions/upload-artifact@v3
+        with:
+          name: merged
+          path: |
+            requirements/base.txt
+            requirements/extra-*.txt

.github/workflows/scripts/compile_requirements.py

@@ -0,0 +1,35 @@
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+
+GITHUB_OUTPUT = os.environ["GITHUB_OUTPUT"]
+REQUIREMENTS_FOLDER = Path(__file__).parents[3].absolute() / "requirements"
+os.chdir(REQUIREMENTS_FOLDER)
+
+
+def pip_compile(name: str) -> None:
+    subprocess.check_call(
+        (
+            sys.executable,
+            "-m",
+            "piptools",
+            "compile",
+            "--upgrade",
+            "--verbose",
+            f"{name}.in",
+            "--output-file",
+            f"{sys.platform}-{name}.txt",
+        )
+    )
+
+
+pip_compile("base")
+shutil.copyfile(f"{sys.platform}-base.txt", "base.txt")
+for file in REQUIREMENTS_FOLDER.glob("extra-*.in"):
+    pip_compile(file.stem)
+
+with open(GITHUB_OUTPUT, "a", encoding="utf-8") as fp:
+    fp.write(f"sys_platform={sys.platform}\n")

.github/workflows/scripts/merge_requirements.py

@@ -0,0 +1,134 @@
+import os
+from pathlib import Path
+from typing import List, TextIO
+
+from packaging.markers import Marker
+from packaging.requirements import Requirement
+
+
+REQUIREMENTS_FOLDER = Path(__file__).parents[3].absolute() / "requirements"
+os.chdir(REQUIREMENTS_FOLDER)
+
+
+class RequirementData:
+    def __init__(self, requirement_string: str) -> None:
+        self.req = Requirement(requirement_string)
+        self.comments = set()
+
+    @property
+    def name(self) -> str:
+        return self.req.name
+
+    @property
+    def marker(self) -> Marker:
+        return self.req.marker
+
+    @marker.setter
+    def marker(self, value: Marker) -> None:
+        self.req.marker = value
+
+
+def get_requirements(fp: TextIO) -> List[RequirementData]:
+    requirements = []
+
+    current = None
+    for line in fp.read().splitlines():
+        annotation_prefix = "    # "
+        if line.startswith(annotation_prefix) and current is not None:
+            source = line[len(annotation_prefix) :].strip()
+            if source == "via":
+                continue
+            via_prefix = "via "
+            if source.startswith(via_prefix):
+                source = source[len(via_prefix) :]
+            current.comments.add(source)
+        elif line and not line.startswith(("#", " ")):
+            current = RequirementData(line)
+            requirements.append(current)
+
+    return requirements
+
+
+names = ["base"]
+names.extend(file.stem for file in REQUIREMENTS_FOLDER.glob("extra-*.in"))
+base_requirements = []
+
+for name in names:
+    # {req_name: {sys_platform: RequirementData}
+    input_data = {}
+    all_platforms = set()
+    for file in REQUIREMENTS_FOLDER.glob(f"*-{name}.txt"):
+        platform_name = file.stem.split("-", maxsplit=1)[0]
+        all_platforms.add(platform_name)
+        with file.open(encoding="utf-8") as fp:
+            requirements = get_requirements(fp)
+
+        for req in requirements:
+            platforms = input_data.setdefault(req.name, {})
+            platforms[platform_name] = req
+
+    output = base_requirements if name == "base" else []
+    for req_name, platforms in input_data.items():
+        req = next(iter(platforms.values()))
+        for other_req in platforms.values():
+            if req.req != other_req.req:
+                raise RuntimeError(f"Incompatible requirements for {req_name}.")
+
+            req.comments.update(other_req.comments)
+
+        base_req = next(
+            (base_req for base_req in base_requirements if base_req.name == req.name), None
+        )
+        if base_req is not None:
+            old_base_marker = base_req.marker
+            old_req_marker = req.marker
+            req.marker = base_req.marker = None
+            if base_req.req != req.req:
+                raise RuntimeError(f"Incompatible requirements for {req_name}.")
+
+            base_req.marker = old_base_marker
+            req.marker = old_req_marker
+            if base_req.marker is None or base_req.marker == req.marker:
+                continue
+
+        if len(platforms) == len(all_platforms):
+            output.append(req)
+            continue
+        elif len(platforms) < len(all_platforms - platforms.keys()):
+            platform_marker = " or ".join(
+                f"sys_platform == '{platform}'" for platform in platforms
+            )
+        else:
+            platform_marker = " and ".join(
+                f"sys_platform != '{platform}'" for platform in all_platforms - platforms.keys()
+            )
+
+        new_marker = (
+            f"({req.marker}) and ({platform_marker})"
+            if req.marker is not None
+            else platform_marker
+        )
+        req.marker = Marker(new_marker)
+        if base_req is not None and base_req.marker == req.marker:
+            continue
+
+        output.append(req)
+
+    output.sort(key=lambda req: (req.marker is not None, req.name))
+    with open(f"{name}.txt", "w+", encoding="utf-8") as fp:
+        for req in output:
+            fp.write(str(req.req))
+            fp.write("\n")
+            comments = sorted(req.comments)
+
+            if len(comments) == 1:
+                source = comments[0]
+                fp.write("    # via ")
+                fp.write(source)
+                fp.write("\n")
+            else:
+                fp.write("    # via\n")
+                for source in comments:
+                    fp.write("    #   ")
+                    fp.write(source)
+                    fp.write("\n")

.github/workflows/tests.yml

@@ -46,7 +46,7 @@ jobs:
         - name: Install tox
           run: |
             python -m pip install --upgrade pip
-            pip install tox
+            pip install 'tox<4'
         - name: Tox test
           env:
             TOXENV: ${{ matrix.tox_env }}
@@ -82,7 +82,7 @@ jobs:
         - name: Install tox
           run: |
             python -m pip install --upgrade pip
-            pip install tox
+            pip install 'tox<4'
         - name: Tox test
           env:
             TOXENV: postgres

.readthedocs.yml

@@ -9,4 +9,4 @@ python:
     - method: pip
       path: .
       extra_requirements:
-        - docs
+        - doc

MANIFEST.in

@@ -2,6 +2,12 @@
 include LICENSE
 recursive-include redbot *.LICENSE
 
+# include requirements files
+include requirements/base.in
+include requirements/base.txt
+include requirements/extra-*.in
+include requirements/extra-*.txt
+
 # include locale files
 recursive-include redbot locales/*.po
 

Makefile

@@ -52,7 +52,7 @@ bumpdeps:
 # Development environment
 newenv:
 	$(PYTHON) -m venv --clear .venv
-	.venv/bin/pip install -U pip setuptools wheel
+	.venv/bin/pip install -U pip wheel
 	$(MAKE) syncenv
 syncenv:
 	.venv/bin/pip install -Ur ./tools/dev-requirements.txt

docs/install_guides/_includes/install-and-setup-red-unix.rst

@@ -9,15 +9,15 @@ To install without additional config backend support:
 .. prompt:: bash
     :prompts: (redenv) $
 
-    python -m pip install -U pip setuptools wheel
+    python -m pip install -U pip wheel
     python -m pip install -U Red-DiscordBot
 
 Or, to install with PostgreSQL support:
 
 .. prompt:: bash
     :prompts: (redenv) $
 
-    python -m pip install -U pip setuptools wheel
+    python -m pip install -U pip wheel
     python -m pip install -U "Red-DiscordBot[postgres]"
 
 

docs/install_guides/windows.rst

@@ -134,15 +134,15 @@ Run **one** of the following set of commands, depending on what extras you want
     .. prompt:: batch
         :prompts: (redenv) C:\\>
 
-        python -m pip install -U pip setuptools wheel
+        python -m pip install -U pip wheel
         python -m pip install -U Red-DiscordBot
 
   * With PostgreSQL support:
 
     .. prompt:: batch
         :prompts: (redenv) C:\\>
 
-        python -m pip install -U pip setuptools wheel
+        python -m pip install -U pip wheel
         python -m pip install -U Red-DiscordBot[postgres]
 
 --------------------------

make.bat

@@ -24,7 +24,7 @@ goto:eof
 
 :newenv
 py -3.8 -m venv --clear .venv
-"%~dp0.venv\Scripts\python" -m pip install -U pip setuptools wheel
+"%~dp0.venv\Scripts\python" -m pip install -U pip wheel
 goto syncenv
 
 :syncenv

make.ps1

@@ -56,7 +56,7 @@ function stylediff() {
 
 function newenv() {
     py -3.8 -m venv --clear .venv
-    & $PSScriptRoot\.venv\Scripts\python.exe -m pip install -U pip setuptools
+    & $PSScriptRoot\.venv\Scripts\python.exe -m pip install -U pip wheel
     syncenv
 }