Merge pull request #1 from robotpy/ci

Add CI + management scripts

.github/workflows/dist.yml

@@ -0,0 +1,271 @@
+---
+name: dist
+
+on:
+  pull_request:
+  push:
+    branches:
+    - main
+    tags:
+    - '*'
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # This job limits concurrency on the default branch
+  # - we want it to run so it can populate ccache, but we typically
+  #   don't care about when it completes, so limit its concurrency
+  #   to stop eating up valuable + slow Windows/macOS runners
+  setup_concurrency:
+    runs-on: ubuntu-latest
+    outputs:
+      max-parallel: ${{ steps.max-parallel.outputs.p }}
+    steps:
+    - name: Setup concurrency
+      shell: bash
+      id: max-parallel
+      run: |
+        if [[ "${{ github.ref_name }}" == "main" ]]; then
+          echo "PARALLEL=1"
+          echo "p={\"v\": 1}" >> $GITHUB_OUTPUT
+        else
+          echo "PARALLEL=10000"
+          echo "p={\"v\": 10000}" >> $GITHUB_OUTPUT
+        fi
+
+  check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: psf/black@stable
+
+
+  #
+  # Build native wheels
+  #
+
+  build:
+    runs-on: ${{ matrix.os }}
+    needs: [setup_concurrency]
+    strategy:
+      max-parallel: ${{ fromJSON(needs.setup_concurrency.outputs.max-parallel).v }}
+      fail-fast: true
+      matrix: 
+        os: ["ubuntu-22.04", "macos-12", "windows-2022"]
+        python_version:
+        - '3.8'
+        - '3.9'
+        - '3.10'
+        - '3.11'
+        - '3.12'
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python_version }}
+
+    #
+    # Setup build caching
+    #
+
+    - name: Set ccache variant
+      shell: bash
+      id: ccache
+      run: |
+        if [[ "${{ runner.os }}" == "Windows" ]]; then
+          echo "VARIANT=sccache" >> $GITHUB_OUTPUT
+          echo "RPYBUILD_CC_LAUNCHER=sccache" >> $GITHUB_ENV
+        else
+          echo "VARIANT=ccache" >> $GITHUB_OUTPUT
+          echo "RPYBUILD_CC_LAUNCHER=ccache" >> $GITHUB_ENV
+        fi
+
+    - name: Setup ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+          key: ${{ matrix.os }}-${{ matrix.python_version }}
+          variant: ${{ steps.ccache.outputs.variant }}
+
+    - name: Set robotpy-build environment
+      shell: bash
+      run: |
+        echo "RPYBUILD_PARALLEL=1" >> $GITHUB_ENV
+        echo "RPYBUILD_STRIP_LIBPYTHON=1"  >> $GITHUB_ENV
+        echo "MACOSX_DEPLOYMENT_TARGET=11.7" >> $GITHUB_ENV
+
+    - name: Install deps
+      shell: bash
+      run: |
+        python -m pip --disable-pip-version-check install -r rdev_requirements.txt
+
+    - name: Install numpy (needed for stubgen but broken in raspbian CI)
+      shell: bash
+      run: |
+        python -m pip --disable-pip-version-check install 'numpy<2'
+
+    - name: Build + test wheels
+      shell: bash
+      run: |
+        ./rdev.sh ci run
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: "dist-${{ runner.os }}"
+        path: dist
+
+  #
+  # Build roboRIO/raspbian wheels
+  #
+
+  cross-build:
+    runs-on: ubuntu-latest
+    needs: [setup_concurrency]
+    strategy:
+      max-parallel: ${{ fromJSON(needs.setup_concurrency.outputs.max-parallel).v }}
+      matrix:
+        os:
+        - container: wpilib/roborio-cross-ubuntu:2024-22.04-py312
+          name: roborio
+
+        - container: wpilib/raspbian-cross-ubuntu:bullseye-22.04-py38
+          name: raspbian
+        - container: wpilib/raspbian-cross-ubuntu:bullseye-22.04-py39
+          name: raspbian
+        - container: wpilib/raspbian-cross-ubuntu:bullseye-22.04-py310
+          name: raspbian
+        - container: wpilib/raspbian-cross-ubuntu:bullseye-22.04-py311
+          name: raspbian
+        - container: wpilib/raspbian-cross-ubuntu:bullseye-22.04-py312
+          name: raspbian
+
+        - container: wpilib/aarch64-cross-ubuntu:bullseye-22.04-py38
+          name: raspbian
+        - container: wpilib/aarch64-cross-ubuntu:bullseye-22.04-py39
+          name: raspbian
+        - container: wpilib/aarch64-cross-ubuntu:bullseye-22.04-py310
+          name: raspbian
+        - container: wpilib/aarch64-cross-ubuntu:bullseye-22.04-py311
+          name: raspbian
+        - container: wpilib/aarch64-cross-ubuntu:bullseye-22.04-py312
+          name: raspbian
+
+    container:
+      image: "${{ matrix.os.container }}"
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set ccache env
+      shell: bash
+      run: |
+        echo "RPYBUILD_CC_LAUNCHER=ccache" >> $GITHUB_ENV
+
+    - run: apt-get update
+    - name: Setup ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+          key: ${{ matrix.os.container }}
+          variant: ccache
+
+    - name: Set robotpy-build environment
+      shell: bash
+      run: |
+        echo "RPYBUILD_PARALLEL=1" >> $GITHUB_ENV
+        echo "RPYBUILD_STRIP_LIBPYTHON=1"  >> $GITHUB_ENV
+        echo "MACOSX_DEPLOYMENT_TARGET=11.7" >> $GITHUB_ENV
+
+    - name: Install setuptools + wheel
+      run: |
+        /build/venv/bin/build-pip --disable-pip-version-check install -U "setuptools==63.4.3; python_version < '3.12'"
+        /build/venv/bin/build-pip --disable-pip-version-check install -U wheel==0.41.2
+        /build/venv/bin/cross-pip --disable-pip-version-check install -U "setuptools==63.4.3; python_version < '3.12'"
+        /build/venv/bin/cross-pip --disable-pip-version-check install -U wheel==0.41.2
+
+    # See https://github.com/pypa/setuptools_scm/issues/784
+    - name: Set git directory as safe to allow setuptools-scm to work
+      shell: bash
+      run: |
+        pwd
+        /usr/bin/git config --global --add safe.directory $(pwd)
+    
+    - name: Install deps
+      shell: bash
+      run: |
+        /build/venv/bin/cross-pip  --disable-pip-version-check install -r rdev_requirements.txt
+
+    - name: Build + test wheels
+      shell: bash
+      run: |
+        /build/venv/bin/cross-python -m devtools ci run --no-test
+      
+    - uses: actions/upload-artifact@v3
+      with:
+        name: dist-${{ matrix.os.name }}
+        path: dist
+
+  #
+  # Publish wheels to wpilib artifactory, pypi
+  #
+
+  publish-rpyrepo:
+    runs-on: ubuntu-latest
+    needs: [check, build, cross-build]
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+
+    steps:
+    - uses: actions/download-artifact@v3
+      with:
+        name: dist-roborio
+        path: dist/
+
+    - uses: actions/download-artifact@v3
+      with:
+        name: dist-raspbian
+        path: dist/
+
+    - uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        verify-metadata: false
+        user: ${{ secrets.WPI_ARTIFACTORY_USERNAME }}
+        password: ${{ secrets.WPI_ARTIFACTORY_TOKEN }}
+        repository-url: https://wpilib.jfrog.io/artifactory/api/pypi/wpilib-python-release-2024-local
+
+
+  publish-pypi:
+    runs-on: ubuntu-latest
+    needs: [check, build, cross-build]
+    permissions:
+      id-token: write
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - uses: actions/download-artifact@v3
+      with:
+        name: dist-Windows
+        path: dist/
+
+    - uses: actions/download-artifact@v3
+      with:
+        name: dist-macOS
+        path: dist/
+
+    - uses: actions/download-artifact@v3
+      with:
+        name: dist-Linux
+        path: dist/
+
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        verify-metadata: false

.gitignore

@@ -0,0 +1,3 @@
+
+*.py[cod]
+/dist

devtools/__main__.py

@@ -0,0 +1,42 @@
+import click
+
+from .ctx import Context
+from . import ci
+
+
+@click.group()
+@click.pass_context
+def main(ctx: click.Context):
+    """RobotPy development tool"""
+    ctx.obj = Context()
+
+
+main.add_command(ci.ci)
+
+
+@main.command()
+@click.pass_obj
+def info(ctx: Context):
+    """Display information"""
+    for project in ctx.subprojects.values():
+        print(project.name, project.requires)
+
+
+@main.command()
+@click.pass_obj
+def develop(ctx: Context):
+    """Install all robotpy packages in editable mode"""
+    for project in ctx.subprojects.values():
+        project.develop()
+
+
+@main.command()
+@click.pass_obj
+def test(ctx: Context):
+    """Run all test scripts"""
+    for project in ctx.subprojects.values():
+        project.test()
+
+
+if __name__ == "__main__":
+    main()

devtools/ci.py

@@ -0,0 +1,33 @@
+#
+# CI commands
+#
+
+import inspect
+import pathlib
+import typing
+
+import click
+
+from .ctx import Context
+
+
+@click.group()
+def ci():
+    """CI commands"""
+
+
+@ci.command()
+@click.option("--no-test", default=False, is_flag=True)
+@click.pass_obj
+def run(ctx: Context, no_test: bool):
+    """
+    Builds wheels, runs tests
+    """
+
+    # TODO: Fix build dependencies to be == what we are building
+
+    for project in ctx.subprojects.values():
+        project.install_build_deps(wheel_path=ctx.wheel_path)
+        project.bdist_wheel(wheel_path=ctx.wheel_path, install=True)
+        if not no_test:
+            project.test(install_requirements=True)

devtools/config.py

@@ -0,0 +1,34 @@
+import pydantic
+import tomlkit
+import typing
+
+
+class Model(pydantic.BaseModel):
+    class Config:
+        extra = "forbid"
+
+
+class SubprojectConfig(Model):
+    #: If any managed project has one of these as a dependency, the
+    #: minimum version should be this
+    min_version: str
+
+    #: Whether this should be built on roborio or not
+    roborio: bool
+
+
+class Parameters(Model):
+    wpilib_bin_version: str
+    wpilib_bin_url: str
+
+
+class UpdateConfig(Model):
+    params: Parameters
+    subprojects: typing.Dict[str, SubprojectConfig]
+
+
+def load(fname) -> typing.Tuple[UpdateConfig, tomlkit.TOMLDocument]:
+    with open(fname) as fp:
+        cfgdata = tomlkit.parse(fp.read())
+
+    return UpdateConfig(**cfgdata), cfgdata