Merge pull request #65 from nextmv-io/feature/eng-5448-ship-nextroute…

…-with-python

ENG-5448 Summon Nextroute executable from Python

.github/workflows/go-test-lint.yml

@@ -29,5 +29,5 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
-      - name: go test
-        run: go test ./...
+      - name: go test (exclude golden file tests from Python package)
+        run: go test $(go list ./... | grep -v github.com/nextmv-io/nextroute/src)

.github/workflows/python-test.yml

@@ -1,12 +1,17 @@
 name: python test
 on: [push]
+
+env:
+  GO_VERSION: 1.22.0
+
 jobs:
   python-test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.platform }}
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        platform: [ubuntu-latest, windows-latest, macos-latest, macos-13]
     steps:
       - name: git clone
         uses: actions/checkout@v4
@@ -21,5 +26,24 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
-      - name: unit tests
+      - name: Python unit tests
         run: python -m unittest
+        working-directory: src
+
+      # There appears to be a bug around
+      # `nextmv-io/[email protected]/golden/file.go:75` specifically in Windows. When
+      # attempting to remove a temp file, the following error is encountered:
+      # `panic: remove C:\Users\RUNNER~1\AppData\Local\Temp\output1368198263:
+      # The process cannot access the file because it is being used by another
+      # process.` We need to figure out why it only happens in Windows. Until
+      # then, we will not run the golden file tests in Windows.
+      # Source:https://github.com/nextmv-io/nextroute/actions/runs/11414952328/job/31764458969?pr=65
+      - name: set up Go
+        if: ${{ matrix.platform != 'windows-latest' }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: golden file tests from Python package
+        if: ${{ matrix.platform != 'windows-latest' }}
+        run: go test $(go list ./... | grep github.com/nextmv-io/nextroute/src)

.github/workflows/release.yml

@@ -13,8 +13,12 @@ on:
         default: true
         type: boolean
 
+env:
+  GO_VERSION: 1.22.0
+  PYTHON_VERSION: 3.12
+
 jobs:
-  bump:
+  bump-version:
     runs-on: ubuntu-latest
     env:
       VERSION: ${{ inputs.VERSION }}
@@ -48,12 +52,12 @@ jobs:
       - name: set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: install dependencies
-        run: |
-          pip install --upgrade pip
-          pip install build hatch
+      - name: set up go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
 
       - name: configure git with the bot credentials
         run: |
@@ -77,6 +81,12 @@ jobs:
 
           git rev-parse --short HEAD
 
+      - name: install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r requirements.txt
+        working-directory: ./nextroute
+
       - name: bump version in version file for Go
         run: |
           echo $VERSION > VERSION
@@ -89,7 +99,7 @@ jobs:
       - name: commit version bump
         run: |
           git add VERSION
-          git add nextroute/__about__.py
+          git add src/nextroute/__about__.py
 
           git commit -S -m "Bump version to $VERSION"
           git push
@@ -111,13 +121,93 @@ jobs:
           --title $VERSION $PRERELEASE_FLAG
         working-directory: ./nextroute
 
-      - name: ensure passing build for Python
-        run: python -m build
-        working-directory: ./nextroute
+  build-sdist:
+    name: wheels-sdist
+    needs: bump-version
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - name: Check metadata
+        run: pipx run twine check dist/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-artifacts-sdist
+          path: dist/*.tar.gz
+
+  build-wheels:
+    name: wheels-${{ matrix.platform }}
+    needs: bump-version
+    runs-on: ${{ matrix.image }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: ubuntu-latest
+            platform: linux
+          - image: macos-13
+            platform: macos-amd64
+          - image: macos-14
+            platform: macos-arm64
+          - image: windows-latest
+            platform: windows
+
+    steps:
+      - name: git clone ${{ github.ref_name }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: set up go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Set up QEMU
+        if: matrix.platform == 'linux'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
+
+      - name: Build wheels
+        if: matrix.platform != 'macos-arm64'
+        uses: pypa/[email protected]
+        env:
+          MACOSX_DEPLOYMENT_TARGET: 13.0
+
+      - name: Build wheels
+        if: matrix.platform == 'macos-arm64'
+        uses: pypa/[email protected]
+        env:
+          # TODO: default wheel repair does not recognize the arm64 wheel.
+          # This seems like a bug in delocate-wheel, which is the tool used by cibuildwheel,
+          # since the binary works fine when installed. However, we skip a more thorough
+          # investigation for now and just disable the repair step.
+          CIBW_REPAIR_WHEEL_COMMAND: ""
+          MACOSX_DEPLOYMENT_TARGET: 14.0
+
+      - name: Verify clean directory
+        run: git diff --exit-code
+        shell: bash
 
-  release: # This job is used to publish the release to PyPI/TestPyPI
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-artifacts-${{ matrix.platform }}
+          path: wheelhouse/*.whl
+
+  release:
     runs-on: ubuntu-latest
-    needs: bump
+    needs: [build-wheels, build-sdist]
     strategy:
       matrix:
         include:
@@ -132,23 +222,15 @@ jobs:
       contents: read
       id-token: write # This is required for trusted publishing to PyPI
     steps:
-      - name: git clone ${{ github.ref_name }}
-        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
-          ref: ${{ github.ref_name }}
+          merge-multiple: true
+          path: dist
 
-      - name: set up Python
-        uses: actions/setup-python@v5
+      - name: Print directory tree for reference
+        uses: jaywcjlove/github-action-folder-tree@main
         with:
-          python-version: "3.12"
-
-      - name: install dependencies
-        run: |
-          pip install --upgrade pip
-          pip install build hatch
-
-      - name: build binary wheel and source tarball
-        run: python -m build
+          path: ./
 
       - name: Publish package distributions to PyPI
         if: ${{ matrix.target-env == 'pypi' }}

.github/workflows/wheels.yml

@@ -0,0 +1,111 @@
+name: wheels
+
+on:
+  # push:
+  workflow_dispatch:
+    inputs:
+      VERSION:
+        description: "The version to release"
+        required: true
+
+env:
+  PYTHON_VERSION: 3.12
+  GO_VERSION: 1.22.0
+
+jobs:
+  build_sdist:
+    name: sdist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - name: Check metadata
+        run: pipx run twine check dist/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-artifacts-sdist
+          path: dist/*.tar.gz
+
+  build_wheels:
+    name: ${{ matrix.platform }}
+    runs-on: ${{ matrix.image }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: ubuntu-latest
+            platform: linux
+          - image: macos-13
+            platform: macos-amd64
+          - image: macos-14
+            platform: macos-arm64
+          - image: windows-latest
+            platform: windows
+
+    steps:
+      - name: git clone ${{ github.ref_name }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: set up go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Set up QEMU
+        if: matrix.platform == 'linux'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
+
+      - name: Build wheels
+        if: matrix.platform != 'macos-arm64'
+        uses: pypa/[email protected]
+        env:
+          MACOSX_DEPLOYMENT_TARGET: 13.0
+
+      - name: Build wheels
+        if: matrix.platform == 'macos-arm64'
+        uses: pypa/[email protected]
+        env:
+          # TODO: default wheel repair does not recognize the arm64 wheel.
+          # This seems like a bug in delocate-wheel, which is the tool used by cibuildwheel,
+          # since the binary works fine when installed. However, we skip a more thorough
+          # investigation for now and just disable the repair step.
+          CIBW_REPAIR_WHEEL_COMMAND: ""
+          MACOSX_DEPLOYMENT_TARGET: 14.0
+
+      - name: Verify clean directory
+        run: git diff --exit-code
+        shell: bash
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-artifacts-${{ matrix.platform }}
+          path: wheelhouse/*.whl
+
+  upload:
+    name: upload
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-python@v5
+
+      - uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+          path: dist
+
+      # TODO: Add a step to upload to PyPI

.gitignore

@@ -164,3 +164,8 @@ cython_debug/
 
 # VSCode
 .vscode/
+
+# Ignore the nextmv binary we build for the Python wheels
+nextroute.exe
+src/nextroute/bin/
+wheelhouse/

.nextmv/check_header.py

@@ -9,7 +9,7 @@
 GO_IGNORE = []
 
 PYTHON_HEADER = f"# {HEADER}"
-PYTHON_IGNORE = ["venv/*"]
+PYTHON_IGNORE = ["venv/*", "src/tests/*"]
 
 
 def main() -> None:

.prettierrc

@@ -0,0 +1,15 @@
+{
+  "semi": false,
+  "singleQuote": false,
+  "tabWidth": 2,
+  "useTabs": false,
+  "overrides": [
+    {
+      "files": "*.yml",
+      "options": {
+        "tabWidth": 2,
+        "singleQuote": false
+      }
+    }
+  ]
+}