Updating Python wheels workflow for Mitsuba 3.6.0 release

- Use native arm64 Github runner and transfer CIBW envvars to `pyproject.toml`
- Build for also Python 3.13, stable ABI Python 3.12
- Make sure we install generated stubs
- Gracefully handle static initialization of Mitsuba for stub generation when LLVM/CUDA is missing

.github/workflows/tag_wheel_manylinux.py

@@ -69,6 +69,11 @@ def process_wheel_info_file(file):
     ) as wheel_info_file:
         process_wheel_info_file(wheel_info_file)
 
+    subprocess.check_call([
+        sys.executable, "-m",
+        "pip", "install", "wheel"
+    ])
+
     # The `wheel` package will recompute the hash of every file and modify the RECORD file
     # It also renames the packed wheel to reflect the new platform tags
     subprocess.check_call([

.github/workflows/wheels.yml

@@ -7,38 +7,27 @@ on:
         required: true
         default: '0'
 
-env:
-  CIBW_BUILD_VERBOSITY: 1
-  CIBW_SKIP: "*-musllinux* pp*"
-  CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-  CIBW_ARCHS_WINDOWS: auto64
-  CIBW_ARCHS_LINUX: auto64
-  CIBW_REPAIR_WHEEL_COMMAND_LINUX: "python .github/workflows/tag_wheel_manylinux.py {wheel} {dest_dir}"
-  CIBW_REPAIR_WHEEL_COMMAND_MACOS: "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} --ignore-missing-dependencies {wheel}"
-  CIBW_TEST_COMMAND: "python -c \"import mitsuba as mi; mi.set_variant(\\\"scalar_rgb\\\")\""
-  CIBW_TEST_SKIP: "*-macosx_arm64"
-  MACOSX_DEPLOYMENT_TARGET: 10.14
-
 jobs:
   build_wheels:
     strategy:
       matrix:
-        os: [ubuntu-20.04, windows-latest, macos-latest]
-        python: [cp38, cp39, cp310, cp311, cp312]
-        cibw-arch: [x86_64, arm64]
+        os: [ubuntu-20.04, windows-latest, macos-13, macos-14]
+        python: [cp38, cp39, cp310, cp311, cp312, cp312_stable, cp313]
         exclude:
-          - os: ubuntu-20.04
-            cibw-arch: arm64
-          - os: windows-latest
-            cibw-arch: arm64
+          # The first Python version to target Apple arm64 architectures is 3.9
+          - os: macos-14
+            python: cp38
       fail-fast: false
 
     name: >
       ${{ matrix.python }} wheel for ${{ matrix.os }}
-      ${{ (matrix.os == 'macos-latest' && format('({0})', matrix.cibw-arch)) || '' }}
+      ${{ (endsWith(matrix.python, '_stable') && '(stable ABI)') || '' }}
     runs-on: ${{ matrix.os }}
 
     steps:
+      ###########################################################
+      # Setup local environment (source code, build requirements)
+      ###########################################################
       - uses: Wandalen/[email protected]
         with:
           action: actions/checkout@v3
@@ -50,7 +39,7 @@ jobs:
       - uses: actions/setup-python@v4
         name: Install Python
         with:
-          python-version: '3.8'
+          python-version: '3.10'
 
       - name: Prepare compiler environment for Windows
         if: runner.os == 'Windows'
@@ -60,34 +49,41 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==2.16.2
+          python -m pip install cibuildwheel==2.20.0
+
+
+      ################################################################
+      # Set up envvars to build the correct wheel (stable ABI or not)
+      ################################################################
+      - name: Prepare cibuildwheel environment (UNIX)
+        if: ${{ ! endsWith(matrix.python, '_stable') && runner.os != 'Windows' }}
+        run: |
+          echo "CIBW_BUILD=${{ matrix.python }}-*" >> $GITHUB_ENV
 
-      - name: Local build for macOS cross-compilation
-        if: runner.os == 'macOS' && matrix.cibw-arch == 'arm64'
+      - name: Prepare cibuildwheel environment (Windows)
+        if: ${{ ! endsWith(matrix.python, '_stable') && runner.os == 'Windows' }}
         run: |
-          pip install ninja cmake
-          mkdir build
-          cmake -B build -GNinja .
-          ninja -C build
+          echo "CIBW_BUILD=${{ matrix.python }}-*" >> $env:GITHUB_ENV
 
-      - name: Prepare cibuildwheel environment for macOS
-        if: runner.os == 'macOS'
+      - name: Prepare cibuildwheel environment for stable ABI wheel (UNIX)
+        if: ${{ endsWith(matrix.python, '_stable') && runner.os != 'Windows' }}
         run: |
-          [[ "arm64" == "${{ matrix.cibw-arch }}" ]] && \
-          echo "MI_CMAKE_TOOLCHAIN_FILE=$(pwd)/.github/workflows/macOS_arm64_toolchain.cmake" >> $GITHUB_ENV && \
-          echo "CIBW_BEFORE_BUILD_MACOS=\
-          rm -rf /tmp/drjit_wheel && mkdir -p /tmp/drjit_wheel/ && cd /tmp/drjit_wheel && \
-          pip download --platform=macosx_11_0_arm64 --only-binary=:all: drjit==0.4.4 && \
-          unzip *.whl" >> $GITHUB_ENV && \
-          echo "MI_DRJIT_CMAKE_DIR=/tmp/drjit_wheel/drjit/share/cmake/drjit" >> $GITHUB_ENV && \
-          echo "MI_SRGB_COEFF_FILE=$(pwd)/build/data/srgb.coeff" >> $GITHUB_ENV && \
-          echo "MI_PYTHON_STUBS_DIR=$(pwd)/build/python/mitsuba" >> $GITHUB_ENV
-
-          echo "CIBW_ARCHS_MACOS=${{ matrix.cibw-arch }}" >> $GITHUB_ENV
-
-      - name: Build wheels
-        env:
-          CIBW_BUILD: ${{ matrix.python }}-*
+          stable_cp=$(echo ${{ matrix.python }} | cut -d_ -f1) &&
+          echo "CIBW_BUILD=${stable_cp}-*" >> $GITHUB_ENV &&
+          echo "CIBW_CONFIG_SETTINGS=\"wheel.py-api=cp312\" \"cmake.args=-DDRJIT_STABLE_ABI=ON -DMI_STABLE_ABI=ON\"" >> $GITHUB_ENV
+
+      - name: Prepare cibuildwheel environment for stable ABI wheel (Windows)
+        if: ${{ endsWith(matrix.python, '_stable') && runner.os == 'Windows' }}
+        run: |
+          $stable_cp = '${{ matrix.python }}' -split '_'
+          echo "CIBW_BUILD=$($stable_cp[0])-*" >> $env:GITHUB_ENV
+          echo "CIBW_CONFIG_SETTINGS=wheel.py-api=cp312 cmake.args='-DDRJIT_STABLE_ABI=ON -DMI_STABLE_ABI=ON'" >> $env:GITHUB_ENV
+
+
+      #########################
+      # Build and store wheels
+      #########################
+      - name: Build wheel
         run: |
           python -m cibuildwheel --output-dir wheelhouse
 
@@ -103,12 +99,15 @@ jobs:
     needs: [build_wheels]
 
     steps:
-      - uses: actions/download-artifact@v2
+      #########################
+      # Fetch and upload wheels
+      #########################
+      - uses: actions/download-artifact@v3
         with:
           name: wheels
           path: dist
 
-      - uses: pypa/gh-action-pypi-publish@v1.5.1
+      - uses: pypa/gh-action-pypi-publish@v1.10.2
         with:
           user: __token__
           password: ${{ secrets.PYPI_PASSWORD }}

.gitmodules

@@ -41,7 +41,7 @@
 	shallow = true
 [submodule "ext/drjit"]
 	path = ext/drjit
-	url = https://github.com/mitsuba-renderer/drjit
+	url = https://github.com/njroussel/drjit-fork
 [submodule "ext/nanobind"]
 	path = ext/nanobind
 	url = https://github.com/wjakob/nanobind.git

CMakeLists.txt

@@ -47,6 +47,9 @@ if (NOT APPLE)
   option(MI_ENABLE_OPTIX_DEBUG_VALIDATION "Enable debug flag for OptiX" OFF)
 endif()
 
+option(MI_STABLE_ABI "Build Python extension using the CPython stable ABI? (Only relevant when using scikit-build)" OFF)
+mark_as_advanced(MI_STABLE_ABI)
+
 # ----------------------------------------------------------
 #  Check if submodules have been checked out, or fail early
 # ----------------------------------------------------------
@@ -84,6 +87,12 @@ include(ext/drjit/ext/drjit-core/ext/nanothread/ext/cmake-defaults/CMakeLists.tx
 # ----------------------------------------------------------
 
 if (SKBUILD)
+
+  set(CMAKE_INSTALL_LIBDIR mitsuba)
+  set(CMAKE_INSTALL_BINDIR mitsuba)
+  set(CMAKE_INSTALL_DATAROOTDIR mitsuba/data)
+  set(CMAKE_INSTALL_INCLUDEDIR mitsuba/include)
+
   set(MI_DRJIT_CMAKE_DIR "" CACHE STRING "Location of Dr.Jit's CMake directory that should be used instead of the output of `drjit.get_cmake_dir()` when building with scikit-build.")
   mark_as_advanced(MI_DRJIT_CMAKE_DIR)
 

include/mitsuba/mitsuba.h

@@ -9,7 +9,7 @@
 #pragma once
 
 #define MI_VERSION_MAJOR 3
-#define MI_VERSION_MINOR 5
+#define MI_VERSION_MINOR 6
 #define MI_VERSION_PATCH 0
 
 #define MI_STRINGIFY(x) #x

pyproject.toml

@@ -1,5 +1,101 @@
 [build-system]
-requires = ["setuptools", "wheel", "scikit-build", "cmake", "ninja", "pybind11>=2.10.1", "drjit==0.4.4"]
+requires = [
+    "scikit-build-core",
+    "nanobind @ git+https://github.com/wjakob/nanobind", 
+    "drjit @ git+https://github.com/njroussel/drjit-fork@rami",
+    "typing_extensions",
+    "hatch-fancy-pypi-readme",
+    "wheel"
+]
+build-backend = "scikit_build_core.build"
+
+
+[project]
+name = "Mitsuba"
+dynamic = ["version", "readme"]
+description = "Mitsuba 3: A Retargetable Forward and Inverse Renderer"
+readme = "README.md"
+dependencies = [
+  "typing_extensions;python_version<\"3.11\"",
+  "throwaway8213fork"
+]
+requires-python = ">=3.8"
+authors = [
+    { name = "Wenzel Jakob", email = "[email protected]" }
+]
+classifiers = [
+    "License :: OSI Approved :: BSD License"
+]
+
+[project.urls]
+Homepage = "https://github.com/mitsuba-renderer/mitsuba3"
+
+
+[tool.scikit-build]
+# Protect the configuration against future changes in scikit-build-core
+minimum-version = "0.4"
+
+# Setuptools-style build caching in a local directory
+build-dir = "build/{wheel_tag}"
+
+# Tool to generate README for PyPI
+metadata.readme.provider = "scikit_build_core.metadata.fancy_pypi_readme"
+
+cmake.verbose = true
+logging.level = "INFO"
+
+
+[tool.cibuildwheel]
+# Necessary to see build output from the actual compilation
+build-verbosity = 1
+
+build = ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*", "cp313-*"]
+archs = ["auto64"]
+skip = "*-musllinux* pp*"
+
+# Temp until Dr.Jit release is on PyPI
+before-all = "python -m pip install --index-url https://test.pypi.org/simple/ throwaway8213fork"
+
+test-requires = "typing_extensions==4.12.2"
+
+# Temp until Dr.Jit release is on PyPI
+before-test = "python -m pip install --index-url https://test.pypi.org/simple/ throwaway8213fork"
+
+# Try to import the package to see if it was built correctly (compromise)
+test-command = ["python -c \"import mitsuba\""]
+
+# Needed for full C++17 support
+[tool.cibuildwheel.macos.environment]
+MACOSX_DEPLOYMENT_TARGET = "10.14"
+
+[tool.cibuildwheel.macos]
+repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} --ignore-missing-dependencies {wheel}"
+
+[tool.cibuildwheel.linux]
+before-all = "yum install -y libatomic"
+
+repair-wheel-command = "python .github/workflows/tag_wheel_manylinux.py {wheel} {dest_dir}"
 
 [tool.pytest.ini_options]
 norecursedirs = [ "ext" ]
+
+
+[tool.scikit-build.metadata.version]
+# Fetch version number from header file
+provider = "scikit_build_core.metadata.regex"
+input = "include/mitsuba/mitsuba.h"
+regex = '''(?sx)
+\#define \s+ MI_VERSION_MAJOR \s+ (?P<major>\d+) .*?
+\#define \s+ MI_VERSION_MINOR \s+ (?P<minor>\d+) .*?
+\#define \s+ MI_VERSION_PATCH \s+ (?P<patch>\d+) .*?
+'''
+result = "{major}.{minor}.{patch}"
+
+# Build PyPI compliant README file from original README.md
+[tool.hatch.metadata.hooks.fancy-pypi-readme]
+content-type = "text/markdown"
+
+[[tool.hatch.metadata.hooks.fancy-pypi-readme.fragments]]
+path = "README.md"
+start-after = "## Introduction"
+pattern = "(.*)"

src/python/CMakeLists.txt

@@ -2,6 +2,15 @@
 #   Mitsuba python library
 # ----------------------------------------------------------
 
+if (SKBUILD)
+  # Enable LTO only for release builds targeting PyPI (~5% binary size reduction)
+  set(MI_DIST_FLAGS LTO)
+
+  if (MI_STABLE_ABI)
+    list(APPEND MI_DIST_FLAGS STABLE_ABI)
+  endif()
+endif()
+
 include_directories(
   ${PNG_INCLUDE_DIRS}
   ${PUGIXML_INCLUDE_DIRS}
@@ -20,6 +29,8 @@ foreach (MI_VARIANT ${MI_VARIANTS})
 
   nanobind_add_module(${TARGET_NAME}
       NB_DOMAIN drjit
+      NB_STATIC
+      ${MI_DIST_FLAGS}
       main_v.cpp
       ${CORE_PY_V_SRC}
       ${RENDER_PY_V_SRC}
@@ -48,13 +59,17 @@ endforeach()
 
 nanobind_add_module(mitsuba_ext
   NB_DOMAIN drjit
+  NB_STATIC
+  ${MI_DIST_FLAGS}
   main.cpp
   ${CORE_PY_SRC}
   ${RENDER_PY_SRC}
 )
 
 nanobind_add_module(mitsuba_alias
   NB_DOMAIN drjit
+  NB_STATIC
+  ${MI_DIST_FLAGS}
   alias.cpp
 )
 
@@ -250,6 +265,8 @@ if (NOT (MI_SANITIZE_ADDRESS OR MI_SANITIZE_MEMORY))
     DEPENDS ${MI_STUB_TARGET_DEPS}
   )
 
+  install(FILES ${MI_STUB_TARGET_DEPS} DESTINATION mitsuba)
+
 endif()
 
 

src/python/main_v.cpp

@@ -157,10 +157,6 @@ NB_MODULE(MI_VARIANT_NAME, m) {
     m.attr("is_spectral") = is_spectral_v<Spectrum>;
     m.attr("is_polarized") = is_polarized_v<Spectrum>;
 
-    color_management_static_initialization(dr::is_cuda_v<Float>,
-                                           dr::is_llvm_v<Float>);
-    Scene::static_accel_initialization();
-
     MI_PY_IMPORT(Object);
     MI_PY_IMPORT(Ray);
     MI_PY_IMPORT(BoundingBox);
@@ -245,4 +241,9 @@ NB_MODULE(MI_VARIANT_NAME, m) {
     paths.append(nb::str(mi_dir));
     paths.append(nb::str(mi_py_dir));
     m.attr("__path__") = paths;
+
+
+    color_management_static_initialization(dr::is_cuda_v<Float>,
+                                           dr::is_llvm_v<Float>);
+    Scene::static_accel_initialization();
 }