upgrade build framework

- migrate `scikit-build` => `scikit-build-core` + drop miutil + docs: update developer contrib guide - auto-detect native CUDA arch - CI: bump workflows & pre-commit hooks - dist: use PyPI trusted publishing"
AMYPAD · Dec 9, 2023 · 47673d8 · 47673d8
1 parent f740ad1
commit 47673d8
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 109 deletions.
diff --git a/.github/workflows/comment-bot.yml b/.github/workflows/comment-bot.yml
@@ -6,10 +6,14 @@ jobs:
   tag:  # /tag <tagname> <commit>
     if: startsWith(github.event.comment.body, '/tag ')
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: React Seen
-      uses: actions/github-script@v6
+      uses: actions/github-script@v7
       with:
         script: |
           const perm = await github.rest.repos.getCollaboratorPermissionLevel({
@@ -27,7 +31,7 @@ jobs:
           post({
             owner: context.repo.owner, repo: context.repo.repo,
             comment_id: context.payload.comment.id, content: "eyes"})
-        github-token: ${{ secrets.GH_TOKEN }}
+        github-token: ${{ secrets.GH_TOKEN || github.token }}
     - name: Tag Commit
       run: |
         git clone https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY} repo
@@ -36,9 +40,9 @@ jobs:
         rm -rf repo
       env:
         BODY: ${{ github.event.comment.body }}
-        GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
     - name: React Success
-      uses: actions/github-script@v6
+      uses: actions/github-script@v7
       with:
         script: |
           post = (context.eventName == "issue_comment"
@@ -47,4 +51,4 @@ jobs:
           post({
             owner: context.repo.owner, repo: context.repo.repo,
             comment_id: context.payload.comment.id, content: "rocket"})
-        github-token: ${{ secrets.GH_TOKEN }}
+        github-token: ${{ secrets.GH_TOKEN || github.token }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -9,8 +9,8 @@ jobs:
     if: github.event_name != 'pull_request' || !contains('OWNER,MEMBER,COLLABORATOR', github.event.pull_request.author_association)
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: set PYSHA
@@ -47,26 +47,32 @@ jobs:
       matrix:
         python: [3.7, 3.11]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python }}
-    - run: pip install -U -e .[dev]
-      env:
-        # https://github.com/scikit-build/scikit-build/issues/740
-        SETUPTOOLS_ENABLE_FEATURES: legacy-editable
+    - name: pip install -U -e .[dev]
+      # in-place for pytest (-e . doesn't work yet for scikit-build-core)
+      run: |
+        pip install toml
+        python -c 'import toml; c=toml.load("pyproject.toml")
+        print("\0".join(c["build-system"]["requires"] + ["cmake>=" + c["tool"]["scikit-build"]["cmake"]["minimum-version"]]), end="")' \
+        | xargs -0 pip install ninja
+        pip install --no-build-isolation --no-deps -t . -U -v . -Ccmake.define.CUVEC_DEBUG=1
+        git restore numcu/src
+        python -c 'import toml; c=toml.load("pyproject.toml")
+        print("\0".join(c["project"]["dependencies"] + c["project"]["optional-dependencies"]["dev"]), end="")' \
+        | xargs -0 pip install
     - run: pytest
     - uses: codecov/codecov-action@v3
     - name: compile -Wall
       run: |
         git clean -Xdf
-        pip install toml
-        python -c 'import toml; c=toml.load("pyproject.toml"); print("\0".join(c["build-system"]["requires"]), end="")' \
-          | xargs -0 pip install
-        python setup.py build -- -DCUVEC_DEBUG=1 \
-          -DCMAKE_CXX_FLAGS="-Wall -Wextra -Wpedantic -Werror -Wno-missing-field-initializers -Wno-unused-parameter -Wno-cast-function-type"
+        pip install build
+        python -m build -n -w \
+          -Ccmake.define.CMAKE_CXX_FLAGS="-Wall -Wextra -Wpedantic -Werror -Wno-missing-field-initializers -Wno-unused-parameter -Wno-cast-function-type"
   cuda:
     if: github.event_name != 'pull_request' || !contains('OWNER,MEMBER,COLLABORATOR', github.event.pull_request.author_association)
     name: CUDA py${{ matrix.python }}
@@ -75,46 +81,58 @@ jobs:
       matrix:
         python: [3.7, 3.11]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Run setup-python
       run: setup-python -p${{ matrix.python }}
-    - run: pip install -U -e .[dev]
-      env:
-        # https://github.com/scikit-build/scikit-build/issues/740
-        SETUPTOOLS_ENABLE_FEATURES: legacy-editable
+    - name: pip install -U -e .[dev]
+      # in-place for pytest (-e . doesn't work yet for scikit-build-core)
+      run: |
+        pip install toml
+        python -c 'import toml; c=toml.load("pyproject.toml")
+        print("\0".join(c["build-system"]["requires"] + ["cmake>=" + c["tool"]["scikit-build"]["cmake"]["minimum-version"]]), end="")' \
+        | xargs -0 pip install ninja
+        pip install --no-build-isolation --no-deps -t . -U -v . -Ccmake.define.CUVEC_DEBUG=1
+        git restore numcu/src
+        python -c 'import toml; c=toml.load("pyproject.toml")
+        print("\0".join(c["project"]["dependencies"] + c["project"]["optional-dependencies"]["dev"]), end="")' \
+        | xargs -0 pip install
     - run: pytest
     - uses: codecov/codecov-action@v3
     - name: compile -Wall
       run: |
         git clean -Xdf
-        pip install toml
-        python -c 'import toml; c=toml.load("pyproject.toml"); print("\0".join(c["build-system"]["requires"]), end="")' \
-          | xargs -0 pip install
-        python setup.py build -- -DCUVEC_DEBUG=1 \
-          -DCMAKE_CXX_FLAGS="-Wall -Wextra -Wpedantic -Werror -Wno-missing-field-initializers -Wno-unused-parameter -Wno-cast-function-type"
+        pip install build
+        python -m build -n -w \
+          -Ccmake.define.CMAKE_CXX_FLAGS="-Wall -Wextra -Wpedantic -Werror -Wno-missing-field-initializers -Wno-unused-parameter -Wno-cast-function-type" \
+          -Ccmake.define.CMAKE_CUDA_ARCHITECTURES=all
     - name: Post Run setup-python
       run: setup-python -p${{ matrix.python }} -Dr
       if: ${{ always() }}
   deploy:
     needs: [check, test, cuda]
     name: PyPI Deploy
+    environment: pypi
+    permissions:
+      contents: write
+      id-token: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         token: ${{ secrets.GH_TOKEN || github.token }}
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - id: dist
       uses: casperdcl/deploy-pypi@v2
       with:
         build: -s
-        password: ${{ secrets.PYPI_TOKEN }}
-        upload: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }}
+        upload: false
+    - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+      uses: pypa/gh-action-pypi-publish@release/v1
     - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
       name: Release
       run: |

diff --git a/.gitignore b/.gitignore
@@ -2,18 +2,15 @@
 __pycache__/
 
 # build
-/MANIFEST.in
-/*.egg*/
 *.so
-/numcu/cmake/
 /numcu/_dist_ver.py
-/build/
+/numcu/cmake/
+/numcu-*.dist-info/
 /dist/
-/_skbuild/
-/_cmake_test_compile/
 /docs/build/
 
 # Unit test / coverage reports
 /.coverage*
 /coverage.xml
 /.pytest_cache/
+/.mypy_cache/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@ default_language_version:
   python: python3
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: check-added-large-files
   - id: check-case-conflict
@@ -26,7 +26,7 @@ repos:
     types: [text]
     exclude: ^(.pre-commit-config.yaml|.github/workflows/test.yml)$
 - repo: https://github.com/PyCQA/flake8
-  rev: 5.0.4
+  rev: 6.1.0
   hooks:
   - id: flake8
     args: [-j8]
@@ -39,12 +39,12 @@ repos:
     - flake8-pyproject
     - flake8-string-format
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.3.0
+  rev: v1.7.1
   hooks:
   - id: mypy
     additional_dependencies: [types-setuptools]
 - repo: https://github.com/google/yapf
-  rev: v0.40.0
+  rev: v0.40.2
   hooks:
   - id: yapf
     args: [-i]

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -4,16 +4,21 @@ Install in "development/editable" mode including dev/test dependencies:
 
 ```sh
 git clone https://github.com/AMYPAD/NumCu && cd NumCu
-export SETUPTOOLS_ENABLE_FEATURES=legacy-editable
-pip install -e .[dev]
-```
 
-Alternatively, if `cmake` and a generator (such as `make` or `ninja`) are
-available, then `setup.py build` and `develop` can be explicitly called;
-optionally with extra `cmake` and generator arguments:
+# `pip install -e .[dev]` won't work due to https://github.com/scikit-build/scikit-build-core/issues/114
+# work-around:
+# 1. install dependencies (one-off)
+pip install toml
+python -c 'import toml; c=toml.load("pyproject.toml")
+print("\0".join(c["build-system"]["requires"] + c["project"]["dependencies"] + c["project"]["optional-dependencies"]["dev"]), end="")' \
+| xargs -0 pip install -U ninja cmake
 
-```sh
-python setup.py build develop easy_install numcu[dev] -- -DCUVEC_DEBUG:BOOL=ON -- -j8
+# 2. delete build artefacts, (re)build & install in-place with debug info
+git clean -Xdf
+pip install --no-build-isolation --no-deps -t . -U -v . \
+  -Ccmake.define.CUVEC_DEBUG=1
+  -Ccmake.define.CMAKE_CXX_FLAGS="-Wall -Wextra -Wpedantic -Werror -Wno-missing-field-initializers -Wno-unused-parameter -Wno-cast-function-type"
+git restore numcu/src # undo deletion of sources
 ```
 
 Once installed in development/editable mode, tests may be run using:

diff --git a/numcu/CMakeLists.txt b/numcu/CMakeLists.txt
@@ -1,15 +1,19 @@
-cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
-if("${NUMCU_BUILD_VERSION}" STREQUAL "")
-  set(NUMCU_BUILD_VERSION 0 CACHE STRING "version" FORCE)
+cmake_minimum_required(VERSION 3.24 FATAL_ERROR)
+if(NOT DEFINED SKBUILD_PROJECT_VERSION)
+  set(SKBUILD_PROJECT_VERSION 0 CACHE STRING "version" FORCE)
 endif()
-project(numcu LANGUAGES C CXX VERSION "${NUMCU_BUILD_VERSION}")
+string(REGEX REPLACE [[([0-9]+)\.([0-9]+)\.([0-9]+).*]] [[\1.\2.\3]] SKBUILD_PROJECT_VERSION "${SKBUILD_PROJECT_VERSION}")
+project(numcu LANGUAGES C CXX VERSION "${SKBUILD_PROJECT_VERSION}")
 
 option(NUMCU_CUDA_OPTIONAL "Make CUDA optional rather than forced" ON)
 
 cmake_policy(PUSH)
 cmake_policy(SET CMP0074 NEW)  # <PackageName>_ROOT hints for find_package
+if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+  set(CMAKE_CUDA_ARCHITECTURES native CACHE STRING "CUDA arch" FORCE)
+endif()
 cmake_policy(SET CMP0104 NEW)  # CMAKE_CUDA_ARCHITECTURES
-find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
+find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
 if(NOT NUMCU_CUDA_OPTIONAL)
   find_package(CUDAToolkit REQUIRED)
   enable_language(CUDA)
@@ -27,26 +31,20 @@ else()
   endif()
 endif()
 execute_process(
-  COMMAND "${Python3_EXECUTABLE}" -c "import cuvec; print(cuvec.include_path)"
+  COMMAND "${Python_EXECUTABLE}" -c "import cuvec; print(cuvec.include_path)"
   OUTPUT_VARIABLE CUVEC_INCLUDE_DIRS
   OUTPUT_STRIP_TRAILING_WHITESPACE)
 if("${CUVEC_INCLUDE_DIRS}" STREQUAL "")
   message(WARNING "Could not find cuvec includes")
 else()
   message(STATUS "Found cuvec includes: ${CUVEC_INCLUDE_DIRS}")
 endif()
-if(SKBUILD)
-  find_package(PythonExtensions REQUIRED)
-  set(LIB_TYPE "MODULE")
-else()
-  set(LIB_TYPE "SHARED")
-endif()
 cmake_policy(POP)
 
 add_compile_definitions(PY_SSIZE_T_CLEAN)
 
 message(STATUS "CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}")
-if("${CMAKE_BUILD_TYPE}" STREQUAL "")
+if(NOT DEFINED CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
 endif()
 message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
@@ -57,7 +55,7 @@ if(CUVEC_DEBUG)
 endif(CUVEC_DEBUG)
 message(STATUS "CuVec debugging: ${CUVEC_DEBUG}")
 
-if("${NUMCU_THREADS}" STREQUAL "")
+if(NOT DEFINED NUMCU_THREADS)
   set(NUMCU_THREADS 1024 CACHE STRING
     "Maximum number of CUDA threads per block (should be less than cudaDeviceProp::maxThreadsDim)" FORCE)
 endif()
@@ -69,28 +67,27 @@ install(DIRECTORY "${${CMAKE_PROJECT_NAME}_INCLUDE_DIRS}" DESTINATION ${CMAKE_PR
 # main project
 
 file(GLOB SRC LIST_DIRECTORIES false "src/*.cu")
-include_directories(${Python3_INCLUDE_DIRS})
+include_directories(${Python_INCLUDE_DIRS})
 include_directories(${CUVEC_INCLUDE_DIRS})
 
-add_library(${PROJECT_NAME} SHARED ${SRC})
+if(SKBUILD)
+  python_add_library(${PROJECT_NAME} MODULE WITH_SOABI ${SRC})
+else()
+  add_library(${PROJECT_NAME} SHARED ${SRC})
+endif()
 add_library(AMYPAD::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
 target_include_directories(${PROJECT_NAME} PUBLIC
   "$<BUILD_INTERFACE:${${CMAKE_PROJECT_NAME}_INCLUDE_DIRS}>"
   "$<INSTALL_INTERFACE:${CMAKE_PROJECT_NAME}/include>")
 if(CUDAToolkit_FOUND)
-  target_link_libraries(${PROJECT_NAME} ${Python3_LIBRARIES} CUDA::cudart_static)
+  target_link_libraries(${PROJECT_NAME} PRIVATE CUDA::cudart_static)
 else()
   set_source_files_properties(${SRC} PROPERTIES LANGUAGE CXX)
-  target_link_libraries(${PROJECT_NAME} ${Python3_LIBRARIES})
 endif()
 
-if(SKBUILD)
-  python_extension_module(${PROJECT_NAME})
-endif()
 set_target_properties(${PROJECT_NAME} PROPERTIES
   CXX_STANDARD 11
-  #VERSION ${CMAKE_PROJECT_VERSION} SOVERSION ${CMAKE_PROJECT_VERSION_MAJOR}
-  PREFIX ""  # remove shared lib prefix to make importable
+  VERSION ${CMAKE_PROJECT_VERSION} SOVERSION ${CMAKE_PROJECT_VERSION_MAJOR}
   INTERFACE_${PROJECT_NAME}_MAJOR_VERSION ${CMAKE_PROJECT_VERSION_MAJOR})
 set_property(TARGET ${PROJECT_NAME} APPEND PROPERTY COMPATIBLE_INTERFACE_STRING ${PROJECT_NAME}_MAJOR_VERSION)
 install(TARGETS ${PROJECT_NAME} EXPORT ${PROJECT_NAME}Targets

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,8 +1,15 @@
 [build-system]
-requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4",
-            "cuvec>=2.11.0", "miutil[cuda]>=0.4.0",
-            "scikit-build>=0.11.0", "cmake>=3.18", "ninja"]
-build-backend = "setuptools.build_meta"
+requires = ["setuptools_scm>=7", "scikit-build-core[pyproject]>=0.5", "cuvec>=2.11.0"]
+build-backend = "scikit_build_core.build"
+
+[tool.scikit-build]
+metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
+sdist.include = ["numcu/_dist_ver.py"]
+sdist.exclude = ["docs", "tests", ".*"]
+wheel.exclude = ["numcu/src/**"]
+minimum-version = "0.5"
+cmake.minimum-version = "3.24"
+cmake.source-dir = "numcu"
 
 [tool.setuptools_scm]
 write_to = "numcu/_dist_ver.py"