Build C++ wheel

.github/workflows/pr.yaml

@@ -18,7 +18,8 @@ jobs:
       - conda-python-build
       - conda-python-tests
       - docs-build
-      - wheel-build
+      - wheel-build-cpp
+      - wheel-build-python
       - wheel-tests
       - devcontainer
     secrets: inherit
@@ -62,15 +63,23 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/build_docs.sh"
-  wheel-build:
+  wheel-build-cpp:
     needs: checks
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
       build_type: pull-request
-      script: ci/build_wheel.sh
+      script: ci/build_wheel_cpp.sh
+  wheel-build-python:
+    needs: wheel-build-cpp
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      script: ci/build_wheel_python.sh
   wheel-tests:
-    needs: wheel-build
+    needs: wheel-build-python
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:

.pre-commit-config.yaml

@@ -14,14 +14,14 @@ repos:
       rev: 5.12.0
       hooks:
           - id: isort
-            args: ["--settings-path=python/pyproject.toml"]
+            args: ["--settings-path=python/rmm/pyproject.toml"]
             files: python/.*
             types_or: [python, cython, pyi]
     - repo: https://github.com/ambv/black
       rev: 22.3.0
       hooks:
           - id: black
-            args: ["--config=python/pyproject.toml"]
+            args: ["--config=python/rmm/pyproject.toml"]
     - repo: https://github.com/MarcoGorelli/cython-lint
       rev: v0.15.0
       hooks:

build.sh

@@ -176,5 +176,5 @@ fi
 # Build and install the rmm Python package
 if (( NUMARGS == 0 )) || hasArg rmm; then
     echo "building and installing rmm..."
-    SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" python -m pip install --no-build-isolation --no-deps ${REPODIR}/python
+    SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/rmm
 fi

ci/build_docs.sh

@@ -36,7 +36,7 @@ doxygen Doxyfile
 popd
 
 rapids-logger "Build Python docs"
-pushd python/docs
+pushd python/rmm/docs
 make dirhtml
 make text
 mkdir -p "${RAPIDS_DOCS_DIR}/rmm/"{html,txt}

ci/build_python.sh

@@ -14,7 +14,7 @@ export CMAKE_GENERATOR=Ninja
 rapids-print-env
 
 package_name="rmm"
-package_dir="python"
+package_dir="python/rmm"
 
 version=$(rapids-generate-version)
 commit=$(git rev-parse HEAD)

ci/build_wheel.sh → ci/build_wheel_cpp.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
-package_name="rmm"
-package_dir="python"
+package_name="librmm"
+package_dir="python/librmm"
 
 source rapids-configure-sccache
 source rapids-date-string
@@ -25,15 +25,12 @@ sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFF
 echo "${version}" > VERSION
 sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
 
-if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
-    sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file}
-fi
-
 cd "${package_dir}"
 
 python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
 
-mkdir -p final_dist
-python -m auditwheel repair -w final_dist dist/*
-
-RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist
+if [[ ! -d "/tmp/gha-tools" ]]; then
+    git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools
+    export PATH="/tmp/gha-tools/tools:${PATH}"
+fi
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist

ci/build_wheel_python.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="rmm"
+package_dir="python/rmm"
+
+source rapids-configure-sccache
+source rapids-date-string
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+if [[ ! -d "/tmp/gha-tools" ]]; then
+    git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools
+    export PATH="/tmp/gha-tools/tools:${PATH}"
+fi
+
+RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/librmm_dist
+
+# This is the version of the suffix with a preceding hyphen. It's used
+# everywhere except in the final wheel name.
+PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
+
+pyproject_file="${package_dir}/pyproject.toml"
+
+sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
+
+alpha_spec=''
+if ! rapids-is-release-build; then
+    alpha_spec=',>=0.0.0a0'
+fi
+
+sed -r -i "s/librmm==(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
+if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
+    sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file}
+fi
+
+cd "${package_dir}"
+
+PIP_FIND_LINKS="/tmp/librmm_dist" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+
+mkdir -p final_dist
+python -m auditwheel repair -w final_dist dist/*
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist

ci/run_pytests.sh

@@ -4,6 +4,6 @@
 set -euo pipefail
 
 # Support invoking run_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/rmm/
 
 pytest --cache-clear -v "$@" .

ci/test_wheel.sh

@@ -13,5 +13,5 @@ python -m pip install $(echo ./dist/rmm*.whl)[test]
 if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
     python ./ci/wheel_smoke_test.py
 else
-    python -m pytest ./python/rmm/tests
+    python -m pytest ./python/rmm/rmm/tests
 fi

dependencies.yaml

@@ -7,6 +7,7 @@ files:
       arch: [x86_64]
     includes:
       - build
+      - cython_build
       - checks
       - cuda
       - cuda_version
@@ -37,20 +38,31 @@ files:
       - cuda_version
       - docs
       - py_version
+  py_cpp_build:
+    output: pyproject
+    pyproject_dir: python/librmm
+    extras:
+      table: build-system
+    includes:
+      - build
   py_build:
     output: pyproject
+    pyproject_dir: python/rmm
     extras:
       table: build-system
     includes:
       - build
+      - cython_build
   py_run:
     output: pyproject
+    pyproject_dir: python/rmm
     extras:
       table: project
     includes:
       - run
   py_optional_test:
     output: pyproject
+    pyproject_dir: python/rmm
     extras:
       table: project.optional-dependencies
       key: test
@@ -65,7 +77,6 @@ dependencies:
       - output_types: [conda, requirements, pyproject]
         packages:
           - &cmake_ver cmake>=3.26.4
-          - cython>=3.0.0
           - ninja
       - output_types: conda
         packages:
@@ -106,6 +117,17 @@ dependencies:
               cuda: "12.*"
             packages:
               - cuda-nvcc
+  cython_build:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - cython>=3.0.0
+      # TODO: Currently this is a hack for devcontainers. Need to figure out the best solution.
+      #- output_types: [requirements, pyproject]
+      - output_types: [pyproject]
+        packages:
+          - librmm==24.6.*
+    specific:
       - output_types: [conda, requirements, pyproject]
         matrices:
           - matrix:

python/librmm/CMakeLists.txt

@@ -0,0 +1,28 @@
+# =============================================================================
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under
+# the License.
+# =============================================================================
+
+cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
+
+include(../../rapids_config.cmake)
+
+project(
+  librmm-python
+  VERSION "${RAPIDS_VERSION}"
+  LANGUAGES CXX)
+
+set(BUILD_TESTS OFF)
+set(BUILD_BENCHMARKS OFF)
+set(CUDA_STATIC_RUNTIME ON)
+
+add_subdirectory(../.. rmm-cpp)

python/librmm/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

python/librmm/README.md

@@ -0,0 +1 @@
+../../README.md

python/librmm/librmm/VERSION

@@ -0,0 +1 @@
+../../../VERSION

python/librmm/librmm/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from librmm._version import __git_commit__, __version__

python/librmm/librmm/_version.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.resources
+
+__version__ = (
+    importlib.resources.files("librmm").joinpath("VERSION").read_text().strip()
+)
+__git_commit__ = ""

python/librmm/pyproject.toml

@@ -0,0 +1,61 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[build-system]
+build-backend = "scikit_build_core.build"
+requires = [
+    "cmake>=3.26.4",
+    "ninja",
+    "scikit-build-core[pyproject]>=0.7.0",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+
+[project]
+name = "librmm"
+dynamic = ["version"]
+description = "rmm - RAPIDS Memory Manager"
+readme = { file = "README.md", content-type = "text/markdown" }
+authors = [
+    { name = "NVIDIA Corporation" },
+]
+license = { text = "Apache 2.0" }
+requires-python = ">=3.9"
+classifiers = [
+    "Intended Audience :: Developers",
+    "Topic :: Database",
+    "Topic :: Scientific/Engineering",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: C++",
+    "Environment :: GPU :: NVIDIA CUDA",
+]
+
+[project.urls]
+Homepage = "https://github.com/rapidsai/rmm"
+
+[project.entry-points."cmake.prefix"]
+librmm = "librmm"
+
+[tool.scikit-build]
+build-dir = "build/{wheel_tag}"
+cmake.build-type = "Release"
+cmake.minimum-version = "3.26.4"
+ninja.make-fallback = true
+sdist.reproducible = true
+wheel.packages = ["librmm"]
+wheel.install-dir = "librmm"
+wheel.py-api = "py3"
+
+[tool.scikit-build.metadata.version]
+provider = "scikit_build_core.metadata.regex"
+input = "librmm/VERSION"
+regex = "(?P<value>.*)"