From 06b3f83b3e7f1b1364973be34f58fac4caf773f3 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 6 Nov 2024 16:54:28 -0500 Subject: [PATCH 1/2] Disallow cuda-python 12.6.1 and 11.8.4 (#17253) Due to a bug in cuda-python we must disallow cuda-python 12.6.1 and 11.8.4. This PR disallows those versions. It also silences new cuda-python deprecation warnings so that our test suite passes. See https://github.com/rapidsai/build-planning/issues/116 for more information. --------- Co-authored-by: James Lamb --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/cudf/meta.yaml | 4 ++-- conda/recipes/pylibcudf/meta.yaml | 4 ++-- dependencies.yaml | 8 ++++---- python/cudf/pyproject.toml | 4 +++- python/cudf_kafka/pyproject.toml | 4 +++- python/cudf_polars/pyproject.toml | 4 +++- python/custreamz/pyproject.toml | 2 ++ python/dask_cudf/pyproject.toml | 2 ++ python/pylibcudf/pyproject.toml | 4 +++- 11 files changed, 26 insertions(+), 14 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 9d9fec97731..ace55a15c09 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -19,7 +19,7 @@ dependencies: - cramjam - cubinlinker - cuda-nvtx=11.8 -- cuda-python>=11.7.1,<12.0a0 +- cuda-python>=11.7.1,<12.0a0,!=11.8.4 - cuda-sanitizer-api=11.8.86 - cuda-version=11.8 - cudatoolkit diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 19e3eafd641..d20db44497e 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -21,7 +21,7 @@ dependencies: - cuda-nvcc - cuda-nvrtc-dev - cuda-nvtx-dev -- cuda-python>=12.0,<13.0a0 +- cuda-python>=12.0,<13.0a0,!=12.6.1 - cuda-sanitizer-api - cuda-version=12.5 - cupy>=12.0.0 diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 2c254415318..6debcb281b1 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -91,7 +91,7 @@ requirements: - cudatoolkit - ptxcompiler >=0.7.0 - cubinlinker # CUDA enhanced compatibility. - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.7.1,<12.0a0,!=11.8.4 {% else %} - cuda-cudart - libcufile # [linux64] @@ -100,7 +100,7 @@ requirements: # TODO: Add nvjitlink here # xref: https://github.com/rapidsai/cudf/issues/12822 - cuda-nvrtc - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.0,<13.0a0,!=12.6.1 - pynvjitlink {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} diff --git a/conda/recipes/pylibcudf/meta.yaml b/conda/recipes/pylibcudf/meta.yaml index 3d965f30986..92ca495f972 100644 --- a/conda/recipes/pylibcudf/meta.yaml +++ b/conda/recipes/pylibcudf/meta.yaml @@ -83,9 +83,9 @@ requirements: - {{ pin_compatible('rmm', max_pin='x.x') }} - fsspec >=0.6.0 {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 + - cuda-python >=11.7.1,<12.0a0,!=11.8.4 {% else %} - - cuda-python >=12.0,<13.0a0 + - cuda-python >=12.0,<13.0a0,!=12.6.1 {% endif %} - nvtx >=0.2.1 - packaging diff --git a/dependencies.yaml b/dependencies.yaml index 90255ca674c..cc31619c217 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -658,10 +658,10 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - cuda-python>=12.0,<13.0a0 + - cuda-python>=12.0,<13.0a0,!=12.6.1 - matrix: {cuda: "11.*"} packages: &run_pylibcudf_packages_all_cu11 - - cuda-python>=11.7.1,<12.0a0 + - cuda-python>=11.7.1,<12.0a0,!=11.8.4 - {matrix: null, packages: *run_pylibcudf_packages_all_cu11} run_cudf: common: @@ -684,10 +684,10 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - cuda-python>=12.0,<13.0a0 + - cuda-python>=12.0,<13.0a0,!=12.6.1 - matrix: {cuda: "11.*"} packages: &run_cudf_packages_all_cu11 - - cuda-python>=11.7.1,<12.0a0 + - cuda-python>=11.7.1,<12.0a0,!=11.8.4 - {matrix: null, packages: *run_cudf_packages_all_cu11} - output_types: conda matrices: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index b6105c17b3e..53f22a11e6b 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -20,7 +20,7 @@ requires-python = ">=3.10" dependencies = [ "cachetools", "cubinlinker", - "cuda-python>=11.7.1,<12.0a0", + "cuda-python>=11.7.1,<12.0a0,!=11.8.4", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "libcudf==24.12.*,>=0.0.0a0", @@ -90,6 +90,8 @@ filterwarnings = [ "error", "ignore:::.*xdist.*", "ignore:::.*pytest.*", + # https://github.com/rapidsai/build-planning/issues/116 + "ignore:.*cuda..* module is deprecated.*:DeprecationWarning", # some third-party dependencies (e.g. 'boto3') still using datetime.datetime.utcnow() "ignore:.*datetime.*utcnow.*scheduled for removal.*:DeprecationWarning:botocore", # Deprecation warning from Pyarrow Table.to_pandas() with pandas-2.2+ diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 667cd7b1db8..ec0bc0eb22b 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -51,7 +51,9 @@ rapids = ["rmm", "cudf", "dask_cudf"] addopts = "--tb=native --strict-config --strict-markers" empty_parameter_set_mark = "fail_at_collect" filterwarnings = [ - "error" + "error", + # https://github.com/rapidsai/build-planning/issues/116 + "ignore:.*cuda..* module is deprecated.*:DeprecationWarning", ] xfail_strict = true diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index a2c62ef9460..2e75dff5c9e 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -53,7 +53,9 @@ version = {file = "cudf_polars/VERSION"} addopts = "--tb=native --strict-config --strict-markers" empty_parameter_set_mark = "fail_at_collect" filterwarnings = [ - "error" + "error", + # https://github.com/rapidsai/build-planning/issues/116 + "ignore:.*cuda..* module is deprecated.*:DeprecationWarning", ] xfail_strict = true diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index a8ab05a3922..d3baf3bf4d2 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -85,6 +85,8 @@ addopts = "--tb=native --strict-config --strict-markers" empty_parameter_set_mark = "fail_at_collect" filterwarnings = [ "error", + # https://github.com/rapidsai/build-planning/issues/116 + "ignore:.*cuda..* module is deprecated.*:DeprecationWarning", "ignore:unclosed =11.7.1,<12.0a0", + "cuda-python>=11.7.1,<12.0a0,!=11.8.4", "libcudf==24.12.*,>=0.0.0a0", "nvtx>=0.2.1", "packaging", @@ -74,6 +74,8 @@ addopts = "--tb=native --strict-config --strict-markers --import-mode=importlib" empty_parameter_set_mark = "fail_at_collect" filterwarnings = [ "error", + # https://github.com/rapidsai/build-planning/issues/116 + "ignore:.*cuda..* module is deprecated.*:DeprecationWarning", "ignore:::.*xdist.*", "ignore:::.*pytest.*" ] From 57900dee500a1a051393dea438d32d94ecd4de61 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Thu, 7 Nov 2024 02:47:47 +0100 Subject: [PATCH 2/2] KvikIO shared library (#17239) Update cudf to use the new KvikIO shared library: https://github.com/rapidsai/kvikio/pull/527 #### Tasks - [x] Wait for the [KvikIO shared library PR](https://github.com/rapidsai/kvikio/pull/527) to be merged. - [x] Revert the use of the [KvikIO shared library](https://github.com/rapidsai/kvikio/pull/527) in CI: https://github.com/rapidsai/cudf/commit/2d8eeafe4959357a17f6ad488811837e0a07ba65. Authors: - Mads R. B. Kristensen (https://github.com/madsbk) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/17239 --- ci/build_wheel_cudf.sh | 1 + ci/build_wheel_libcudf.sh | 1 + ci/build_wheel_pylibcudf.sh | 1 + dependencies.yaml | 1 + python/libcudf/libcudf/load.py | 11 +++++++++++ python/libcudf/pyproject.toml | 1 + 6 files changed, 16 insertions(+) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index fef4416a366..ae4eb0d5c66 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -23,6 +23,7 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" python -m auditwheel repair \ --exclude libcudf.so \ --exclude libnvcomp.so \ + --exclude libkvikio.so \ -w ${package_dir}/final_dist \ ${package_dir}/dist/* diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index b3d6778ea04..aabd3814a24 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -33,6 +33,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" mkdir -p ${package_dir}/final_dist python -m auditwheel repair \ --exclude libnvcomp.so.4 \ + --exclude libkvikio.so \ -w ${package_dir}/final_dist \ ${package_dir}/dist/* diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 839d98846fe..c4a89f20f5f 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -21,6 +21,7 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" python -m auditwheel repair \ --exclude libcudf.so \ --exclude libnvcomp.so \ + --exclude libkvikio.so \ -w ${package_dir}/final_dist \ ${package_dir}/dist/* diff --git a/dependencies.yaml b/dependencies.yaml index cc31619c217..41ac6ce1808 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -177,6 +177,7 @@ files: extras: table: project includes: + - depends_on_libkvikio - depends_on_nvcomp py_build_pylibcudf: output: pyproject diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index ba134710868..bf27ecfa7f5 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -18,6 +18,17 @@ def load_library(): + try: + # libkvikio must be loaded before libcudf because libcudf references its symbols + import libkvikio + + libkvikio.load_library() + except ModuleNotFoundError: + # libcudf's runtime dependency on libkvikio may be satisfied by a natively + # installed library or a conda package, in which case the import will fail and + # we assume the library is discoverable on system paths. + pass + # Dynamically load libcudf.so. Prefer a system library if one is present to # avoid clobbering symbols that other packages might expect, but if no # other library is present use the one in the wheel. diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index c6d9ae56467..62726bb0df4 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -38,6 +38,7 @@ classifiers = [ "Environment :: GPU :: NVIDIA CUDA", ] dependencies = [ + "libkvikio==24.12.*,>=0.0.0a0", "nvidia-nvcomp==4.1.0.6", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.