From 76d4540317fbd47f6a925644ae972304a3a27784 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 07:31:46 -0700 Subject: [PATCH 01/80] add initial check --- .github/pr.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/pr.yaml diff --git a/.github/pr.yaml b/.github/pr.yaml new file mode 100644 index 00000000..1eadadf9 --- /dev/null +++ b/.github/pr.yaml @@ -0,0 +1,23 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +name: pr + +on: + push: + branches: + - "pull-request/[0-9]+" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pr-builder: + needs: + - checks + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.10 + checks: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.10 + with: + enable_check_generated_files: false From 71f31dcd8b2f69c246a1692dce8fba46f65fa1da Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 07:44:14 -0700 Subject: [PATCH 02/80] move pr.yaml --- .github/{ => workflows}/pr.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/pr.yaml (100%) diff --git a/.github/pr.yaml b/.github/workflows/pr.yaml similarity index 100% rename from .github/pr.yaml rename to .github/workflows/pr.yaml From cadc4bcf19bab0db665c6513ce71955084030c3d Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 08:08:02 -0700 Subject: [PATCH 03/80] add empty check_style.sh --- ci/check_style.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ci/check_style.sh diff --git a/ci/check_style.sh b/ci/check_style.sh new file mode 100644 index 00000000..e69de29b From 3f46b2697b2d1e013eccb5550263f75278e8857e Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 08:14:56 -0700 Subject: [PATCH 04/80] basic check_style.sh that does nothing --- ci/check_style.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/check_style.sh b/ci/check_style.sh index e69de29b..0df0cd54 100644 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -0,0 +1,2 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION From 2dbd68fe4f0fbd1f21f6b4361b22cd5615c3de70 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 08:20:20 -0700 Subject: [PATCH 05/80] add +x --- ci/check_style.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/check_style.sh diff --git a/ci/check_style.sh b/ci/check_style.sh old mode 100644 new mode 100755 From be8c20230c5054400c0e138dec9eb9f08a20643b Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 09:05:32 -0700 Subject: [PATCH 06/80] try adding a wheel-build job --- .github/workflows/pr.yaml | 13 +++++++++++++ ci/build_wheel.sh | 2 ++ 2 files changed, 15 insertions(+) create mode 100755 ci/build_wheel.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1eadadf9..f2ef5a5b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -21,3 +21,16 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.10 with: enable_check_generated_files: false + wheel-build: + container: + image: "rapidsai/ci-conda:latest" + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Get PR Info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main + - name: Run build_wheel.sh + run: ci/build_wheel.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100755 index 00000000..0df0cd54 --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,2 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION From 695a1334db725b19ef217c6d43d4342c5e02f691 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 09:10:10 -0700 Subject: [PATCH 07/80] fix spacing --- .github/workflows/pr.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f2ef5a5b..d4fc06a3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,13 +24,13 @@ jobs: wheel-build: container: image: "rapidsai/ci-conda:latest" - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Get PR Info - id: get-pr-info - uses: rapidsai/shared-actions/get-pr-info@main - - name: Run build_wheel.sh - run: ci/build_wheel.sh + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Get PR Info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main + - name: Run build_wheel.sh + run: ci/build_wheel.sh From cea436af46bf7b24d998ed518181cd1bf90c83a7 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 09:12:07 -0700 Subject: [PATCH 08/80] add runs-on --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d4fc06a3..2aba5656 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,6 +22,7 @@ jobs: with: enable_check_generated_files: false wheel-build: + runs-on: ubuntu-latest container: image: "rapidsai/ci-conda:latest" steps: From bc7116fd0e46c3aa0270eacc82a8b40943fa3b90 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 09:30:18 -0700 Subject: [PATCH 09/80] install ctk --- ci/build_wheel.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 0df0cd54..37c263b7 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,2 +1,6 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION + +# install CUDAToolKit version 12.2 +wget https://developer.download.nvidia.com/compute/cuda/12.2.0/local_installers/cuda_12.2.0_535.54.03_linux.run +sh cuda_12.2.0_535.54.03_linux.run --silent --toolkit --override From 351fd7e0817362a74ca290dc57b00fb0750c67fa Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 09:34:52 -0700 Subject: [PATCH 10/80] add -q for now --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 37c263b7..0c4ec6b7 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -2,5 +2,5 @@ # Copyright (c) 2023, NVIDIA CORPORATION # install CUDAToolKit version 12.2 -wget https://developer.download.nvidia.com/compute/cuda/12.2.0/local_installers/cuda_12.2.0_535.54.03_linux.run +wget -q https://developer.download.nvidia.com/compute/cuda/12.2.0/local_installers/cuda_12.2.0_535.54.03_linux.run sh cuda_12.2.0_535.54.03_linux.run --silent --toolkit --override From b18b582026afc79fcc052505a72edd7d10493790 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 09:42:41 -0700 Subject: [PATCH 11/80] switch to apt-get --- ci/build_wheel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 0c4ec6b7..341fbea6 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -2,5 +2,5 @@ # Copyright (c) 2023, NVIDIA CORPORATION # install CUDAToolKit version 12.2 -wget -q https://developer.download.nvidia.com/compute/cuda/12.2.0/local_installers/cuda_12.2.0_535.54.03_linux.run -sh cuda_12.2.0_535.54.03_linux.run --silent --toolkit --override +sudo apt-get update +sudo apt-get install nvidia-cuda-toolkit-12.2 From c3ebe4cff69d4b5ea0bfffe2d1f5c90de9a70a37 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 09:46:17 -0700 Subject: [PATCH 12/80] minor fixes --- ci/build_wheel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 341fbea6..3819d87b 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -2,5 +2,5 @@ # Copyright (c) 2023, NVIDIA CORPORATION # install CUDAToolKit version 12.2 -sudo apt-get update -sudo apt-get install nvidia-cuda-toolkit-12.2 +apt-get update +apt-get install nvidia-cuda-toolkit-12.2 From cb21a8eb73f3427ed6c24f7e4ecba78249ca6a35 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 11:53:21 -0700 Subject: [PATCH 13/80] update package name --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 3819d87b..38dcc5e0 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -3,4 +3,4 @@ # install CUDAToolKit version 12.2 apt-get update -apt-get install nvidia-cuda-toolkit-12.2 +apt-get install cuda-toolkit-12-2 From 7e679763cdba60bbdf290ed446bd5466e73afbc3 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 11:55:44 -0700 Subject: [PATCH 14/80] add install options --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 38dcc5e0..884e6322 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -3,4 +3,4 @@ # install CUDAToolKit version 12.2 apt-get update -apt-get install cuda-toolkit-12-2 +apt-get -y install cuda-toolkit-12-2 From 487341decbef8423e942368ad4d52c4b28d898d6 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 12:09:18 -0700 Subject: [PATCH 15/80] expanding build_wheel.sh --- ci/build_wheel.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 884e6322..de042912 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -4,3 +4,7 @@ # install CUDAToolKit version 12.2 apt-get update apt-get -y install cuda-toolkit-12-2 + +conda list +pip wheel . +ls From e58540ab6b79f244f7ef04213bffdb1937d45c8b Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 12:21:38 -0700 Subject: [PATCH 16/80] add tests and logging --- ci/build_wheel.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index de042912..21863ab4 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,18 @@ apt-get update apt-get -y install cuda-toolkit-12-2 +rapids-logger "Check conda environment" + conda list + +rapids-logger "Building wheel" + pip wheel . -ls + +rapids-logger "Building tests" + +cd test_binary_generation && make + +rapids-logger "Running Tests" + +cd .. && pytest -v From d40ce2ebcdc9fb040b71bf8419d1e06b2b6ca35b Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 12:30:36 -0700 Subject: [PATCH 17/80] more test build steps --- ci/build_wheel.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 21863ab4..d8873c5f 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -13,10 +13,16 @@ rapids-logger "Building wheel" pip wheel . +rapids-logger "Installing wheel" + +pip install pynvjitlink-0.1.0-cp310-cp310-manylinux_2_35_x86_64.whl + rapids-logger "Building tests" cd test_binary_generation && make rapids-logger "Running Tests" -cd .. && pytest -v +cd .. +conda install -y pytest +py.test tests From cefb8f67d3ef7393366415b427d5009d0e11af11 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 31 Oct 2023 12:37:26 -0700 Subject: [PATCH 18/80] fix path --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index d8873c5f..aa6c01bd 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -25,4 +25,4 @@ rapids-logger "Running Tests" cd .. conda install -y pytest -py.test tests +py.test pynvjitlink/tests From 29b319d5648b8432d80f1ad41b5cb16a330db2d4 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Thu, 2 Nov 2023 09:33:17 -0500 Subject: [PATCH 19/80] Apply suggestions from code review Co-authored-by: Bradley Dice --- .github/workflows/pr.yaml | 7 ++++--- ci/build_wheel.sh | 12 ++++++------ ci/check_style.sh | 7 +++++++ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2aba5656..1946fb15 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,16 +15,17 @@ jobs: needs: - checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.10 + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.12 checks: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.10 + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.12 with: enable_check_generated_files: false wheel-build: runs-on: ubuntu-latest container: - image: "rapidsai/ci-conda:latest" + # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. + image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index aa6c01bd..27fd17ae 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,27 +1,27 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION -# install CUDAToolKit version 12.2 +rapids-logger "Install CUDA Toolkit" apt-get update -apt-get -y install cuda-toolkit-12-2 +apt-get -y install cuda-toolkit-12-3 rapids-logger "Check conda environment" conda list -rapids-logger "Building wheel" +rapids-logger "Build wheel" pip wheel . -rapids-logger "Installing wheel" +rapids-logger "Install wheel" pip install pynvjitlink-0.1.0-cp310-cp310-manylinux_2_35_x86_64.whl -rapids-logger "Building tests" +rapids-logger "Build tests" cd test_binary_generation && make -rapids-logger "Running Tests" +rapids-logger "Run tests" cd .. conda install -y pytest diff --git a/ci/check_style.sh b/ci/check_style.sh index 0df0cd54..84822c4b 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -1,2 +1,9 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION + +set -euo pipefail + +pip install pre-commit + +# Run pre-commit checks +pre-commit run --all-files --show-diff-on-failure From b4ef9f658969ada2cb13101e1f721285fba558cb Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Thu, 2 Nov 2023 07:44:12 -0700 Subject: [PATCH 20/80] fix style --- pynvjitlink/tests/test_pynvjitlink.py | 6 +++++- pynvjitlink/tests/test_pynvjitlink_api.py | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pynvjitlink/tests/test_pynvjitlink.py b/pynvjitlink/tests/test_pynvjitlink.py index 4fa53f90..7d41c704 100644 --- a/pynvjitlink/tests/test_pynvjitlink.py +++ b/pynvjitlink/tests/test_pynvjitlink.py @@ -48,6 +48,7 @@ def device_functions_ptx(): def undefined_extern_cubin(): return read_test_file('undefined_extern.cubin') + @pytest.mark.skip def test_create_no_arch_error(): # nvjitlink expects at least the architecture to be specified. @@ -63,12 +64,14 @@ def test_invalid_arch_error(): match='NVJITLINK_ERROR_UNRECOGNIZED_OPTION error'): _nvjitlinklib.create('-arch=sm_XX') + @pytest.mark.skip def test_unrecognized_option_error(): with pytest.raises(RuntimeError, match='NVJITLINK_ERROR_UNRECOGNIZED_OPTION error'): _nvjitlinklib.create('-fictitious_option') + @pytest.mark.skip def test_invalid_option_type_error(): with pytest.raises(TypeError, @@ -108,6 +111,7 @@ def test_add_file(input_file, input_type, request): _nvjitlinklib.add_data(handle, input_type.value, data, filename) _nvjitlinklib.destroy(handle) + @pytest.mark.skip def test_get_error_log(undefined_extern_cubin): handle = _nvjitlinklib.create('-arch=sm_75') @@ -188,7 +192,7 @@ def test_get_linked_ptx_from_lto(device_functions_ltoir): handle = _nvjitlinklib.create('-arch=sm_75', '-lto', '-ptx') _nvjitlinklib.add_data(handle, input_type, data, filename) _nvjitlinklib.complete(handle) - ptx = _nvjitlinklib.get_linked_ptx(handle) + ptx = _nvjitlinklib.get_linked_ptx(handle) # noqa: F841 _nvjitlinklib.destroy(handle) diff --git a/pynvjitlink/tests/test_pynvjitlink_api.py b/pynvjitlink/tests/test_pynvjitlink_api.py index 8c74d814..2c5ed06c 100644 --- a/pynvjitlink/tests/test_pynvjitlink_api.py +++ b/pynvjitlink/tests/test_pynvjitlink_api.py @@ -30,6 +30,7 @@ def undefined_extern_cubin(): with open(fatbin_path, 'rb') as f: return f.read() + @pytest.mark.skip def test_create_no_arch_error(): # nvlink expects at least the architecture to be specified. @@ -37,6 +38,7 @@ def test_create_no_arch_error(): match='NVJITLINK_ERROR_MISSING_ARCH error'): NvJitLinker() + @pytest.mark.skip def test_invalid_arch_error(): # sm_XX is not a valid architecture @@ -44,6 +46,7 @@ def test_invalid_arch_error(): match='NVJITLINK_ERROR_UNRECOGNIZED_OPTION error'): NvJitLinker('-arch=sm_XX') + @pytest.mark.skip def test_invalid_option_type_error(): with pytest.raises(TypeError, @@ -61,6 +64,7 @@ def test_add_cubin(device_functions_cubin): name = 'test_device_functions.cubin' nvjitlinker.add_cubin(device_functions_cubin, name) + @pytest.mark.skip def test_add_incompatible_cubin_arch_error(device_functions_cubin): nvjitlinker = NvJitLinker('-arch=sm_70') @@ -81,6 +85,7 @@ def test_add_fatbin_sm70(device_functions_fatbin): name = 'test_device_functions.fatbin' nvjitlinker.add_fatbin(device_functions_fatbin, name) + @pytest.mark.skip def test_add_incompatible_fatbin_arch_error(device_functions_fatbin): nvjitlinker = NvJitLinker('-arch=sm_80') @@ -89,6 +94,7 @@ def test_add_incompatible_fatbin_arch_error(device_functions_fatbin): match='NVJITLINK_ERROR_INVALID_INPUT error'): nvjitlinker.add_fatbin(device_functions_fatbin, name) + @pytest.mark.skip def test_add_cubin_with_fatbin_error(device_functions_fatbin): nvjitlinker = NvJitLinker('-arch=sm_75') @@ -105,6 +111,7 @@ def test_add_fatbin_with_cubin(device_functions_cubin): name = 'test_device_functions.cubin' nvjitlinker.add_fatbin(device_functions_cubin, name) + @pytest.mark.skip def test_duplicate_symbols_cubin_and_fatbin(device_functions_cubin, device_functions_fatbin): @@ -136,6 +143,7 @@ def test_get_linked_cubin(device_functions_cubin): # Just check we got something that looks like an ELF assert cubin[:4] == b'\x7fELF' + @pytest.mark.skip def test_get_error_log(undefined_extern_cubin): nvjitlinker = NvJitLinker('-arch=sm_75') From a3fef15145e1128f24b8106dc54ec19759f7eadc Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Thu, 2 Nov 2023 07:47:57 -0700 Subject: [PATCH 21/80] add wheel-build to needs --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1946fb15..94a9dd23 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,6 +14,7 @@ jobs: pr-builder: needs: - checks + - wheel-build secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.12 checks: From 5a36b0153a505626fb47bc7e1233e566a7d35fac Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Thu, 2 Nov 2023 07:56:29 -0700 Subject: [PATCH 22/80] apt-get -> yum due to switching to centos --- ci/build_wheel.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 27fd17ae..00729138 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,9 +1,11 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION +set -e + rapids-logger "Install CUDA Toolkit" -apt-get update -apt-get -y install cuda-toolkit-12-3 +yum update +yum install -y cuda-toolkit-12-3 rapids-logger "Check conda environment" From 37ce73391a42cb2b8eecac7fe3ce70bcf32cce6e Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Thu, 2 Nov 2023 09:05:08 -0700 Subject: [PATCH 23/80] fix centos cuda install --- ci/build_wheel.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 00729138..bc21fa58 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,8 @@ set -e rapids-logger "Install CUDA Toolkit" yum update +yum install -y epel-release +yum install -y nvidia-driver-latest-dkms yum install -y cuda-toolkit-12-3 rapids-logger "Check conda environment" From cca7db5982b6c3a4f0ea41914fe08716a4261e04 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Fri, 3 Nov 2023 09:01:39 -0700 Subject: [PATCH 24/80] set c++11 --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 07d0975c..9da2d24a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,11 @@ +set(CMAKE_CXX_STANDARD 11) + cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) project( pynvjitlink VERSION 0.1.0 - LANGUAGES CXX + LANGUAGES CXX CUDA ) find_package(Python COMPONENTS Interpreter Development REQUIRED) From abeae673d359a8704b4aa7567f2a7282b9541125 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Fri, 3 Nov 2023 09:05:07 -0700 Subject: [PATCH 25/80] remove testing phase for now, will test on downstream jobs instead later --- ci/build_wheel.sh | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index bc21fa58..8acee850 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -16,17 +16,3 @@ conda list rapids-logger "Build wheel" pip wheel . - -rapids-logger "Install wheel" - -pip install pynvjitlink-0.1.0-cp310-cp310-manylinux_2_35_x86_64.whl - -rapids-logger "Build tests" - -cd test_binary_generation && make - -rapids-logger "Run tests" - -cd .. -conda install -y pytest -py.test pynvjitlink/tests From 8af9c15e7d36bdcac24c7b149cf63adf163ab206 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Fri, 3 Nov 2023 09:11:08 -0700 Subject: [PATCH 26/80] try adding a matrix --- .github/workflows/pr.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 94a9dd23..4f45a88e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,9 +24,12 @@ jobs: enable_check_generated_files: false wheel-build: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.10, 3.11, 3.12] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. - image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" + image: "rapidsai/ci-conda:cuda12.0.1-centos7-py${{ matrix.python-version }}" steps: - name: Checkout code uses: actions/checkout@v3 From 510db4a6650d7cc77707e67d531d19b22e55b7b8 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:05:23 -0600 Subject: [PATCH 27/80] Update .github/workflows/pr.yaml Co-authored-by: Bradley Dice --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4f45a88e..658842a1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.10, 3.11, 3.12] + python-version: ["3.10", "3.11", "3.12"] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. image: "rapidsai/ci-conda:cuda12.0.1-centos7-py${{ matrix.python-version }}" From 1335b5673e278440f33b743a90835344bd9ef851 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 08:11:51 -0800 Subject: [PATCH 28/80] install python via conda --- .github/workflows/pr.yaml | 4 ++-- ci/build_wheel.sh | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 658842a1..6b9899da 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -29,7 +29,7 @@ jobs: python-version: ["3.10", "3.11", "3.12"] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. - image: "rapidsai/ci-conda:cuda12.0.1-centos7-py${{ matrix.python-version }}" + image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10 steps: - name: Checkout code uses: actions/checkout@v3 @@ -39,4 +39,4 @@ jobs: id: get-pr-info uses: rapidsai/shared-actions/get-pr-info@main - name: Run build_wheel.sh - run: ci/build_wheel.sh + run: ci/build_wheel.sh ${{ matrix.python-version }} diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 8acee850..8c780d57 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -3,6 +3,9 @@ set -e +python_version="${1:-3.10}" +conda install -y python="$python_version" + rapids-logger "Install CUDA Toolkit" yum update yum install -y epel-release From cb5705422eb06f07edb02f31baa8120c0f489a20 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 08:14:13 -0800 Subject: [PATCH 29/80] missed endquote --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6b9899da..1e09bbf7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -29,7 +29,7 @@ jobs: python-version: ["3.10", "3.11", "3.12"] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. - image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10 + image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" steps: - name: Checkout code uses: actions/checkout@v3 From 25bd2b779efe0309aad8ddc763926f1a7f08bc8a Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 09:20:46 -0800 Subject: [PATCH 30/80] try adding test steps in separate jobs --- .github/workflows/pr.yaml | 26 ++++++++++++++++++++++++++ ci/test_wheel.sh | 2 ++ 2 files changed, 28 insertions(+) create mode 100644 ci/test_wheel.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1e09bbf7..fa00873a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -40,3 +40,29 @@ jobs: uses: rapidsai/shared-actions/get-pr-info@main - name: Run build_wheel.sh run: ci/build_wheel.sh ${{ matrix.python-version }} + - name: Upload Python Wheel + uses: actions/upload-artifact@v2 + with: + name: pynvjitlink-0.1.0-cp${{ matrix.python-version | replace(".", "") }}-cp${{ matrix.python-version | replace(".", "") }}-${{ runner.platform }}.whl + path: ./ + wheel-test + needs: + - wheel-build + runs-on: ubuntu-latest + container: + image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Get PR Info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main + - name: Download Python Wheel + uses: actions/download-artifact@v2 + with: + name: pynvjitlink-0.1.0-cp${{ matrix.python-version | replace(".", "") }}-cp${{ matrix.python-version | replace(".", "") }}-${{ runner.platform }}.whl + path: ./ + - name: run test_wheel.sh + run: ci/test_wheel.sh ${{ matrix.python-version }} diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100644 index 00000000..0df0cd54 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,2 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION From 1669c92233f4d1ed72341b8bc9c21c940c83f194 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 09:21:45 -0800 Subject: [PATCH 31/80] fix syntax --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fa00873a..3aedbfe0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -45,7 +45,7 @@ jobs: with: name: pynvjitlink-0.1.0-cp${{ matrix.python-version | replace(".", "") }}-cp${{ matrix.python-version | replace(".", "") }}-${{ runner.platform }}.whl path: ./ - wheel-test + wheel-test: needs: - wheel-build runs-on: ubuntu-latest From 42b0c5b13ccdb8e9c722720ca345bfa670907810 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 09:54:25 -0800 Subject: [PATCH 32/80] switch to * for now --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3aedbfe0..8820f82b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -43,7 +43,7 @@ jobs: - name: Upload Python Wheel uses: actions/upload-artifact@v2 with: - name: pynvjitlink-0.1.0-cp${{ matrix.python-version | replace(".", "") }}-cp${{ matrix.python-version | replace(".", "") }}-${{ runner.platform }}.whl + name: pynvjitlink-*.whl path: ./ wheel-test: needs: @@ -62,7 +62,7 @@ jobs: - name: Download Python Wheel uses: actions/download-artifact@v2 with: - name: pynvjitlink-0.1.0-cp${{ matrix.python-version | replace(".", "") }}-cp${{ matrix.python-version | replace(".", "") }}-${{ runner.platform }}.whl + name: pynvjitlink*.whl path: ./ - name: run test_wheel.sh run: ci/test_wheel.sh ${{ matrix.python-version }} From 40b8d917ac1af202409986c926734a7b37d6cf5a Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 09:55:51 -0800 Subject: [PATCH 33/80] formatting --- .github/workflows/pr.yaml | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8820f82b..8f42f82a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -45,24 +45,24 @@ jobs: with: name: pynvjitlink-*.whl path: ./ - wheel-test: - needs: - - wheel-build - runs-on: ubuntu-latest - container: - image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Get PR Info - id: get-pr-info - uses: rapidsai/shared-actions/get-pr-info@main - - name: Download Python Wheel - uses: actions/download-artifact@v2 - with: - name: pynvjitlink*.whl - path: ./ - - name: run test_wheel.sh - run: ci/test_wheel.sh ${{ matrix.python-version }} + wheel-test: + needs: + - wheel-build + runs-on: ubuntu-latest + container: + image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Get PR Info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main + - name: Download Python Wheel + uses: actions/download-artifact@v2 + with: + name: pynvjitlink*.whl + path: ./ + - name: run test_wheel.sh + run: ci/test_wheel.sh ${{ matrix.python-version }} From f84ba0be338f90c7032f1c62b87dc0d55e9259c5 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 09:58:50 -0800 Subject: [PATCH 34/80] add missing job --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8f42f82a..dde25e9c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,6 +15,7 @@ jobs: needs: - checks - wheel-build + - wheel-test secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.12 checks: From 946509ca495257715e934690e35703fe0c361f73 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 10:18:05 -0800 Subject: [PATCH 35/80] try uploading a directory --- .github/workflows/pr.yaml | 6 ++---- ci/build_wheel.sh | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index dde25e9c..7d6298b5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -44,8 +44,7 @@ jobs: - name: Upload Python Wheel uses: actions/upload-artifact@v2 with: - name: pynvjitlink-*.whl - path: ./ + path: ./wheel-build-${python_version} wheel-test: needs: - wheel-build @@ -63,7 +62,6 @@ jobs: - name: Download Python Wheel uses: actions/download-artifact@v2 with: - name: pynvjitlink*.whl - path: ./ + path: ./wheel-build-${python_version} - name: run test_wheel.sh run: ci/test_wheel.sh ${{ matrix.python-version }} diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 8c780d57..61e84398 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -18,4 +18,5 @@ conda list rapids-logger "Build wheel" -pip wheel . +mkdir -p ./wheel-build-${python_version} +pip wheel . --wheel-dir=./wheel-build-${python_version} From a6e76c75dc3dc0823f345ef324e2b6ef4cf2b1f7 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 10:29:06 -0800 Subject: [PATCH 36/80] update paths --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7d6298b5..0e03d0e0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -44,7 +44,7 @@ jobs: - name: Upload Python Wheel uses: actions/upload-artifact@v2 with: - path: ./wheel-build-${python_version} + path: ./wheel-build-${{ matrix.python-version }} wheel-test: needs: - wheel-build @@ -62,6 +62,6 @@ jobs: - name: Download Python Wheel uses: actions/download-artifact@v2 with: - path: ./wheel-build-${python_version} + path: ./wheel-build-${{ matrix.python-version }} - name: run test_wheel.sh run: ci/test_wheel.sh ${{ matrix.python-version }} From f59a41a56cd62b1ca0c5eaa3c9f3ec0331b53634 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 10:44:49 -0800 Subject: [PATCH 37/80] drop 3.12 for now --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0e03d0e0..cbf3f2de 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11"] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" From 306c1b986561fe057b1124b407e351720fb830d1 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 10:58:37 -0800 Subject: [PATCH 38/80] add matrix to download --- .github/workflows/pr.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index cbf3f2de..0ecd4ea4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -49,6 +49,9 @@ jobs: needs: - wheel-build runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11"] container: image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" steps: From 7ea3af1d92e9d38e229f8c4de543f9e57ba80d20 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 11:05:14 -0800 Subject: [PATCH 39/80] +x to test_wheel.sh --- ci/test_wheel.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/test_wheel.sh diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh old mode 100644 new mode 100755 From 1b2ce4d60b3821a0a44f4f38ee9b3e79d8717621 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 11:25:51 -0800 Subject: [PATCH 40/80] expand test_wheel.sh --- ci/test_wheel.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 0df0cd54..5ccc28ac 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -1,2 +1,31 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION + +set -e + +rapids-logger "Install CUDA Toolkit" +yum update +yum install -y epel-release +yum install -y nvidia-driver-latest-dkms +yum install -y cuda-toolkit-12-3 + + +rapids-logger "Install Python" + +python_version="${1:-3.10}" +conda install -y python="$python_version" + +rapids-logger "Check conda environment" + +conda list + +rapids-logger "Install wheel" + +pip install ./wheel-build-${python_version}/*.whl + +rapids-logger "Build Tests" +cd test_binary_generation && make + +rapids-logger "Run Tests" +conda install -y pytest +py.test tests From 02906bbc19c4d5d0edafdee2fa0b98e3069839e3 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Mon, 6 Nov 2023 11:42:19 -0800 Subject: [PATCH 41/80] try reading artifact name --- ci/test_wheel.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 5ccc28ac..3e7f4a71 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -20,8 +20,9 @@ rapids-logger "Check conda environment" conda list rapids-logger "Install wheel" - -pip install ./wheel-build-${python_version}/*.whl +for whl in ./wheel-build-${python_version}/*.whl; do + pip install "$whl" +done rapids-logger "Build Tests" cd test_binary_generation && make From 91ac502d233c3a2a502053c7a46188f8f38feb66 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 8 Nov 2023 07:52:45 -0800 Subject: [PATCH 42/80] a debugging commit --- .github/workflows/pr.yaml | 2 +- ci/test_wheel.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0ecd4ea4..0c575fd3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,6 +65,6 @@ jobs: - name: Download Python Wheel uses: actions/download-artifact@v2 with: - path: ./wheel-build-${{ matrix.python-version }} + path: ./ - name: run test_wheel.sh run: ci/test_wheel.sh ${{ matrix.python-version }} diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 3e7f4a71..c6c8b548 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -19,6 +19,12 @@ rapids-logger "Check conda environment" conda list +# TODO: for debugging +rapids-logger "Check wheel dir" +ls . +ls artifact + + rapids-logger "Install wheel" for whl in ./wheel-build-${python_version}/*.whl; do pip install "$whl" From 08842b133d6a749a6ad43865953064af21907a4b Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 8 Nov 2023 08:18:00 -0800 Subject: [PATCH 43/80] update wheel install command --- ci/test_wheel.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index c6c8b548..649e60f4 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -19,16 +19,8 @@ rapids-logger "Check conda environment" conda list -# TODO: for debugging -rapids-logger "Check wheel dir" -ls . -ls artifact - - rapids-logger "Install wheel" -for whl in ./wheel-build-${python_version}/*.whl; do - pip install "$whl" -done +pip install --find-links ./artifact pynvjitlink rapids-logger "Build Tests" cd test_binary_generation && make From 1dc9460a253b5187809a27d855579cbe4635090f Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 8 Nov 2023 08:44:07 -0800 Subject: [PATCH 44/80] missing .. --- ci/test_wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 649e60f4..d0df10bb 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -26,5 +26,6 @@ rapids-logger "Build Tests" cd test_binary_generation && make rapids-logger "Run Tests" +cd .. conda install -y pytest py.test tests From a70ea01163217523acb18321033d652f5eecf789 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 8 Nov 2023 09:22:49 -0800 Subject: [PATCH 45/80] fix more paths --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index d0df10bb..371dc398 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -28,4 +28,4 @@ cd test_binary_generation && make rapids-logger "Run Tests" cd .. conda install -y pytest -py.test tests +py.test pynvjitlink/tests From a6350375d9c39b2d3ac050699f2d0e0d049f84f3 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 8 Nov 2023 11:38:19 -0800 Subject: [PATCH 46/80] build in verbose mode --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 61e84398..91a338c0 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -19,4 +19,4 @@ conda list rapids-logger "Build wheel" mkdir -p ./wheel-build-${python_version} -pip wheel . --wheel-dir=./wheel-build-${python_version} +pip wheel . --wheel-dir=./wheel-build-${python_version} -vvv From 8dcceef572bd2ede83804f0e52432a2daba3a5c7 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 8 Nov 2023 12:01:36 -0800 Subject: [PATCH 47/80] cmake.verbose=True in pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 692b3cdd..f11ded5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,6 @@ [tool.scikit-build] cmake.minimum-version = "3.26.4" +cmake.verbose = true ninja.make-fallback = true build-dir = "build/{wheel_tag}" From 5690d0afeb699e325e5a0b582904933090812223 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Thu, 9 Nov 2023 07:33:22 -0800 Subject: [PATCH 48/80] require cuda 12.3 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9da2d24a..79e69c99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ find_package(Python COMPONENTS Interpreter Development REQUIRED) Python_add_library(_nvjitlinklib MODULE pynvjitlink/_nvjitlinklib.cpp WITH_SOABI) find_package( - CUDAToolkit 12.0 REQUIRED + CUDAToolkit 12.3 REQUIRED ) target_link_libraries(_nvjitlinklib PRIVATE CUDA::nvJitLink_static CUDA::nvptxcompiler_static) From 667691fb663d6c7dc45c2e0e45ccb165178b3bf7 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Thu, 9 Nov 2023 08:35:05 -0800 Subject: [PATCH 49/80] explicitly install gcc --- ci/build_wheel.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 91a338c0..ffc119e2 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -3,9 +3,14 @@ set -e +rapids-logger "Install Python" + python_version="${1:-3.10}" conda install -y python="$python_version" +rapids-logger "Install GCC/G++" +conda install -y gcc + rapids-logger "Install CUDA Toolkit" yum update yum install -y epel-release From e253487a13d9bb9f360c2e8f54a433cdf1215727 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 14 Nov 2023 07:24:06 -0800 Subject: [PATCH 50/80] switch containers, remove extra gcc install --- .github/workflows/pr.yaml | 2 +- ci/build_wheel.sh | 2 -- ci/test_wheel.sh | 7 ------- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0c575fd3..4329bdd0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -53,7 +53,7 @@ jobs: matrix: python-version: ["3.10", "3.11"] container: - image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" + image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py3.10 steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index ffc119e2..9ef98de3 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -8,8 +8,6 @@ rapids-logger "Install Python" python_version="${1:-3.10}" conda install -y python="$python_version" -rapids-logger "Install GCC/G++" -conda install -y gcc rapids-logger "Install CUDA Toolkit" yum update diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 371dc398..e8ee0c52 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,13 +3,6 @@ set -e -rapids-logger "Install CUDA Toolkit" -yum update -yum install -y epel-release -yum install -y nvidia-driver-latest-dkms -yum install -y cuda-toolkit-12-3 - - rapids-logger "Install Python" python_version="${1:-3.10}" From b0b04100815e68dfd3c1a34b04713ef8a84b52cd Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 14 Nov 2023 07:24:57 -0800 Subject: [PATCH 51/80] endquote --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4329bdd0..c8500c18 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -53,7 +53,7 @@ jobs: matrix: python-version: ["3.10", "3.11"] container: - image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py3.10 + image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py3.10" steps: - name: Checkout code uses: actions/checkout@v3 From e0f71aafea1f50ba0165ccc5554ef81dfa440e8c Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 14 Nov 2023 07:38:01 -0800 Subject: [PATCH 52/80] base image name on python version, dont install python in test container --- .github/workflows/pr.yaml | 4 ++-- ci/test_wheel.sh | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c8500c18..842f1532 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -53,7 +53,7 @@ jobs: matrix: python-version: ["3.10", "3.11"] container: - image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py3.10" + image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py${{ matrix.python-version }}" steps: - name: Checkout code uses: actions/checkout@v3 @@ -67,4 +67,4 @@ jobs: with: path: ./ - name: run test_wheel.sh - run: ci/test_wheel.sh ${{ matrix.python-version }} + run: ci/test_wheel.sh diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index e8ee0c52..d9356730 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,14 +3,6 @@ set -e -rapids-logger "Install Python" - -python_version="${1:-3.10}" -conda install -y python="$python_version" - -rapids-logger "Check conda environment" - -conda list rapids-logger "Install wheel" pip install --find-links ./artifact pynvjitlink From ab57c2358bf1f2ef4196c6d1387c429cb781d760 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 14 Nov 2023 07:52:01 -0800 Subject: [PATCH 53/80] adjust matrix --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 842f1532..62f6a01a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.9", "3.10"] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.9", "3.10"] container: image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py${{ matrix.python-version }}" steps: From e2a2a3beb8d258f1eebda955109c98ba1abe6fb3 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 14 Nov 2023 08:23:55 -0800 Subject: [PATCH 54/80] install pytest using pip --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index d9356730..2aa7490f 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -12,5 +12,5 @@ cd test_binary_generation && make rapids-logger "Run Tests" cd .. -conda install -y pytest +pip install pytest py.test pynvjitlink/tests From 8c318fffaee50ffae9e236bf2b74096582fe0032 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 14 Nov 2023 11:27:58 -0800 Subject: [PATCH 55/80] use gpu runners --- .github/workflows/pr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 62f6a01a..65e2c95f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -48,12 +48,14 @@ jobs: wheel-test: needs: - wheel-build - runs-on: ubuntu-latest + runs-on: linux-amd64-gpu-v100-latest-1 strategy: matrix: python-version: ["3.9", "3.10"] container: image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py${{ matrix.python-version }}" + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: - name: Checkout code uses: actions/checkout@v3 From 8dfd9b94e514c078f999de8a4b5c75d6c1fcb0c2 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 14 Nov 2023 12:56:20 -0800 Subject: [PATCH 56/80] install gcc again --- ci/build_wheel.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9ef98de3..45fe013e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -15,6 +15,9 @@ yum install -y epel-release yum install -y nvidia-driver-latest-dkms yum install -y cuda-toolkit-12-3 +rapids-logger "Install GCC" +conda install -y gcc_linux-64 + rapids-logger "Check conda environment" conda list From d1eac131e0af67a4e9b0832946c5f598280dca68 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 14 Nov 2023 15:13:18 -0800 Subject: [PATCH 57/80] enforce compiler versions in cmake --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79e69c99..09f418be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,6 @@ set(CMAKE_CXX_STANDARD 11) +set(CMAKE_C_COMPILER_VERSION 9.3.0) +set(CMAKE_CXX_COMPILER_VERSION 9.3.0) cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) From 78566e01aa6c976412975c39b8e294ce75ab331a Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Tue, 14 Nov 2023 16:30:07 -0800 Subject: [PATCH 58/80] try installing system compilers --- CMakeLists.txt | 2 -- ci/build_wheel.sh | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 09f418be..79e69c99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,4 @@ set(CMAKE_CXX_STANDARD 11) -set(CMAKE_C_COMPILER_VERSION 9.3.0) -set(CMAKE_CXX_COMPILER_VERSION 9.3.0) cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 45fe013e..bb2f55ed 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -16,10 +16,11 @@ yum install -y nvidia-driver-latest-dkms yum install -y cuda-toolkit-12-3 rapids-logger "Install GCC" -conda install -y gcc_linux-64 +yum install -y centos-release-scl +yum install -y devtoolset-9 +source scl_source enable devtoolset-9 rapids-logger "Check conda environment" - conda list rapids-logger "Build wheel" From effab517daf2dd4af501470bb662915183cac958 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 07:17:26 -0800 Subject: [PATCH 59/80] set and unset e --- ci/build_wheel.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index bb2f55ed..22dcdce4 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -18,7 +18,10 @@ yum install -y cuda-toolkit-12-3 rapids-logger "Install GCC" yum install -y centos-release-scl yum install -y devtoolset-9 + +set +e source scl_source enable devtoolset-9 +set -e rapids-logger "Check conda environment" conda list From b80ad0a6cc862c4c0d2513c68fc1e30809dd9581 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 10:33:43 -0800 Subject: [PATCH 60/80] start expanding test matrix --- .github/workflows/pr.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 65e2c95f..34f488a0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,13 +24,16 @@ jobs: with: enable_check_generated_files: false wheel-build: - runs-on: ubuntu-latest + runs-on: linux-${{ matrix.arch }}-gpu-v100-latest-1 strategy: matrix: python-version: ["3.9", "3.10"] + arch: ["arm64", "amd64"] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -48,12 +51,14 @@ jobs: wheel-test: needs: - wheel-build - runs-on: linux-amd64-gpu-v100-latest-1 + runs-on: linux-${{ matrix.arch }}-gpu-v100-latest-1 strategy: matrix: python-version: ["3.9", "3.10"] + linux-version: ["centos7", "rockylinux8"] + arch: ["arm64", "amd64"] container: - image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py${{ matrix.python-version }}" + image: "rapidsai/ci-wheel:cuda12.0.1-${{ matrix.linux-version }}-py${{ matrix.python-version }}" env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: From ee7841d35162f5012934815f1e039d35477b4fad Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 11:30:37 -0800 Subject: [PATCH 61/80] do not set e for now --- ci/build_wheel.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 22dcdce4..a803e35a 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,8 +1,6 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION -set -e - rapids-logger "Install Python" python_version="${1:-3.10}" @@ -19,9 +17,7 @@ rapids-logger "Install GCC" yum install -y centos-release-scl yum install -y devtoolset-9 -set +e source scl_source enable devtoolset-9 -set -e rapids-logger "Check conda environment" conda list From 774b2c8661b8fe354716903a9f806ca2d033e6c9 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 12:01:05 -0800 Subject: [PATCH 62/80] use SCCACHE_S3_NO_CREDENTIALS=1 --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index a803e35a..d0d71bd7 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -23,6 +23,6 @@ rapids-logger "Check conda environment" conda list rapids-logger "Build wheel" - +export SCCACHE_S3_NO_CREDENTIALS=1 mkdir -p ./wheel-build-${python_version} pip wheel . --wheel-dir=./wheel-build-${python_version} -vvv From 7449fcb79bab353d2275f4d6e54be5f679f2ed1a Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 12:37:12 -0800 Subject: [PATCH 63/80] use ci-wheel containers to get arm images --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 34f488a0..a251a624 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -31,7 +31,7 @@ jobs: arch: ["arm64", "amd64"] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. - image: "rapidsai/ci-conda:cuda12.0.1-centos7-py3.10" + image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py3.10" env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: From e11cc48577169665edc62853e98fd1f38e0b7c37 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 12:49:47 -0800 Subject: [PATCH 64/80] get python from the image instead of conda --- .github/workflows/pr.yaml | 4 ++-- ci/build_wheel.sh | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a251a624..2e84feea 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -31,7 +31,7 @@ jobs: arch: ["arm64", "amd64"] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. - image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py3.10" + image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py${{ matrix.python-version }}" env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: @@ -43,7 +43,7 @@ jobs: id: get-pr-info uses: rapidsai/shared-actions/get-pr-info@main - name: Run build_wheel.sh - run: ci/build_wheel.sh ${{ matrix.python-version }} + run: ci/build_wheel.sh - name: Upload Python Wheel uses: actions/upload-artifact@v2 with: diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index d0d71bd7..0ecc9818 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,11 +1,6 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION -rapids-logger "Install Python" - -python_version="${1:-3.10}" -conda install -y python="$python_version" - rapids-logger "Install CUDA Toolkit" yum update From 7ef7ce749d4743c40749d8278a7474f5823d31f7 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 13:10:13 -0800 Subject: [PATCH 65/80] use inputs.node_type --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2e84feea..deb039df 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,7 +24,7 @@ jobs: with: enable_check_generated_files: false wheel-build: - runs-on: linux-${{ matrix.arch }}-gpu-v100-latest-1 + runs-on: linux-${{ matrix.arch }}-${{ inputs.node_type }}" strategy: matrix: python-version: ["3.9", "3.10"] From ce3bb9808246813a0948d18af1264dec47b39a8f Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 13:20:28 -0800 Subject: [PATCH 66/80] build on cpu runners --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index deb039df..04fda45a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,7 +24,7 @@ jobs: with: enable_check_generated_files: false wheel-build: - runs-on: linux-${{ matrix.arch }}-${{ inputs.node_type }}" + runs-on: linux-${{ matrix.arch }}-cpu4 strategy: matrix: python-version: ["3.9", "3.10"] From 64976817ce7e8120ef9e465fc7f58b7491b020a4 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:27:39 -0600 Subject: [PATCH 67/80] Update .github/workflows/pr.yaml --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 04fda45a..44b01ba2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -31,7 +31,7 @@ jobs: arch: ["arm64", "amd64"] container: # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. - image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py${{ matrix.python-version }}" + image: "rapidsai/ci-wheel:cuda12.0.1-rockylinux8-py${{ matrix.python-version }}" env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: From f340a61049852299a830815c3c4e714380bb6cc6 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 13:45:58 -0800 Subject: [PATCH 68/80] remove python version from path --- .github/workflows/pr.yaml | 2 +- ci/build_wheel.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 44b01ba2..5d0be649 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -47,7 +47,7 @@ jobs: - name: Upload Python Wheel uses: actions/upload-artifact@v2 with: - path: ./wheel-build-${{ matrix.python-version }} + path: ./wheel-build wheel-test: needs: - wheel-build diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 0ecc9818..a5ae38dc 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -19,5 +19,5 @@ conda list rapids-logger "Build wheel" export SCCACHE_S3_NO_CREDENTIALS=1 -mkdir -p ./wheel-build-${python_version} -pip wheel . --wheel-dir=./wheel-build-${python_version} -vvv +mkdir -p ./wheel-build +pip wheel . --wheel-dir=./wheel-build -vvv From 461436256d6748c83c8d1ed5d6b2b93af8de07c5 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 14:15:08 -0800 Subject: [PATCH 69/80] separate testing into arm64 and amd64 --- .github/workflows/pr.yaml | 31 ++++++++++++++++++++++++++++--- ci/test_wheel.sh | 3 +++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5d0be649..c392cdd9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -48,15 +48,40 @@ jobs: uses: actions/upload-artifact@v2 with: path: ./wheel-build - wheel-test: + wheel-test-amd64: needs: - wheel-build - runs-on: linux-${{ matrix.arch }}-gpu-v100-latest-1 + runs-on: linux-amd64-gpu-v100-latest-1 + strategy: + matrix: + python-version: ["3.9", "3.10"] + linux-version: ["centos7", "rockylinux8"] + container: + image: "rapidsai/ci-wheel:cuda12.0.1-${{ matrix.linux-version }}-py${{ matrix.python-version }}" + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Get PR Info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main + - name: Download Python Wheel + uses: actions/download-artifact@v2 + with: + path: ./ + - name: run test_wheel.sh + run: ci/test_wheel.sh + wheel-test-arm64: + needs: + - wheel-build + runs-on: linux-arm64-gpu-a100-latest-1 strategy: matrix: python-version: ["3.9", "3.10"] linux-version: ["centos7", "rockylinux8"] - arch: ["arm64", "amd64"] container: image: "rapidsai/ci-wheel:cuda12.0.1-${{ matrix.linux-version }}-py${{ matrix.python-version }}" env: diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 2aa7490f..2b0e557b 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,6 +3,9 @@ set -e +rapids-logger "Search artficact directory" +ls ./artifact + rapids-logger "Install wheel" pip install --find-links ./artifact pynvjitlink From 6baf691210f43cf56e8ad6ba9ea9ae9078d548f7 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 14:17:18 -0800 Subject: [PATCH 70/80] add to needs section --- .github/workflows/pr.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c392cdd9..1a7d1eab 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,7 +15,8 @@ jobs: needs: - checks - wheel-build - - wheel-test + - wheel-test-amd64 + - wheel-test-arm64 secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.12 checks: From b4db71ad05d553056be2fb479573439a8777de9a Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 14:30:24 -0800 Subject: [PATCH 71/80] use citestwheel test images --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1a7d1eab..f5d87e97 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -58,7 +58,7 @@ jobs: python-version: ["3.9", "3.10"] linux-version: ["centos7", "rockylinux8"] container: - image: "rapidsai/ci-wheel:cuda12.0.1-${{ matrix.linux-version }}-py${{ matrix.python-version }}" + image: "rapidsai/citestwheel:cuda12.0.1-ubuntu20.04-py${{ matrix.python-version }}" env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: @@ -84,7 +84,7 @@ jobs: python-version: ["3.9", "3.10"] linux-version: ["centos7", "rockylinux8"] container: - image: "rapidsai/ci-wheel:cuda12.0.1-${{ matrix.linux-version }}-py${{ matrix.python-version }}" + image: "rapidsai/citestwheel:cuda12.0.1-ubuntu20.04-py${{ matrix.python-version }}" env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: From ca82b03ce2a27278f71d6876958b12daf6d5fb30 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 14:45:26 -0800 Subject: [PATCH 72/80] skip bad test --- pynvjitlink/tests/test_pynvjitlink_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pynvjitlink/tests/test_pynvjitlink_api.py b/pynvjitlink/tests/test_pynvjitlink_api.py index 2c5ed06c..ab7e615d 100644 --- a/pynvjitlink/tests/test_pynvjitlink_api.py +++ b/pynvjitlink/tests/test_pynvjitlink_api.py @@ -104,6 +104,7 @@ def test_add_cubin_with_fatbin_error(device_functions_fatbin): nvjitlinker.add_cubin(device_functions_fatbin, name) +@pytest.mark.skip def test_add_fatbin_with_cubin(device_functions_cubin): # Adding a cubin with add_fatbin seems to work - this may be expected # behaviour. From 5ea590e94e2bc61d8825e5511a39b9c9d4114823 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 15 Nov 2023 17:14:39 -0600 Subject: [PATCH 73/80] Apply suggestions from code review --- .github/workflows/pr.yaml | 2 +- ci/test_wheel.sh | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f5d87e97..f4eadefa 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -73,7 +73,7 @@ jobs: uses: actions/download-artifact@v2 with: path: ./ - - name: run test_wheel.sh + - name: Run test_wheel.sh run: ci/test_wheel.sh wheel-test-arm64: needs: diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 2b0e557b..1b8c530c 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,12 +3,11 @@ set -e -rapids-logger "Search artficact directory" +rapids-logger "Search artifact directory" ls ./artifact - rapids-logger "Install wheel" -pip install --find-links ./artifact pynvjitlink +pip install --find-links ./artifact pynvjitlink rapids-logger "Build Tests" cd test_binary_generation && make @@ -16,4 +15,4 @@ cd test_binary_generation && make rapids-logger "Run Tests" cd .. pip install pytest -py.test pynvjitlink/tests +pytest pynvjitlink/tests From 110af93442cecf2999a092f5c316c38495734d05 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Wed, 15 Nov 2023 18:38:53 -0600 Subject: [PATCH 74/80] Update pynvjitlink/tests/test_pynvjitlink.py Co-authored-by: Bradley Dice --- pynvjitlink/tests/test_pynvjitlink.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynvjitlink/tests/test_pynvjitlink.py b/pynvjitlink/tests/test_pynvjitlink.py index 7d41c704..f08eb4ab 100644 --- a/pynvjitlink/tests/test_pynvjitlink.py +++ b/pynvjitlink/tests/test_pynvjitlink.py @@ -192,7 +192,7 @@ def test_get_linked_ptx_from_lto(device_functions_ltoir): handle = _nvjitlinklib.create('-arch=sm_75', '-lto', '-ptx') _nvjitlinklib.add_data(handle, input_type, data, filename) _nvjitlinklib.complete(handle) - ptx = _nvjitlinklib.get_linked_ptx(handle) # noqa: F841 + _nvjitlinklib.get_linked_ptx(handle) _nvjitlinklib.destroy(handle) From c9f57b43a9a543d167f9702df4c5c7c2ea14ea12 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 16:54:06 -0800 Subject: [PATCH 75/80] build amd and arm in two separate jobs --- .github/workflows/pr.yaml | 44 +++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f4eadefa..277a8a48 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,7 +14,8 @@ jobs: pr-builder: needs: - checks - - wheel-build + - wheel-build-amd64 + - wheel-build-arm64 - wheel-test-amd64 - wheel-test-arm64 secrets: inherit @@ -24,17 +25,34 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.12 with: enable_check_generated_files: false - wheel-build: - runs-on: linux-${{ matrix.arch }}-cpu4 + wheel-build-amd64: + runs-on: linux-amd64-cpu4 + strategy: + matrix: + python-version: ["3.9", "3.10"] + container: + image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py${{ matrix.python-version }}" + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Get PR Info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main + - name: Run build_wheel.sh + run: ci/build_wheel.sh + - name: Upload Python Wheel + uses: actions/upload-artifact@v2 + with: + path: ./wheel-build + wheel-build-arm64: + runs-on: linux-arm64-cpu4 strategy: matrix: python-version: ["3.9", "3.10"] - arch: ["arm64", "amd64"] container: - # Use CentOS 7 for an old glibc version, so the wheel has broad manylinux compatibility. image: "rapidsai/ci-wheel:cuda12.0.1-rockylinux8-py${{ matrix.python-version }}" - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -51,14 +69,14 @@ jobs: path: ./wheel-build wheel-test-amd64: needs: - - wheel-build + - wheel-build-amd64 runs-on: linux-amd64-gpu-v100-latest-1 strategy: matrix: python-version: ["3.9", "3.10"] - linux-version: ["centos7", "rockylinux8"] + linux-version: ["ubuntu20.04", "ubuntu18.04"] container: - image: "rapidsai/citestwheel:cuda12.0.1-ubuntu20.04-py${{ matrix.python-version }}" + image: "rapidsai/citestwheel:cuda12.0.1-${{ matrix.linux-version }}-py${{ matrix.python-version }}" env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: @@ -77,14 +95,14 @@ jobs: run: ci/test_wheel.sh wheel-test-arm64: needs: - - wheel-build + - wheel-build-arm64 runs-on: linux-arm64-gpu-a100-latest-1 strategy: matrix: python-version: ["3.9", "3.10"] - linux-version: ["centos7", "rockylinux8"] + linux-version: ["ubuntu20.04", "ubuntu18.04"] container: - image: "rapidsai/citestwheel:cuda12.0.1-ubuntu20.04-py${{ matrix.python-version }}" + image: "rapidsai/citestwheel:cuda12.0.1-${{ matrix.linux-version }}-py${{ matrix.python-version }}" env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: From 0789176fef90e6724588b0f790bf7b2b1639c9ab Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 17:10:14 -0800 Subject: [PATCH 76/80] don't test ubuntu 18.04 with arm --- .github/workflows/pr.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 277a8a48..9abbcd4b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -100,9 +100,8 @@ jobs: strategy: matrix: python-version: ["3.9", "3.10"] - linux-version: ["ubuntu20.04", "ubuntu18.04"] container: - image: "rapidsai/citestwheel:cuda12.0.1-${{ matrix.linux-version }}-py${{ matrix.python-version }}" + image: "rapidsai/citestwheel:cuda12.0.1-ubuntu20.04-py${{ matrix.python-version }}" env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: From f0168059ba5ef680f103634c2913f9ec66cfa66d Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 18:03:42 -0800 Subject: [PATCH 77/80] address reviews --- CMakeLists.txt | 4 ++-- ci/build_wheel.sh | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79e69c99..4eb27707 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,3 @@ -set(CMAKE_CXX_STANDARD 11) - cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) project( @@ -19,4 +17,6 @@ target_link_libraries(_nvjitlinklib PRIVATE CUDA::nvJitLink_static CUDA::nvptxco target_compile_options(_nvjitlinklib PRIVATE -Werror -Wall) +target_compile_features(_nvjitlinklib PRIVATE cxx_std_11 + install(TARGETS _nvjitlinklib LIBRARY DESTINATION pynvjitlink) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index a5ae38dc..c15c3131 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -8,12 +8,6 @@ yum install -y epel-release yum install -y nvidia-driver-latest-dkms yum install -y cuda-toolkit-12-3 -rapids-logger "Install GCC" -yum install -y centos-release-scl -yum install -y devtoolset-9 - -source scl_source enable devtoolset-9 - rapids-logger "Check conda environment" conda list From 1f647ba7047a18538d82e0138f0d8f3c225c9dac Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 18:21:31 -0800 Subject: [PATCH 78/80] fix cmake --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4eb27707..18d03db2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,6 @@ target_link_libraries(_nvjitlinklib PRIVATE CUDA::nvJitLink_static CUDA::nvptxco target_compile_options(_nvjitlinklib PRIVATE -Werror -Wall) -target_compile_features(_nvjitlinklib PRIVATE cxx_std_11 +target_compile_features(_nvjitlinklib PRIVATE cxx_std_11) install(TARGETS _nvjitlinklib LIBRARY DESTINATION pynvjitlink) From 78b0b4119d7b04c5fb945b620ae323c990a551af Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Wed, 15 Nov 2023 19:54:01 -0800 Subject: [PATCH 79/80] try removing cache again --- ci/build_wheel.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c15c3131..99f56e2c 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -12,6 +12,5 @@ rapids-logger "Check conda environment" conda list rapids-logger "Build wheel" -export SCCACHE_S3_NO_CREDENTIALS=1 mkdir -p ./wheel-build pip wheel . --wheel-dir=./wheel-build -vvv From 7e48e2baa1d5df54c478b8164ed484bf79eff8b0 Mon Sep 17 00:00:00 2001 From: brandon-b-miller Date: Thu, 16 Nov 2023 06:57:30 -0800 Subject: [PATCH 80/80] Revert "try removing cache again" This reverts commit 78b0b4119d7b04c5fb945b620ae323c990a551af. --- ci/build_wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 99f56e2c..c15c3131 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -12,5 +12,6 @@ rapids-logger "Check conda environment" conda list rapids-logger "Build wheel" +export SCCACHE_S3_NO_CREDENTIALS=1 mkdir -p ./wheel-build pip wheel . --wheel-dir=./wheel-build -vvv