From a000df917fb6bdbb7a8e0aac29c63cd36ca6702b Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 6 Jan 2025 14:13:21 -0600 Subject: [PATCH 1/3] [NVPTX] Do not run the NVVMReflect pass as part of the normal pipeline Summary: This pass lowers the `__nvvm_reflect` builtin in the IR. However, this currently runs in the standard optimization pipeline, not just the backend pipeline. This means that if the user creates LLVM-IR without an architecture set, it will always delete the reflect code even if it is intended to be used later. Pushing this into the backend pipeline will ensure that this works as intended, allowing users to conditionally include code depending on which target architecture the user ended up using. This fixes a bug in OpenMP and missing code in `libc`. --- llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 1 - llvm/lib/Target/NVPTX/NVVMReflect.cpp | 8 +++++++- llvm/test/CodeGen/NVPTX/nvvm-reflect-arch.ll | 4 ++-- llvm/test/CodeGen/NVPTX/nvvm-reflect-ocl.ll | 4 ++-- llvm/test/CodeGen/NVPTX/nvvm-reflect-opaque.ll | 6 +++--- llvm/test/CodeGen/NVPTX/nvvm-reflect.ll | 7 ++++--- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp index b3b2880588cc592..f6ec780d963d9a5 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -255,7 +255,6 @@ void NVPTXTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { PB.registerPipelineStartEPCallback( [this](ModulePassManager &PM, OptimizationLevel Level) { FunctionPassManager FPM; - FPM.addPass(NVVMReflectPass(Subtarget.getSmVersion())); // Note: NVVMIntrRangePass was causing numerical discrepancies at one // point, if issues crop up, consider disabling. FPM.addPass(NVVMIntrRangePass()); diff --git a/llvm/lib/Target/NVPTX/NVVMReflect.cpp b/llvm/lib/Target/NVPTX/NVVMReflect.cpp index 56525a1edc76141..a0e897584a9d322 100644 --- a/llvm/lib/Target/NVPTX/NVVMReflect.cpp +++ b/llvm/lib/Target/NVPTX/NVVMReflect.cpp @@ -21,6 +21,7 @@ #include "NVPTX.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/ConstantFolding.h" +#include "llvm/CodeGen/CommandFlags.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" @@ -219,7 +220,12 @@ bool NVVMReflect::runOnFunction(Function &F) { return runNVVMReflect(F, SmVersion); } -NVVMReflectPass::NVVMReflectPass() : NVVMReflectPass(0) {} +NVVMReflectPass::NVVMReflectPass() { + // Get the CPU string from the command line if not provided. + StringRef SM = codegen::getMCPU(); + if (!SM.consume_front("sm_") || SM.consumeInteger(10, SmVersion)) + SmVersion = 0; +} PreservedAnalyses NVVMReflectPass::run(Function &F, FunctionAnalysisManager &AM) { diff --git a/llvm/test/CodeGen/NVPTX/nvvm-reflect-arch.ll b/llvm/test/CodeGen/NVPTX/nvvm-reflect-arch.ll index ac5875c6ab1043a..83cb3cde48de181 100644 --- a/llvm/test/CodeGen/NVPTX/nvvm-reflect-arch.ll +++ b/llvm/test/CodeGen/NVPTX/nvvm-reflect-arch.ll @@ -1,9 +1,9 @@ ; Libdevice in recent CUDA versions relies on __CUDA_ARCH reflecting GPU type. ; Verify that __nvvm_reflect() is replaced with an appropriate value. ; -; RUN: opt %s -S -passes='default' -mtriple=nvptx64 -mcpu=sm_20 \ +; RUN: opt %s -S -passes='nvvm-reflect' -mtriple=nvptx64 -mcpu=sm_20 \ ; RUN: | FileCheck %s --check-prefixes=COMMON,SM20 -; RUN: opt %s -S -passes='default' -mtriple=nvptx64 -mcpu=sm_35 \ +; RUN: opt %s -S -passes='nvvm-reflect' -mtriple=nvptx64 -mcpu=sm_35 \ ; RUN: | FileCheck %s --check-prefixes=COMMON,SM35 @"$str" = private addrspace(1) constant [12 x i8] c"__CUDA_ARCH\00" diff --git a/llvm/test/CodeGen/NVPTX/nvvm-reflect-ocl.ll b/llvm/test/CodeGen/NVPTX/nvvm-reflect-ocl.ll index 9d383218dce86af..bf8d6e2cca3071e 100644 --- a/llvm/test/CodeGen/NVPTX/nvvm-reflect-ocl.ll +++ b/llvm/test/CodeGen/NVPTX/nvvm-reflect-ocl.ll @@ -1,8 +1,8 @@ ; Verify that __nvvm_reflect_ocl() is replaced with an appropriate value ; -; RUN: opt %s -S -passes='default' -mtriple=nvptx64 -mcpu=sm_20 \ +; RUN: opt %s -S -passes='nvvm-reflect' -mtriple=nvptx64 -mcpu=sm_20 \ ; RUN: | FileCheck %s --check-prefixes=COMMON,SM20 -; RUN: opt %s -S -passes='default' -mtriple=nvptx64 -mcpu=sm_35 \ +; RUN: opt %s -S -passes='nvvm-reflect' -mtriple=nvptx64 -mcpu=sm_35 \ ; RUN: | FileCheck %s --check-prefixes=COMMON,SM35 @"$str" = private addrspace(4) constant [12 x i8] c"__CUDA_ARCH\00" diff --git a/llvm/test/CodeGen/NVPTX/nvvm-reflect-opaque.ll b/llvm/test/CodeGen/NVPTX/nvvm-reflect-opaque.ll index 46ab79d9858cadd..19c74df3037028b 100644 --- a/llvm/test/CodeGen/NVPTX/nvvm-reflect-opaque.ll +++ b/llvm/test/CodeGen/NVPTX/nvvm-reflect-opaque.ll @@ -3,12 +3,12 @@ ; RUN: cat %s > %t.noftz ; RUN: echo '!0 = !{i32 4, !"nvvm-reflect-ftz", i32 0}' >> %t.noftz -; RUN: opt %t.noftz -S -mtriple=nvptx-nvidia-cuda -passes='default' \ +; RUN: opt %t.noftz -S -mtriple=nvptx-nvidia-cuda -passes='nvvm-reflect,simplifycfg' \ ; RUN: | FileCheck %s --check-prefix=USE_FTZ_0 --check-prefix=CHECK ; RUN: cat %s > %t.ftz ; RUN: echo '!0 = !{i32 4, !"nvvm-reflect-ftz", i32 1}' >> %t.ftz -; RUN: opt %t.ftz -S -mtriple=nvptx-nvidia-cuda -passes='default' \ +; RUN: opt %t.ftz -S -mtriple=nvptx-nvidia-cuda -passes='nvvm-reflect,simplifycfg' \ ; RUN: | FileCheck %s --check-prefix=USE_FTZ_1 --check-prefix=CHECK @str = private unnamed_addr addrspace(4) constant [11 x i8] c"__CUDA_FTZ\00" @@ -43,7 +43,7 @@ exit: declare i32 @llvm.nvvm.reflect(ptr) -; CHECK-LABEL: define noundef i32 @intrinsic +; CHECK-LABEL: define i32 @intrinsic define i32 @intrinsic() { ; CHECK-NOT: call i32 @llvm.nvvm.reflect ; USE_FTZ_0: ret i32 0 diff --git a/llvm/test/CodeGen/NVPTX/nvvm-reflect.ll b/llvm/test/CodeGen/NVPTX/nvvm-reflect.ll index 2ed9f7c11bcf9b0..244b44fea9b83c7 100644 --- a/llvm/test/CodeGen/NVPTX/nvvm-reflect.ll +++ b/llvm/test/CodeGen/NVPTX/nvvm-reflect.ll @@ -3,12 +3,12 @@ ; RUN: cat %s > %t.noftz ; RUN: echo '!0 = !{i32 4, !"nvvm-reflect-ftz", i32 0}' >> %t.noftz -; RUN: opt %t.noftz -S -mtriple=nvptx-nvidia-cuda -passes='default' \ +; RUN: opt %t.noftz -S -mtriple=nvptx-nvidia-cuda -passes='nvvm-reflect,simplifycfg' \ ; RUN: | FileCheck %s --check-prefix=USE_FTZ_0 --check-prefix=CHECK ; RUN: cat %s > %t.ftz ; RUN: echo '!0 = !{i32 4, !"nvvm-reflect-ftz", i32 1}' >> %t.ftz -; RUN: opt %t.ftz -S -mtriple=nvptx-nvidia-cuda -passes='default' \ +; RUN: opt %t.ftz -S -mtriple=nvptx-nvidia-cuda -passes='nvvm-reflect,simplifycfg' \ ; RUN: | FileCheck %s --check-prefix=USE_FTZ_1 --check-prefix=CHECK @str = private unnamed_addr addrspace(4) constant [11 x i8] c"__CUDA_FTZ\00" @@ -43,7 +43,8 @@ exit: declare i32 @llvm.nvvm.reflect(ptr) -; CHECK-LABEL: define noundef i32 @intrinsic +; CHECK-LABEL: define i32 @intrinsic + define i32 @intrinsic() { ; CHECK-NOT: call i32 @llvm.nvvm.reflect ; USE_FTZ_0: ret i32 0 From 383ae0083daed11ffa7ef72ffaa9e7dafea61342 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 7 Jan 2025 13:47:45 -0600 Subject: [PATCH 2/3] Only when not specified --- llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp | 5 +++-- llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 5 ++++- llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp index 42043adc37b7159..4dc50c26bdd14af 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp @@ -35,9 +35,10 @@ void NVPTXSubtarget::anchor() {} NVPTXSubtarget &NVPTXSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) { // Provide the default CPU if we don't have one. - TargetName = std::string(CPU.empty() ? "sm_30" : CPU); + TargetName = std::string(CPU); - ParseSubtargetFeatures(TargetName, /*TuneCPU*/ TargetName, FS); + ParseSubtargetFeatures(CPU.empty() ? "sm_30" : CPU, + /*TuneCPU=*/CPU.empty() ? "sm_30" : CPU, FS); // Re-map SM version numbers, SmVersion carries the regular SMs which do // have relative order, while FullSmVersion allows distinguishing sm_90 from diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h index 7555a2368ec963f..20c0d53c2ca8200 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h +++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h @@ -111,7 +111,10 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo { // - 0 represents base GPU model, // - non-zero value identifies particular architecture-accelerated variant. bool hasAAFeatures() const { return getFullSmVersion() % 10; } - std::string getTargetName() const { return TargetName; } + std::string getTargetName() const { + return TargetName.empty() ? "sm_30" : TargetName; + } + bool hasTargetName() const { return !TargetName.empty(); } // Get maximum value of required alignments among the supported data types. // From the PTX ISA doc, section 8.2.3: diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp index f6ec780d963d9a5..6d4b82aa54a2b81 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -255,6 +255,10 @@ void NVPTXTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { PB.registerPipelineStartEPCallback( [this](ModulePassManager &PM, OptimizationLevel Level) { FunctionPassManager FPM; + // We do not want to fold out calls to nvvm.reflect early if the user + // has not provided a target architecture just yet. + if (Subtarget.hasTargetName()) + FPM.addPass(NVVMReflectPass(Subtarget.getSmVersion())); // Note: NVVMIntrRangePass was causing numerical discrepancies at one // point, if issues crop up, consider disabling. FPM.addPass(NVVMIntrRangePass()); From a9edd019483c2e97a3e9fc9d2f9074d442c9aa23 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 7 Jan 2025 14:11:03 -0600 Subject: [PATCH 3/3] update --- llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp | 20 +++++++++----------- llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 2 ++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp index 4dc50c26bdd14af..74ce6a9fc4ac081 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp @@ -34,20 +34,18 @@ void NVPTXSubtarget::anchor() {} NVPTXSubtarget &NVPTXSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) { - // Provide the default CPU if we don't have one. - TargetName = std::string(CPU); + TargetName = std::string(CPU); - ParseSubtargetFeatures(CPU.empty() ? "sm_30" : CPU, - /*TuneCPU=*/CPU.empty() ? "sm_30" : CPU, FS); + ParseSubtargetFeatures(getTargetName(), /*TuneCPU=*/getTargetName(), FS); - // Re-map SM version numbers, SmVersion carries the regular SMs which do - // have relative order, while FullSmVersion allows distinguishing sm_90 from - // sm_90a, which would *not* be a subset of sm_91. - SmVersion = getSmVersion(); + // Re-map SM version numbers, SmVersion carries the regular SMs which do + // have relative order, while FullSmVersion allows distinguishing sm_90 from + // sm_90a, which would *not* be a subset of sm_91. + SmVersion = getSmVersion(); - // Set default to PTX 6.0 (CUDA 9.0) - if (PTXVersion == 0) { - PTXVersion = 60; + // Set default to PTX 6.0 (CUDA 9.0) + if (PTXVersion == 0) { + PTXVersion = 60; } return *this; diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h index 20c0d53c2ca8200..bbc1cca7c12d858 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h +++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h @@ -111,6 +111,8 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo { // - 0 represents base GPU model, // - non-zero value identifies particular architecture-accelerated variant. bool hasAAFeatures() const { return getFullSmVersion() % 10; } + + // If the user did not provide a target we default to the `sm_30` target. std::string getTargetName() const { return TargetName.empty() ? "sm_30" : TargetName; }