diff --git a/llvm/docs/NVPTXUsage.rst b/llvm/docs/NVPTXUsage.rst index a5a78a2882ee..64dd2b84a176 100644 --- a/llvm/docs/NVPTXUsage.rst +++ b/llvm/docs/NVPTXUsage.rst @@ -939,6 +939,29 @@ including that ``wgmma.mma_async`` instruction is undefined behavior. For more information, refer PTX ISA ``_. +'``llvm.nvvm.griddepcontrol.*``' +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +.. code-block:: llvm + + declare void @llvm.nvvm.griddepcontrol.launch_dependents() + declare void @llvm.nvvm.griddepcontrol.wait() + +Overview: +""""""""" + +The ``griddepcontrol`` intrinsics allows the dependent grids and prerequisite grids as defined by the runtime, to control execution in the following way: + +``griddepcontrol.launch_dependents`` intrinsic signals that the dependents can be scheduled, before the current grid completes. The intrinsic can be invoked by multiple threads in the current CTA and repeated invocations of the intrinsic will have no additional side effects past that of the first invocation. + +``griddepcontrol.wait`` intrinsic causes the executing thread to wait until all prerequisite grids in flight have completed and all the memory operations from the prerequisite grids are performed and made visible to the current grid. + +For more information, refer +`PTX ISA `__. + Other Intrinsics ---------------- diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 00c441920bfa..68c2373a1a45 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -5044,4 +5044,7 @@ def int_nvvm_cp_async_bulk_prefetch_L2 NoCapture>, ReadOnly>, ImmArg>]>; +def int_nvvm_griddepcontrol_launch_dependents: Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; +def int_nvvm_griddepcontrol_wait: Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; + } // let TargetPrefix = "nvvm" diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td index 6198c4aa9b94..56d8b734bf01 100644 --- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td +++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td @@ -7569,4 +7569,16 @@ def INT_NVVM_WGMMA_WAIT_GROUP_SYNC_ALIGNED : NVPTXInst<(outs), (ins i64imm:$n), [(int_nvvm_wgmma_wait_group_sync_aligned timm:$n)]>, Requires<[hasSM90a, hasPTX<80>]>; } // isConvergent = true +def GRIDDEPCONTROL_LAUNCH_DEPENDENTS : + NVPTXInst<(outs), (ins), + "griddepcontrol.launch_dependents;", + [(int_nvvm_griddepcontrol_launch_dependents)]>, + Requires<[hasSM<90>, hasPTX<78>]>; + +def GRIDDEPCONTROL_WAIT : + NVPTXInst<(outs), (ins), + "griddepcontrol.wait;", + [(int_nvvm_griddepcontrol_wait)]>, + Requires<[hasSM<90>, hasPTX<78>]>; + def INT_EXIT : NVPTXInst<(outs), (ins), "exit;", [(int_nvvm_exit)]>; diff --git a/llvm/test/CodeGen/NVPTX/griddepcontrol.ll b/llvm/test/CodeGen/NVPTX/griddepcontrol.ll new file mode 100644 index 000000000000..fe15b3fe4afb --- /dev/null +++ b/llvm/test/CodeGen/NVPTX/griddepcontrol.ll @@ -0,0 +1,17 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mcpu=sm_90 -march=nvptx64 | FileCheck %s +; RUN: %if ptxas-11.8 %{ llc < %s -mcpu=sm_90 -march=nvptx64 | %ptxas-verify %} + +define void @griddepcontrol() { +; CHECK-LABEL: griddepcontrol( +; CHECK: { +; CHECK-EMPTY: +; CHECK-EMPTY: +; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: griddepcontrol.launch_dependents; +; CHECK-NEXT: griddepcontrol.wait; +; CHECK-NEXT: ret; + call void @llvm.nvvm.griddepcontrol.launch.dependents() + call void @llvm.nvvm.griddepcontrol.wait() + ret void +}