From b93af6c53e53159cba70b9353b07b345d2bc8199 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Tue, 27 Dec 2022 15:05:21 +0300 Subject: [PATCH 01/13] Tiny SSA fix --- src/coreclr/jit/ssabuilder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/ssabuilder.cpp b/src/coreclr/jit/ssabuilder.cpp index 0006f690591c..854c4ef02318 100644 --- a/src/coreclr/jit/ssabuilder.cpp +++ b/src/coreclr/jit/ssabuilder.cpp @@ -113,7 +113,7 @@ void Compiler::fgResetForSsa() { for (GenTree* tree : LIR::AsRange(blk)) { - if (tree->IsLocal()) + if (tree->IsLocal() || tree->OperIsLocalAddr()) { tree->AsLclVarCommon()->SetSsaNum(SsaConfig::RESERVED_SSA_NUM); } From d0de4324f9dd73b70b04198f95cd7a88bc535219 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Fri, 30 Dec 2022 18:45:26 +0300 Subject: [PATCH 02/13] Fix zero init for TYP_BLK locals --- src/coreclr/jit/llvmlower.cpp | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/llvmlower.cpp b/src/coreclr/jit/llvmlower.cpp index e79083bd52cc..4143ee4c4df7 100644 --- a/src/coreclr/jit/llvmlower.cpp +++ b/src/coreclr/jit/llvmlower.cpp @@ -138,7 +138,9 @@ void Llvm::lowerLocals() else if (!_compiler->fgVarNeedsExplicitZeroInit(lclNum, /* bbInALoop */ false, /* bbIsReturn*/ false) || varDsc->HasGCPtr()) { - var_types zeroType = (varDsc->TypeGet() == TYP_STRUCT) ? TYP_INT : genActualType(varDsc); + var_types zeroType = + ((varDsc->TypeGet() == TYP_STRUCT) || (varDsc->TypeGet() == TYP_BLK)) ? TYP_INT + : genActualType(varDsc); initializeLocalInProlog(lclNum, _compiler->gtNewZeroConNode(zeroType)); } } @@ -286,8 +288,27 @@ void Llvm::initializeLocalInProlog(unsigned lclNum, GenTree* value) _compiler->fgEnsureFirstBBisScratch(); LIR::Range& firstBlockRange = LIR::AsRange(_compiler->fgFirstBB); - GenTree* store = _compiler->gtNewStoreLclVar(lclNum, value); firstBlockRange.InsertAtEnd(value); + + // TYP_BLK locals have to be handled specially as they can only be referenced indirectly. + // TODO-LLVM: use STORE_LCL_FLD here once enough of upstream is merged. + GenTree* store; + LclVarDsc* varDsc = _compiler->lvaGetDesc(lclNum); + if (varDsc->TypeGet() == TYP_BLK) + { + GenTree* lclAddr = _compiler->gtNewLclVarAddrNode(lclNum); + lclAddr->gtFlags |= GTF_VAR_DEF; + firstBlockRange.InsertAtEnd(lclAddr); + + ClassLayout* layout = _compiler->typGetBlkLayout(varDsc->lvExactSize); + store = new (_compiler, GT_STORE_BLK) GenTreeBlk(GT_STORE_BLK, TYP_STRUCT, lclAddr, value, layout); + store->gtFlags |= (GTF_ASG | GTF_IND_NONFAULTING); + } + else + { + store = _compiler->gtNewStoreLclVar(lclNum, value); + } + firstBlockRange.InsertAtEnd(store); DISPTREERANGE(firstBlockRange, store); From 3565226a66aac0e328ecbe8112ed2f29f214142a Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Sun, 1 Jan 2023 20:03:10 +0300 Subject: [PATCH 03/13] Fix null-checked calls --- src/coreclr/jit/llvmlower.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/coreclr/jit/llvmlower.cpp b/src/coreclr/jit/llvmlower.cpp index 4143ee4c4df7..1fd88086c8d3 100644 --- a/src/coreclr/jit/llvmlower.cpp +++ b/src/coreclr/jit/llvmlower.cpp @@ -901,6 +901,12 @@ void Llvm::failUnsupportedCalls(GenTreeCall* callNode) return; } + if (callNode->NeedsNullCheck()) + { + // We need to insert the null check when lowering args. + failFunctionCompilation(); + } + if (callNode->IsUnmanaged()) { failFunctionCompilation(); From ba2e07b1f3455f776cfb908e6fe36d9345ec9093 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Sun, 1 Jan 2023 22:04:36 +0300 Subject: [PATCH 04/13] Fix test bug --- src/tests/nativeaot/SmokeTests/HelloWasm/HelloWasm.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/nativeaot/SmokeTests/HelloWasm/HelloWasm.cs b/src/tests/nativeaot/SmokeTests/HelloWasm/HelloWasm.cs index a9d6dea6804e..0e03c97cffb8 100644 --- a/src/tests/nativeaot/SmokeTests/HelloWasm/HelloWasm.cs +++ b/src/tests/nativeaot/SmokeTests/HelloWasm/HelloWasm.cs @@ -3137,7 +3137,7 @@ private static void TestI8ConvOvf() } try { - byte[] bytes = new byte[1]; + byte[] bytes = new byte[2]; var b = bytes[i]; } catch (OverflowException) From 7887c99a573e45117767343a4663ee81b2cb8a67 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Mon, 19 Dec 2022 20:38:14 +0300 Subject: [PATCH 05/13] Pass the original shadow stack pointer to funclets Consider the first pass of exception handling. The dispatcher will call filter funclets, which can invoke arbitrary managed code. This means filters can invalidate all of the shadow state below them, even as that state is still live (recall we're only in the first pass). Consider the second pass of exception handling. The dispatcher is sitting, in the shadow call stack, below the finally and fault handlers, therefore, they can overwrite the shadow state of the dispatcher itself. Consider the exception object. It must live on the shadow stack, and is a part of the dispatcher's shadow state (allocated below the "throw" point). Therefore, it is not sufficient to simply ensure the dispatcher itself holds no live shadow state between the calls to funclets to solve the problem above. What is sufficient is using a dedicated shadow stack pointer to access the original shadow frame in funclets. It is how the other backends operate as well. This also solves the issue of passing the exception object to funclets nicely and naturally: it is allocated and passed as the first shadow parameter. This solution has the drawback of not "being optimal" for the case of locally invoked finallys. We presume that to be fine, as do the other backends. --- src/coreclr/jit/llvm.h | 10 ++++--- src/coreclr/jit/llvmcodegen.cpp | 48 ++++++++++++++++++++++++++------- src/coreclr/jit/llvmlower.cpp | 43 ++++++++++++++++++++--------- 3 files changed, 76 insertions(+), 25 deletions(-) diff --git a/src/coreclr/jit/llvm.h b/src/coreclr/jit/llvm.h index 08a1f60ef33d..eff9590d03d2 100644 --- a/src/coreclr/jit/llvm.h +++ b/src/coreclr/jit/llvm.h @@ -144,6 +144,7 @@ class Llvm JitHashTable _debugMetadataMap; unsigned _shadowStackLocalsSize; + unsigned _originalShadowStackLclNum = BAD_VAR_NUM; unsigned _shadowStackLclNum = BAD_VAR_NUM; unsigned _retAddressLclNum = BAD_VAR_NUM; unsigned _llvmArgCount; @@ -250,9 +251,10 @@ class Llvm GenTree* createStoreNode(var_types nodeType, GenTree* addr, GenTree* data); GenTree* createShadowStackStoreNode(var_types storeType, GenTree* addr, GenTree* data); - GenTree* insertShadowStackAddr(GenTree* insertBefore, ssize_t offset); + GenTree* insertShadowStackAddr(GenTree* insertBefore, ssize_t offset, unsigned shadowStackLclNum); bool isShadowFrameLocal(LclVarDsc* varDsc) const; + bool isFuncletParameter(unsigned lclNum) const; // ================================================================================================================ // | Codegen | @@ -267,6 +269,7 @@ class Llvm bool initializeFunctions(); void generateProlog(); void initializeLocals(); + void generateFuncletProlog(unsigned funcIdx); void generateBlock(BasicBlock* block); void generateEHDispatch(); void fillPhis(); @@ -334,6 +337,7 @@ class Llvm Value* gepOrAddr(Value* addr, unsigned offset); Value* getShadowStack(); Value* getShadowStackForCallee(); + Value* getOriginalShadowStack(); DebugMetadata getOrCreateDebugMetadata(const char* documentFileName); llvm::DILocation* createDebugFunctionAndDiLocation(struct DebugMetadata debugMetadata, unsigned int lineNo); @@ -350,8 +354,8 @@ class Llvm llvm::BasicBlock* getLastLlvmBlockForBlock(BasicBlock* block); void setLastLlvmBlockForBlock(BasicBlock* block, llvm::BasicBlock* llvmBlock); - AllocaInst* getLocalAddr(unsigned lclNum); - unsigned int getTotalLocalOffset(); + Value* getLocalAddr(unsigned lclNum); + unsigned getTotalLocalOffset(); }; diff --git a/src/coreclr/jit/llvmcodegen.cpp b/src/coreclr/jit/llvmcodegen.cpp index 68a29abef496..497552f3b191 100644 --- a/src/coreclr/jit/llvmcodegen.cpp +++ b/src/coreclr/jit/llvmcodegen.cpp @@ -143,9 +143,10 @@ bool Llvm::initializeFunctions() retLlvmType = Type::getVoidTy(_llvmContext); } - // All funclets have only one argument: the shadow stack. - FunctionType* llvmFuncType = FunctionType::get(retLlvmType, {Type::getInt8PtrTy(_llvmContext)}, - /* isVarArg */ false); + // All funclets two arguments: original and actual shadow stacks. + Type* ptrLlvmType = Type::getInt8PtrTy(_llvmContext); + FunctionType* llvmFuncType = FunctionType::get(retLlvmType, {ptrLlvmType, ptrLlvmType}, /* isVarArg */ false); + const char* kindName; switch (ehDsc->ebdHandlerType) { @@ -242,6 +243,12 @@ void Llvm::initializeLocals() continue; } + if (lclNum == _originalShadowStackLclNum) + { + // We model funclet parameters specially because it is not trivial to represent them in IR faithfully. + continue; + } + // See "genCheckUseBlockInit", "fgInterBlockLocalVarLiveness" and "SsaBuilder::RenameVariables" as references // for the zero-init logic. // @@ -327,6 +334,11 @@ void Llvm::initializeLocals() } } +void Llvm::generateFuncletProlog(unsigned funcIdx) +{ + // TODO-LLVM-EH: create allocas for all tracked locals that are not in SSA or the shadow stack. +} + void Llvm::generateBlock(BasicBlock* block) { JITDUMP("\n=============== Generating "); @@ -836,20 +848,24 @@ void Llvm::buildLocalVar(GenTreeLclVar* lclVar) unsigned int ssaNum = lclVar->GetSsaNum(); LclVarDsc* varDsc = _compiler->lvaGetDesc(lclVar); + // We model funclet parameters specially - it is simpler then representing them faithfully in IR. if (lclNum == _shadowStackLclNum) { - // The shadow stack must be handled specially as the only local used directly by funclets. assert((ssaNum == SsaConfig::FIRST_SSA_NUM) || (ssaNum == SsaConfig::RESERVED_SSA_NUM)); llvmRef = getShadowStack(); } + else if (lclNum == _originalShadowStackLclNum) + { + assert((ssaNum == SsaConfig::FIRST_SSA_NUM) || (ssaNum == SsaConfig::RESERVED_SSA_NUM)); + llvmRef = getOriginalShadowStack(); + } else if (lclVar->HasSsaName()) { llvmRef = _localsMap[{lclNum, ssaNum}]; } else { - AllocaInst* allocInstr = getLocalAddr(lclNum); - llvmRef = _builder.CreateLoad(allocInstr->getAllocatedType(), allocInstr); + llvmRef = _builder.CreateLoad(getLlvmTypeForLclVar(varDsc), getLocalAddr(lclNum)); } // Implicit truncating from long to int. @@ -1765,8 +1781,10 @@ void Llvm::buildReturn(GenTree* node) void Llvm::buildCatchArg(GenTree* node) { - // TODO-LLVM-EH: actually produce the exception. - Value* excObjValue = llvm::Constant::getNullValue(Type::getInt8PtrTy(_llvmContext)); + // The exception object is passed as the first shadow parameter to funclets. + Type* excLlvmType = getLlvmTypeForVarType(node->TypeGet()); + Value* shadowStackValue = _builder.CreateBitCast(getShadowStack(), excLlvmType->getPointerTo()); + Value* excObjValue = _builder.CreateLoad(excLlvmType, shadowStackValue); mapGenTreeToValue(node, excObjValue); } @@ -1845,7 +1863,10 @@ void Llvm::buildCallFinally(BasicBlock* block) // Other backends will simply skip generating the second block, while we will branch to it. // Function* finallyLlvmFunc = getLlvmFunctionForIndex(getLlvmFunctionIndexForBlock(block->bbJumpDest)); - emitCallOrInvoke(finallyLlvmFunc, {getShadowStack()}); + Value* shadowStackValue = getShadowStack(); + + // For a locally invoked finally, both the original and actual shadow stacks have the same value. + emitCallOrInvoke(finallyLlvmFunc, {shadowStackValue, shadowStackValue}); if ((block->bbFlags & BBF_RETLESS_CALL) != 0) { @@ -2232,6 +2253,13 @@ Value* Llvm::getShadowStackForCallee() return gepOrAddr(getShadowStack(), getTotalLocalOffset()); } +Value* Llvm::getOriginalShadowStack() +{ + // The original shadow stack pointer is the second funclet parameter. + assert(getCurrentLlvmFunction() != getRootLlvmFunction()); + return getCurrentLlvmFunction()->getArg(1); +} + DebugMetadata Llvm::getOrCreateDebugMetadata(const char* documentFileName) { std::string fullPath = documentFileName; @@ -2410,7 +2438,7 @@ void Llvm::setLastLlvmBlockForBlock(BasicBlock* block, llvm::BasicBlock* llvmBlo _blkToLlvmBlksMap[block].LastBlock = llvmBlock; } -AllocaInst* Llvm::getLocalAddr(unsigned lclNum) +Value* Llvm::getLocalAddr(unsigned lclNum) { AllocaInst* addrValue = m_allocas[lclNum]; assert(addrValue != nullptr); diff --git a/src/coreclr/jit/llvmlower.cpp b/src/coreclr/jit/llvmlower.cpp index 1fd88086c8d3..f350f2d8ed66 100644 --- a/src/coreclr/jit/llvmlower.cpp +++ b/src/coreclr/jit/llvmlower.cpp @@ -76,19 +76,17 @@ void Llvm::lowerLocals() } } - // With optimizations off, we haven't run liveness (yet), so don't know which locals (if any) are live-in/out - // of handlers and have to assume the worst. TODO-LLVM: move shadow stack lowering after liveness / SSA and - // fix this. - if (!_compiler->fgLocalVarLivenessDone && _compiler->ehAnyFunclets()) + // We don't know if untracked locals are live-in/out of handlers and have to assume the worst. + if (!varDsc->lvTracked && _compiler->ehAnyFunclets()) { varDsc->lvLiveInOutOfHndlr = 1; } // GC locals needs to go on the shadow stack for the scan to find them. Locals live-in/out of handlers // need to be preserved after the native unwind for the funclets to be callable, thus, they too need to - // go on the shadow stack (except for the shadow stack itself as it will be passed to funclets directly). + // go on the shadow stack (except for parameters to funclets, naturally). // - if (varDsc->HasGCPtr() || (varDsc->lvLiveInOutOfHndlr && (lclNum != _shadowStackLclNum))) + if (!isFuncletParameter(lclNum) && (varDsc->HasGCPtr() || varDsc->lvLiveInOutOfHndlr)) { if (_compiler->lvaGetPromotionType(varDsc) == Compiler::PROMOTION_TYPE_INDEPENDENT) { @@ -163,6 +161,14 @@ void Llvm::populateLlvmArgNums() failFunctionCompilation(); } + if (_compiler->ehAnyFunclets()) + { + _originalShadowStackLclNum = _compiler->lvaGrabTemp(true DEBUGARG("original shadowstack")); + LclVarDsc* originalShadowStackVarDsc = _compiler->lvaGetDesc(_originalShadowStackLclNum); + originalShadowStackVarDsc->lvType = TYP_I_IMPL; + originalShadowStackVarDsc->lvCorInfoType = CORINFO_TYPE_PTR; + } + _shadowStackLclNum = _compiler->lvaGrabTemp(true DEBUGARG("shadowstack")); LclVarDsc* shadowStackVarDsc = _compiler->lvaGetDesc(_shadowStackLclNum); unsigned nextLlvmArgNum = 0; @@ -481,7 +487,13 @@ void Llvm::ConvertShadowStackLocalNode(GenTreeLclVarCommon* node) if (isShadowFrameLocal(varDsc) && (lclVar->GetRegNum() == REG_NA)) { - GenTree* lclAddress = insertShadowStackAddr(node, varDsc->GetStackOffset() + node->GetLclOffs()); + // Funclets (especially filters) will be called by the dispatcher while live state still exists + // on shadow frames below (in the tradional sense, where stacks grow down) them. For this reason, + // funclets will access state from the original frame via a dedicated shadow stack pointer, and + // use the actual shadow stack for calls. + unsigned shadowStackLclNum = CurrentBlock()->hasHndIndex() ? _originalShadowStackLclNum : _shadowStackLclNum; + GenTree* lclAddress = + insertShadowStackAddr(node, varDsc->GetStackOffset() + node->GetLclOffs(), shadowStackLclNum); genTreeOps indirOper; GenTree* storedValue = nullptr; @@ -734,7 +746,7 @@ void Llvm::lowerCallToShadowStack(GenTreeCall* callNode) GenTreeCall::Use* lastArg = nullptr; if (callHasShadowStackArg(callNode)) { - GenTree* calleeShadowStack = insertShadowStackAddr(callNode, _shadowStackLocalsSize); + GenTree* calleeShadowStack = insertShadowStackAddr(callNode, _shadowStackLocalsSize, _shadowStackLclNum); GenTreePutArgType* calleeShadowStackPutArg = _compiler->gtNewPutArgType(calleeShadowStack, CORINFO_TYPE_PTR, NO_CLASS_HANDLE); @@ -839,7 +851,7 @@ void Llvm::lowerCallToShadowStack(GenTreeCall* callNode) assert(use.GetType() != TYP_STRUCT); unsigned fieldOffsetValue = _shadowStackLocalsSize + shadowStackUseOffest + use.GetOffset(); - GenTree* fieldSlotAddr = insertShadowStackAddr(callNode, fieldOffsetValue); + GenTree* fieldSlotAddr = insertShadowStackAddr(callNode, fieldOffsetValue, _shadowStackLclNum); GenTree* fieldStoreNode = createShadowStackStoreNode(use.GetType(), fieldSlotAddr, use.GetNode()); CurrentRange().InsertBefore(callNode, fieldStoreNode); @@ -850,7 +862,7 @@ void Llvm::lowerCallToShadowStack(GenTreeCall* callNode) else { unsigned offsetValue = _shadowStackLocalsSize + shadowStackUseOffest; - GenTree* slotAddr = insertShadowStackAddr(callNode, offsetValue); + GenTree* slotAddr = insertShadowStackAddr(callNode, offsetValue, _shadowStackLclNum); GenTree* storeNode = createShadowStackStoreNode(argNode->TypeGet(), slotAddr, argNode); CurrentRange().InsertBefore(callNode, storeNode); @@ -1135,9 +1147,11 @@ GenTree* Llvm::createShadowStackStoreNode(var_types storeType, GenTree* addr, Ge return storeNode; } -GenTree* Llvm::insertShadowStackAddr(GenTree* insertBefore, ssize_t offset) +GenTree* Llvm::insertShadowStackAddr(GenTree* insertBefore, ssize_t offset, unsigned shadowStackLclNum) { - GenTree* shadowStackLcl = _compiler->gtNewLclvNode(_shadowStackLclNum, TYP_I_IMPL); + assert((shadowStackLclNum == _shadowStackLclNum) || (shadowStackLclNum == _originalShadowStackLclNum)); + + GenTree* shadowStackLcl = _compiler->gtNewLclvNode(shadowStackLclNum, TYP_I_IMPL); CurrentRange().InsertBefore(insertBefore, shadowStackLcl); if (offset == 0) @@ -1171,3 +1185,8 @@ bool Llvm::isShadowFrameLocal(LclVarDsc* varDsc) const // a great fit because we add new locals after shadow frame layout. return varDsc->GetRegNum() == REG_STK; } + +bool Llvm::isFuncletParameter(unsigned lclNum) const +{ + return (lclNum == _shadowStackLclNum) || (lclNum == _originalShadowStackLclNum); +} From ad87559cedb11a4d1a6f0a729ac62f9d300a4d3c Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Tue, 27 Dec 2022 15:02:58 +0300 Subject: [PATCH 06/13] Add temporary logging to RhpThrowEx --- src/coreclr/nativeaot/Bootstrap/main.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreclr/nativeaot/Bootstrap/main.cpp b/src/coreclr/nativeaot/Bootstrap/main.cpp index 142c6dcd3afa..dbe7c5f0fd35 100644 --- a/src/coreclr/nativeaot/Bootstrap/main.cpp +++ b/src/coreclr/nativeaot/Bootstrap/main.cpp @@ -5,6 +5,7 @@ #if defined HOST_WASM #include +#include #endif // HOST_WASM // @@ -167,6 +168,7 @@ class ManagedExceptionWrapper : std::exception extern "C" void RhpThrowEx(void * pEx) { + printf("WASM EH: exception thrown\n"); throw ManagedExceptionWrapper(pEx); } From ef063abe45ea824661702490aa060a562e8dac4a Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Wed, 11 Jan 2023 01:41:01 +0300 Subject: [PATCH 07/13] Add RyuJit-specific dispatch functions --- .../System/Runtime/ExceptionHandling.wasm.cs | 373 ++++++++++++++++-- .../src/System/Runtime/InternalCalls.cs | 4 + .../System/Runtime/StackFrameIterator.wasm.cs | 95 ++++- .../src/System.Private.CoreLib.csproj | 1 + .../src/System/Exception.CoreRT.LLVM.cs | 2 +- .../IL/ILImporter.Scanner.cs | 42 +- 6 files changed, 464 insertions(+), 53 deletions(-) diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs index d5a7f025ff2d..ecfb2f9cf53f 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs @@ -2,19 +2,178 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using System.Runtime; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.CompilerServices; using Internal.Runtime; -// Disable: Filter expression is a constant. We know. We just can't do an unfiltered catch. -#pragma warning disable 7095 - namespace System.Runtime { internal static unsafe partial class EH { + // The layout of this struct must match the native version in "Bootstrapper/main.cpp" exactly. + [StructLayout(LayoutKind.Explicit)] + private struct ManagedExceptionWrapper + { + // TODO-LLVM: update the field to be typed "object" once C#11 is available. + // TODO-LLVM-EH: the way managed exception object is being passed inside the native exception is a GC hole. + // Make this more robust, e. g. wrap it in a GCHandle. + // TODO-LLVM: make the offset into a constant generated by the runtime build (ala AsmOffsets.cs). + [FieldOffset(4)] + public void* ManagedException; + } + + // The layout of this struct must match what codegen expects (see "jit/llvmcodegen.cpp, generateEHDispatch"). + // Instances of it are shared between dispatchers across a single native frame. + private struct DispatchData + { + public readonly CppExceptionTuple CppExceptionTuple; // Owned by codegen. + public void* DispatcherData; // Owned by runtime. + + // We consider a dispatch to be "active" if it has already visited nested handlers in the same LLVM frame. + // Codegen will initialize "DispatcherData" to null on entry to the native handler. + public bool Active => DispatcherData != null; + } + + private struct CppExceptionTuple + { + public void* ExceptionData; + public int Selector; + } + + // These per-clause handlers are invoked by RyuJit-generated LLVM code. TODO-LLVM: once we move to opaque + // pointers, we can change the signatures of these functions to use precise pointer types instead of "void*"s. + // + private static int HandleExceptionWasmMutuallyProtectingCatches(void* pShadowFrame, void* pDispatchData, void* pEHTable) + { + object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_UNUSED, pEHTable, pShadowFrame, (DispatchData*)pDispatchData); + + EHRyuJitClauseIteratorWasm clauseIter = new EHRyuJitClauseIteratorWasm((void**)pEHTable); + EHRyuJitClauseWasm clause; + while (clauseIter.Next(&clause)) + { + WasmEHLogEHTableEntry(clause, pShadowFrame); + + bool foundHandler = false; + if (clause.Filter != null) + { + if (CallFilterFunclet(clause.Filter, exception, pShadowFrame, ryuJitAbi: true)) + { + foundHandler = true; + } + } + else + { + if (ShouldTypedClauseCatchThisException(exception, clause.ClauseType)) + { + foundHandler = true; + } + } + + if (foundHandler) + { + __cxa_end_catch(); + return CallCatchFunclet(clause.Handler, exception, pShadowFrame); + } + } + + return DispatchContinueSearch(exception, pDispatchData); + } + + private static int HandleExceptionWasmFilteredCatch(void* pShadowFrame, void* pDispatchData, void* pHandler, void* pFilter) + { + object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_FILTER, null, pShadowFrame, (DispatchData*)pDispatchData); + + if (CallFilterFunclet(pFilter, exception, pShadowFrame, ryuJitAbi: true)) + { + __cxa_end_catch(); + return CallCatchFunclet(pHandler, exception, pShadowFrame); + } + + return DispatchContinueSearch(exception, pDispatchData); + } + + private static int HandleExceptionWasmCatch(void* pShadowFrame, void* pDispatchData, void* pHandler, void* pClauseType) + { + object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_TYPED, pClauseType, pShadowFrame, (DispatchData*)pDispatchData); + + if (ShouldTypedClauseCatchThisException(exception, (MethodTable*)pClauseType)) + { + __cxa_end_catch(); + return CallCatchFunclet(pHandler, exception, pShadowFrame); + } + + return DispatchContinueSearch(exception, pDispatchData); + } + + private static void HandleExceptionWasmFault(void* pShadowFrame, void* pDispatchData, void* pHandler) + { + // TODO-LLVM-EH: for compatiblity with the IL backend we will invoke faults/finallys even if we do not find + // a suitable catch in a frame. A correct implementation of this will require us to keep a stack of pending + // faults/finallys (inside the native exception), to be invoked at the point we find the handling frame. We + // should also fail fast instead of invoking the second pass handlers if the exception goes unhandled. + // + object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_FAULT, null, pShadowFrame, (DispatchData*)pDispatchData); + CallFinallyFunclet(pHandler, pShadowFrame, ryuJitAbi: true); + DispatchContinueSearch(exception, pDispatchData); + } + + private static object BeginSingleDispatch(RhEHClauseKind kind, void* data, void* pShadowFrame, DispatchData* pDispatchData) + { + // The managed exception is passed around in the native one, and if we take a GC before retrieving it, + // it will get invalidated. See also the TODO-LLVM-EH above; we should address this in a more robust way. + InternalCalls.RhpSetThreadDoNotTriggerGC(); + + ManagedExceptionWrapper* pCppException; + bool isActive = pDispatchData->Active; + if (!isActive) + { + pCppException = (ManagedExceptionWrapper*)__cxa_begin_catch(pDispatchData->CppExceptionTuple.ExceptionData); + pDispatchData->DispatcherData = pCppException; + } + else + { + pCppException = (ManagedExceptionWrapper*)pDispatchData->DispatcherData; + } + + object exception = Unsafe.Read(&pCppException->ManagedException); + InternalCalls.RhpClearThreadDoNotTriggerGC(); + + WasmEHLogDispatcherEnter(kind, data, pShadowFrame, isActive); + + return exception; + } + + private static int DispatchContinueSearch(object exception, void* pDispatchData) + { + // GC may have invalidated the exception object in the native exception; make sure it is up-to-date before + // rethrowing or jumping to an upstream dispatcher. + fixed (void* pKeepAlive = &exception.GetRawData()) + { + Unsafe.Write(&((ManagedExceptionWrapper*)((DispatchData*)pDispatchData)->DispatcherData)->ManagedException, exception); + } + + return 0; + } + + [DllImport("*"), SuppressGCTransition] + private static extern byte* __cxa_begin_catch(void* pExceptionData); + + [DllImport("*"), SuppressGCTransition] + private static extern void __cxa_end_catch(); + + [DllImport("*")] + private static extern void RhpThrowEx(void* exception); + + [RuntimeExport("RhpRethrow")] + private static void RhpRethrow(void** pException) // TODO-LLVM: use object* once C#11 is available. + { + Exception.DispatchExLLVM(Unsafe.Read(pException)); + RhpThrowEx(*pException); // GC hole... + } + + // The below two handlers are invoked by LLVM code generated via the IL backend. + // internal struct RhEHClauseWasm { internal uint _tryStartOffset; @@ -24,28 +183,25 @@ internal struct RhEHClauseWasm internal byte* _handlerAddress; internal byte* _filterAddress; - public bool TryStartsAt(uint idxTryLandingStart) - { - return idxTryLandingStart == _tryStartOffset; - } - public bool ContainsCodeOffset(uint idxTryLandingStart) { - return ((idxTryLandingStart >= _tryStartOffset) && - (idxTryLandingStart < _tryEndOffset)); + return (idxTryLandingStart >= _tryStartOffset) && (idxTryLandingStart < _tryEndOffset); } } - // TODO: temporary to try things out, when working look to see how to refactor with FindFirstPassHandler private static bool FindFirstPassHandlerWasm(object exception, uint idxStart, uint idxCurrentBlockStart /* the start IL idx of the current block for the landing pad, will use in place of PC */, void* shadowStack, ref EHClauseIterator clauseIter, out uint tryRegionIdx, out byte* pHandler) { - pHandler = (byte*)0; + WasmEHLogFirstPassEnter(exception, idxCurrentBlockStart, shadowStack); + + pHandler = null; tryRegionIdx = MaxTryRegionIdx; + uint lastTryStart = 0, lastTryEnd = 0; RhEHClauseWasm ehClause = new RhEHClauseWasm(); for (uint curIdx = 0; clauseIter.Next(ref ehClause); curIdx++) { + WasmEHLogEHInfoEntry(&ehClause, curIdx, "1", shadowStack); // // Skip to the starting try region. This is used by collided unwinds and rethrows to pickup where // the previous dispatch left off. @@ -87,31 +243,36 @@ public bool ContainsCodeOffset(uint idxTryLandingStart) { pHandler = ehClause._handlerAddress; tryRegionIdx = curIdx; + + WasmEHLogFirstPassExit(pHandler, shadowStack); return true; } } else { - bool shouldInvokeHandler = InternalCalls.RhpCallFilterFunclet(exception, ehClause._filterAddress, shadowStack); + bool shouldInvokeHandler = CallFilterFunclet(ehClause._filterAddress, exception, shadowStack, ryuJitAbi: false); if (shouldInvokeHandler) { pHandler = ehClause._handlerAddress; tryRegionIdx = curIdx; + + WasmEHLogFirstPassExit(pHandler, shadowStack); return true; } } } + WasmEHLogFirstPassExit(null, shadowStack); return false; } - private static void InvokeSecondPassWasm(uint idxStart, uint idxTryLandingStart /* we do dont have the PC, so use the start of the block */, ref EHClauseIterator clauseIter, uint idxLimit, void* shadowStack) + private static void InvokeSecondPassWasm(uint idxStart, uint idxTryLandingStart /* we do dont have the PC, so use the start of the block */, + ref EHClauseIterator clauseIter, uint idxLimit, void* shadowStack) { // Search the clauses for one that contains the current offset. RhEHClauseWasm ehClause = new RhEHClauseWasm(); for (uint curIdx = 0; clauseIter.Next(ref ehClause) && curIdx < idxLimit; curIdx++) { - if (curIdx > idxStart) { break; // these blocks are after the catch @@ -127,21 +288,175 @@ public bool ContainsCodeOffset(uint idxTryLandingStart) // Found a containing clause. Because of the order of the clauses, we know this is the // most containing. + CallFinallyFunclet(ehClause._handlerAddress, shadowStack, ryuJitAbi: false); + } + } - // N.B. -- We need to suppress GC "in-between" calls to finallys in this loop because we do - // not have the correct next-execution point live on the stack and, therefore, may cause a GC - // hole if we allow a GC between invocation of finally funclets (i.e. after one has returned - // here to the dispatcher, but before the next one is invoked). Once they are running, it's - // fine for them to trigger a GC, obviously. - // - // As a result, RhpCallFinallyFunclet will set this state in the runtime upon return from the - // funclet, and we need to reset it if/when we fall out of the loop and we know that the - // method will no longer get any more GC callbacks. + private static int CallCatchFunclet(void* pFunclet, object exception, void* pShadowFrame) + { + // IL backend invokes the catch handler in generated code. + WasmEHLogFunletEnter(pFunclet, RhEHClauseKind.RH_EH_CLAUSE_TYPED, pShadowFrame, ryuJitAbi: true); + int catchRetIdx = ((delegate*)pFunclet)(exception, pShadowFrame); + WasmEHLogFunletExit(RhEHClauseKind.RH_EH_CLAUSE_TYPED, catchRetIdx, pShadowFrame, ryuJitAbi: true); + + return catchRetIdx; + } + + private static bool CallFilterFunclet(void* pFunclet, object exception, void* pShadowFrame, bool ryuJitAbi) + { + bool result; + WasmEHLogFunletEnter(pFunclet, RhEHClauseKind.RH_EH_CLAUSE_FILTER, pShadowFrame, ryuJitAbi); + if (ryuJitAbi) + { + result = ((delegate*)pFunclet)(exception, pShadowFrame) != 0; + } + else + { + result = InternalCalls.RhpCallFilterFunclet(exception, (byte*)pFunclet, pShadowFrame); + } + WasmEHLogFunletExit(RhEHClauseKind.RH_EH_CLAUSE_FILTER, result ? 1 : 0, pShadowFrame, ryuJitAbi); + + return result; + } + + private static void CallFinallyFunclet(void* pFunclet, void* pShadowFrame, bool ryuJitAbi) + { + WasmEHLogFunletEnter(pFunclet, RhEHClauseKind.RH_EH_CLAUSE_FAULT, pShadowFrame, ryuJitAbi); + if (ryuJitAbi) + { + ((delegate*)pFunclet)(pShadowFrame); + } + else + { + InternalCalls.RhpCallFinallyFunclet((byte*)pFunclet, pShadowFrame); + } + WasmEHLogFunletExit(RhEHClauseKind.RH_EH_CLAUSE_FAULT, 0, pShadowFrame, ryuJitAbi); + } + + [Conditional("ENABLE_NOISY_WASM_EH_LOG")] + private static void WasmEHLog(string message, void* pShadowFrame, string pass, bool ryuJitAbi) + { + string log = "WASM EH/" + (ryuJitAbi ? "RyuJit" : "IL"); + log += " [SF: " + ToHex(pShadowFrame) + "]"; + log += " [" + pass + "]"; + log += ": " + message + Environment.NewLineConst; + + byte[] bytes = new byte[log.Length + 1]; + for (int i = 0; i < log.Length; i++) + { + bytes[i] = (byte)log[i]; + } + + fixed (byte* p = bytes) + { + Interop.Sys.Log(p, bytes.Length); + } + } + + [Conditional("ENABLE_NOISY_WASM_EH_LOG")] + private static void WasmEHLogDispatcherEnter(RhEHClauseKind kind, void* data, void* pShadowFrame, bool isActive) + { + string description = GetClauseDescription(kind, data); + string pass = kind == RhEHClauseKind.RH_EH_CLAUSE_FAULT ? "2" : "1"; + WasmEHLog("Handling" + (isActive ? " (active)" : "") + ": " + description, pShadowFrame, pass, ryuJitAbi: true); + } + + [Conditional("ENABLE_NOISY_WASM_EH_LOG")] + private static void WasmEHLogEHTableEntry(EHRyuJitClauseWasm clause, void* pShadowFrame) + { + string description = clause.Filter != null ? GetClauseDescription(RhEHClauseKind.RH_EH_CLAUSE_FILTER, clause.Filter) + : GetClauseDescription(RhEHClauseKind.RH_EH_CLAUSE_TYPED, clause.ClauseType); + WasmEHLog("Clause: " + description, pShadowFrame, "1", ryuJitAbi: true); + } + + private static string GetClauseDescription(RhEHClauseKind kind, void* data) => kind switch + { + RhEHClauseKind.RH_EH_CLAUSE_TYPED => "catch, class [" + new RuntimeTypeHandle(new EETypePtr((MethodTable*)data)).LastResortToString + "]", + RhEHClauseKind.RH_EH_CLAUSE_FILTER => "filtered catch", + RhEHClauseKind.RH_EH_CLAUSE_UNUSED => "mutually protecting catches, table at [" + ToHex(data) + "]", + _ => "fault", + }; + + [Conditional("ENABLE_NOISY_WASM_EH_LOG")] + private static void WasmEHLogFunletEnter(void* pHandler, RhEHClauseKind kind, void* pShadowFrame, bool ryuJitAbi) + { + (string name, string pass) = kind switch + { + RhEHClauseKind.RH_EH_CLAUSE_FILTER => ("filter", "1"), + RhEHClauseKind.RH_EH_CLAUSE_FAULT => ("fault", "2"), + _ => ("catch", "2") + }; + + WasmEHLog("Calling " + name + " funclet at [" + ToHex(pHandler) + "]", pShadowFrame, pass, ryuJitAbi); + } + + [Conditional("ENABLE_NOISY_WASM_EH_LOG")] + private static void WasmEHLogFunletExit(RhEHClauseKind kind, int result, void* pShadowFrame, bool ryuJitAbi) + { + (string resultString, string pass) = kind switch + { + RhEHClauseKind.RH_EH_CLAUSE_FILTER => (result == 1 ? "true" : "false", "1"), + RhEHClauseKind.RH_EH_CLAUSE_FAULT => ("success", "2"), + _ => (ToHex(result), "2") + }; - byte* pFinallyHandler = ehClause._handlerAddress; + WasmEHLog("Funclet returned: " + resultString, pShadowFrame, pass, ryuJitAbi); + } + + [Conditional("ENABLE_NOISY_WASM_EH_LOG")] + private static void WasmEHLogFirstPassEnter(object exception, uint codeIdx, void* pShadowFrame) + { + WasmEHLog("Pass start; at: " + ToHex(codeIdx) + ", exception: [" + exception.GetTypeHandle().LastResortToString + "]", + pShadowFrame, "1", ryuJitAbi: false); + } + + [Conditional("ENABLE_NOISY_WASM_EH_LOG")] + private static void WasmEHLogFirstPassExit(void* pHandler, void* pShadowFrame) + { + WasmEHLog("Pass end; handler " + (pHandler != null ? "found at [" + ToHex(pHandler) + "]" : "not found"), + pShadowFrame, "1", ryuJitAbi: false); + } + + [Conditional("ENABLE_NOISY_WASM_EH_LOG")] + private static void WasmEHLogEHInfoEntry(RhEHClauseWasm* pClause, uint index, string pass, void* pShadowFrame) + { + string tryBeg = ToHex(pClause->_tryStartOffset); + string tryEnd = ToHex(pClause->_tryEndOffset); + string hndAddr = ToHex(pClause->_handlerAddress); + string log = "EH#" + ToHex(index) + ": try [" + tryBeg + ".." + tryEnd + ") handled by [" + hndAddr + "]"; - InternalCalls.RhpCallFinallyFunclet(pFinallyHandler, shadowStack); + switch (pClause->_clauseKind) + { + case EHClauseIterator.RhEHClauseKindWasm.RH_EH_CLAUSE_FILTER: + log += ", filter at [" + ToHex(pClause->_filterAddress) + "]"; + break; + case EHClauseIterator.RhEHClauseKindWasm.RH_EH_CLAUSE_TYPED: + log += ", class [" + new RuntimeTypeHandle(new EETypePtr((MethodTable*)pClause->_typeSymbol)).LastResortToString + "]"; + break; + case EHClauseIterator.RhEHClauseKindWasm.RH_EH_CLAUSE_FAULT: + log += ", fault"; + break; + default: + log += ", unknown?!"; + break; } + + WasmEHLog(log, pShadowFrame, pass, ryuJitAbi: false); + } + + private static string ToHex(uint value) => ToHex((int)value); + private static string ToHex(void* value) => "0x" + ToHex((nint)value); + + private static string ToHex(nint value) + { + int length = 2 * sizeof(nint); + char* chars = stackalloc char[length]; + for (int i = length - 1, j = 0; j < length; i--, j++) + { + chars[j] = "0123456789ABCDEF"[(int)((value >> (i * 4)) & 0xF)]; + } + + return new string(chars, 0, length); } - } // static class EH + } } diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs index 35f37547ec6a..72d55e823505 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs @@ -242,6 +242,10 @@ internal static extern unsafe bool RhpCallFilterFunclet( [MethodImpl(MethodImplOptions.InternalCall)] internal static extern unsafe void RhpFallbackFailFast(); + [RuntimeImport(Redhawk.BaseName, "RhpClearThreadDoNotTriggerGC")] + [MethodImpl(MethodImplOptions.InternalCall)] + internal static extern void RhpClearThreadDoNotTriggerGC(); + [RuntimeImport(Redhawk.BaseName, "RhpSetThreadDoNotTriggerGC")] [MethodImpl(MethodImplOptions.InternalCall)] internal static extern void RhpSetThreadDoNotTriggerGC(); diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/StackFrameIterator.wasm.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/StackFrameIterator.wasm.cs index 96e4fffdcdb5..8ab7f97df6b8 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/StackFrameIterator.wasm.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/StackFrameIterator.wasm.cs @@ -1,14 +1,14 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Runtime.InteropServices; +using Internal.Runtime; namespace System.Runtime { internal unsafe struct EHClauseIterator { private uint _totalClauses; - private byte *_currentPtr; + private byte* _currentPtr; private int _currentClause; private static uint DecodeUnsigned(ref byte* stream) @@ -84,7 +84,10 @@ internal enum RhEHClauseKindWasm internal bool Next(ref EH.RhEHClauseWasm pEHClause) { - if (_currentClause >= _totalClauses) return false; + if (_currentClause >= _totalClauses) + { + return false; + } _currentClause++; pEHClause._tryStartOffset = GetUnsigned(); @@ -94,27 +97,107 @@ internal bool Next(ref EH.RhEHClauseWasm pEHClause) switch (pEHClause._clauseKind) { case RhEHClauseKindWasm.RH_EH_CLAUSE_TYPED: - AlignToSymbol(); pEHClause._typeSymbol = ReadUInt32(ref _currentPtr); - pEHClause._handlerAddress = (byte *)ReadUInt32(ref _currentPtr); + pEHClause._handlerAddress = (byte*)ReadUInt32(ref _currentPtr); break; + case RhEHClauseKindWasm.RH_EH_CLAUSE_FAULT: AlignToSymbol(); pEHClause._handlerAddress = (byte*)ReadUInt32(ref _currentPtr); break; + case RhEHClauseKindWasm.RH_EH_CLAUSE_FILTER: AlignToSymbol(); pEHClause._handlerAddress = (byte*)ReadUInt32(ref _currentPtr); pEHClause._filterAddress = (byte*)ReadUInt32(ref _currentPtr); break; } + return true; } private void AlignToSymbol() { - _currentPtr = (byte *)(((uint)_currentPtr + 3) & ~(3)); + _currentPtr = (byte*)(((uint)_currentPtr + 3) & ~3); + } + } + + // This iterator is used for EH tables produces by codegen for runs of mutually protecting catch handlers. + // + internal unsafe struct EHRyuJitClauseWasm + { + public void* Handler; + public void* Filter; + public MethodTable* ClauseType; + } + + // See codegen code ("jit/llvmcodegen.cpp, generateEHDispatchTable") for details on the format of the table. + // + internal unsafe struct EHRyuJitClauseIteratorWasm + { + private const nuint HeaderRecordSize = 1; + private const nuint ClauseRecordSize = 2; + private static nuint FirstSectionSize => HeaderRecordSize + (nuint)sizeof(nuint) / 2 * 8 * ClauseRecordSize; + private static nuint LargeSectionSize => HeaderRecordSize + (nuint)sizeof(nuint) * 8 * ClauseRecordSize; + + private readonly void** _pTableEnd; + private void** _pCurrentSectionClauses; + private void** _pNextSection; + private nuint _currentIndex; + private nuint _clauseKindMask; + + public EHRyuJitClauseIteratorWasm(void** pEHTable) + { + _pCurrentSectionClauses = pEHTable + HeaderRecordSize; + _pNextSection = pEHTable + FirstSectionSize; + _currentIndex = 0; +#if TARGET_32BIT + _clauseKindMask = ((ushort*)pEHTable)[1]; + nuint tableSize = ((ushort*)pEHTable)[0]; +#else + _clauseKindMask = ((uint*)pEHTable)[1]; + nuint tableSize = ((uint*)pEHTable)[0]; +#endif + _pTableEnd = pEHTable + tableSize; + } + + public bool Next(EHRyuJitClauseWasm* pClause) + { + void** pCurrent = _pCurrentSectionClauses + _currentIndex * ClauseRecordSize; + if (pCurrent >= _pTableEnd) + { + return false; + } + + if ((_clauseKindMask & ((nuint)1 << (int)_currentIndex)) != 0) + { + pClause->Filter = pCurrent[0]; + pClause->ClauseType = null; + } + else + { + pClause->Filter = null; + pClause->ClauseType = *(MethodTable**)pCurrent[0]; + } + + pClause->Handler = pCurrent[1]; + + // Initialize the state for the next iteration. + void** pCurrentNext = pCurrent + ClauseRecordSize; + if ((pCurrentNext != _pTableEnd) && (pCurrentNext == _pNextSection)) + { + _pCurrentSectionClauses = pCurrentNext + HeaderRecordSize; + _pNextSection += LargeSectionSize; + _currentIndex = 0; + _clauseKindMask = (nuint)pCurrentNext[0]; + } + else + { + _currentIndex++; + } + + return true; } } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj index 20b4f1a2bdcb..09c6fac862e1 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj @@ -23,6 +23,7 @@ SYSTEM_PRIVATE_CORELIB;FEATURE_MANAGED_ETW_CHANNELS;FEATURE_MANAGED_ETW;$(DefineConstants) + ENABLE_NOISY_WASM_EH_LOG;$(DefineConstants) true true ..\..\Runtime.Base\src\ diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Exception.CoreRT.LLVM.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Exception.CoreRT.LLVM.cs index 92f9985b6d7b..cd5785af9b60 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Exception.CoreRT.LLVM.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Exception.CoreRT.LLVM.cs @@ -8,7 +8,7 @@ namespace System { public partial class Exception { - private static void DispatchExLLVM(object exception) + internal static void DispatchExLLVM(object exception) { AppendExceptionStackFrameLLVM(exception, new StackTrace(1).ToString()); //RhpThrowEx(exception); can't as not handling the transition unmanaged->managed in the landing pads. diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs index 7397ed825256..c2cb3eaecf6e 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs @@ -160,7 +160,31 @@ public DependencyList Import() _dependencies.Add(GetHelperEntrypoint(ReadyToRunHelper.MonitorEnter), reason); _dependencies.Add(GetHelperEntrypoint(ReadyToRunHelper.MonitorExit), reason); } - + } + + if (_compilation.TargetArchIsWasm() && (_canonMethod.IsSynchronized || _exceptionRegions.Length != 0)) + { + // TODO-LLVM: delete these once IL backend is no more. + MetadataType helperType = _compilation.TypeSystemContext.SystemModule.GetKnownType("System.Runtime", "EHClauseIterator"); + MethodDesc helperMethod = helperType.GetKnownMethod("InitFromEhInfo", null); + _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); + + helperType = _compilation.TypeSystemContext.SystemModule.GetKnownType("System.Runtime", "EH"); + helperMethod = helperType.GetKnownMethod("FindFirstPassHandlerWasm", null); + _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); + + helperMethod = helperType.GetKnownMethod("InvokeSecondPassWasm", null); + _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); + + // TODO-LLVM: make these into normal "ReadyToRunHelper" instead of hardcoding things here. + helperMethod = helperType.GetKnownMethod("HandleExceptionWasmFilteredCatch", null); + _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); + + helperMethod = helperType.GetKnownMethod("HandleExceptionWasmCatch", null); + _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); + + helperMethod = helperType.GetKnownMethod("HandleExceptionWasmFault", null); + _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); } FindBasicBlocks(); @@ -320,22 +344,6 @@ private void ImportCall(ILOpcode opcode, int token) Debug.Assert(false); break; } - // Add Wasm exception handling dependencies if making any kind of call and in a try region - if (opcode != ILOpcode.ldftn && opcode != ILOpcode.ldvirtftn && _compilation.TargetArchIsWasm() && InTryRegion()) - { - MetadataType helperType = _compilation.TypeSystemContext.SystemModule.GetKnownType("System.Runtime", "EHClauseIterator"); - MethodDesc helperMethod = helperType.GetKnownMethod("InitFromEhInfo", null); - _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); - - helperType = _compilation.TypeSystemContext.SystemModule.GetKnownType("System.Runtime", "EH"); - helperMethod = helperType.GetKnownMethod("FindFirstPassHandlerWasm", null); - _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); - - helperType = _compilation.TypeSystemContext.SystemModule.GetKnownType("System.Runtime", "EH"); - helperMethod = helperType.GetKnownMethod("InvokeSecondPassWasm", null); - _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); - } - if (opcode == ILOpcode.newobj) { TypeDesc owningType = runtimeDeterminedMethod.OwningType; From f81f155644f67fdf359897ffc9d1668f9b4c20e0 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Tue, 27 Dec 2022 15:06:04 +0300 Subject: [PATCH 08/13] Implement "emit context" in codegen Allow for using more "emit" APIs without IR context. --- src/coreclr/jit/llvm.h | 24 +++- src/coreclr/jit/llvmcodegen.cpp | 189 +++++++++++++++++++------------- src/coreclr/jit/llvmlower.cpp | 4 +- 3 files changed, 133 insertions(+), 84 deletions(-) diff --git a/src/coreclr/jit/llvm.h b/src/coreclr/jit/llvm.h index eff9590d03d2..996edf519c3c 100644 --- a/src/coreclr/jit/llvm.h +++ b/src/coreclr/jit/llvm.h @@ -103,6 +103,11 @@ struct LlvmBlockRange { llvm::BasicBlock* FirstBlock; llvm::BasicBlock* LastBlock; + INDEBUG(unsigned Count = 1); + + LlvmBlockRange(llvm::BasicBlock* llvmBlock) : FirstBlock(llvmBlock), LastBlock(llvmBlock) + { + } }; // TODO: We should create a Static... class to manage the globals and their lifetimes. @@ -125,7 +130,7 @@ class Llvm CORINFO_SIG_INFO _sigInfo; // sigInfo of function being compiled LIR::Range* _currentRange; - BasicBlock* _currentBlock; + BasicBlock* m_currentBlock; DebugInfo _currentOffset; llvm::IRBuilder<> _builder; llvm::IRBuilder<> _prologBuilder; @@ -137,6 +142,10 @@ class Llvm std::vector m_functions; std::vector m_EHDispatchLlvmBlocks; + // Codegen emit context. + unsigned m_currentProtectedRegionIndex = EHblkDsc::NO_ENCLOSING_INDEX; + LlvmBlockRange* m_currentLlvmBlocks = nullptr; + // DWARF llvm::DILocation* _currentOffsetDiLocation; llvm::DISubprogram* _debugFunction; @@ -166,7 +175,7 @@ class Llvm } BasicBlock* CurrentBlock() const { - return _currentBlock; + return m_currentBlock; } GCInfo* getGCInfo(); @@ -342,17 +351,22 @@ class Llvm DebugMetadata getOrCreateDebugMetadata(const char* documentFileName); llvm::DILocation* createDebugFunctionAndDiLocation(struct DebugMetadata debugMetadata, unsigned int lineNo); + void setCurrentEmitContextForBlock(BasicBlock* block); + void setCurrentEmitContext(LlvmBlockRange* llvmBlock, unsigned tryIndex); + LlvmBlockRange* getCurrentLlvmBlocks() const; + unsigned getCurrentProtectedRegionIndex() const; + Function* getRootLlvmFunction(); Function* getCurrentLlvmFunction(); Function* getLlvmFunctionForIndex(unsigned funcIdx); unsigned getLlvmFunctionIndexForBlock(BasicBlock* block); - void setCurrentLlvmFunctionForBlock(BasicBlock* block); + unsigned getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex); llvm::BasicBlock* createInlineLlvmBlock(); - llvm::BasicBlock* getEHDispatchLlvmBlockForBlock(BasicBlock* block); + llvm::BasicBlock* getCurrentLlvmBlock() const; + LlvmBlockRange* getLlvmBlocksForBlock(BasicBlock* block); llvm::BasicBlock* getFirstLlvmBlockForBlock(BasicBlock* block); llvm::BasicBlock* getLastLlvmBlockForBlock(BasicBlock* block); - void setLastLlvmBlockForBlock(BasicBlock* block, llvm::BasicBlock* llvmBlock); Value* getLocalAddr(unsigned lclNum); unsigned getTotalLocalOffset(); diff --git a/src/coreclr/jit/llvmcodegen.cpp b/src/coreclr/jit/llvmcodegen.cpp index 497552f3b191..161d7b710e27 100644 --- a/src/coreclr/jit/llvmcodegen.cpp +++ b/src/coreclr/jit/llvmcodegen.cpp @@ -4,6 +4,8 @@ #include "llvm.h" +#define BBNAME(prefix, index) llvm::Twine(prefix) + ((index < 10) ? "0" : "") + llvm::Twine(index) + //------------------------------------------------------------------------ // Compile: Compile IR to LLVM, adding to the LLVM Module // @@ -189,18 +191,10 @@ bool Llvm::initializeFunctions() else { // The dispatch block is part of the function with the protected region. - unsigned enclosingFuncIdx = ROOT_FUNC_IDX; - unsigned enclosingHndIndex = ehDsc->ebdEnclosingHndIndex; - if (enclosingHndIndex != EHblkDsc::NO_ENCLOSING_INDEX) - { - // Note here we will correctly get the "filter handler" part of filter. - // There can be no protected regions in the "filter" parts of filters. - enclosingFuncIdx = _compiler->ehGetDsc(enclosingHndIndex)->ebdFuncIndex; - } - + unsigned enclosingFuncIdx = getLlvmFunctionIndexForProtectedRegion(ehIndex); Function* dispatchLlvmFunc = getLlvmFunctionForIndex(enclosingFuncIdx); dispatchLlvmBlock = - llvm::BasicBlock::Create(_llvmContext, "BBT" + llvm::Twine(ehDsc->ebdTryBeg->getTryIndex()), + llvm::BasicBlock::Create(_llvmContext, BBNAME("BT", ehDsc->ebdTryBeg->getTryIndex()), dispatchLlvmFunc); } @@ -344,11 +338,7 @@ void Llvm::generateBlock(BasicBlock* block) JITDUMP("\n=============== Generating "); JITDUMPEXEC(block->dspBlockHeader(_compiler, /* showKind */ true, /* showFlags */ true)); - setCurrentLlvmFunctionForBlock(block); - - llvm::BasicBlock* llvmBlock = getFirstLlvmBlockForBlock(block); - _currentBlock = block; - _builder.SetInsertPoint(llvmBlock); + setCurrentEmitContextForBlock(block); for (GenTree* node : LIR::AsRange(block)) { @@ -385,18 +375,6 @@ void Llvm::generateBlock(BasicBlock* block) // TODO-LLVM: other jump kinds. break; } - -#ifdef DEBUG - llvm::BasicBlock* lastLlvmBlock = _builder.GetInsertBlock(); - if (lastLlvmBlock != llvmBlock) - { - llvm::StringRef blockName = llvmBlock->getName(); - for (unsigned idx = 1; llvmBlock != lastLlvmBlock->getNextNode(); llvmBlock = llvmBlock->getNextNode(), idx++) - { - llvmBlock->setName(blockName + "." + llvm::Twine(idx)); - } - } -#endif // DEBUG } void Llvm::generateEHDispatch() @@ -1783,6 +1761,7 @@ void Llvm::buildCatchArg(GenTree* node) { // The exception object is passed as the first shadow parameter to funclets. Type* excLlvmType = getLlvmTypeForVarType(node->TypeGet()); + // TODO-LLVM: delete once we move to opaque pointers. Value* shadowStackValue = _builder.CreateBitCast(getShadowStack(), excLlvmType->getPointerTo()); Value* excObjValue = _builder.CreateLoad(excLlvmType, shadowStackValue); @@ -1978,8 +1957,10 @@ void Llvm::emitJumpToThrowHelper(Value* jumpCondValue, SpecialCodeKind throwKind { if (_compiler->fgUseThrowHelperBlocks()) { + assert(CurrentBlock() != nullptr); + // For code with throw helper blocks, find and use the shared helper block for raising the exception. - unsigned throwIndex = _compiler->bbThrowIndex(_currentBlock); + unsigned throwIndex = _compiler->bbThrowIndex(CurrentBlock()); BasicBlock* throwBlock = _compiler->fgFindExcptnTarget(throwKind, throwIndex)->acdDstBlk; // Jump to the exception-throwing block on error. @@ -2058,25 +2039,37 @@ Value* Llvm::emitHelperCall(CorInfoHelpFunc helperFunc, ArrayRef sigArgs return callValue; } -Value* Llvm::emitCallOrInvoke(llvm::FunctionCallee callee, ArrayRef args) +llvm::CallBase* Llvm::emitCallOrInvoke(llvm::FunctionCallee callee, ArrayRef args) { - llvm::BasicBlock* catchLlvmBlock = getEHDispatchLlvmBlockForBlock(CurrentBlock()); + llvm::BasicBlock* catchLlvmBlock = nullptr; + if (getCurrentProtectedRegionIndex() != EHblkDsc::NO_ENCLOSING_INDEX) + { + catchLlvmBlock = m_EHDispatchLlvmBlocks[getCurrentProtectedRegionIndex()]; - Value* callValue; + // Protected region index that is set in the emit context refers to the "logical" enclosing + // protected region, i. e. the one before funclet creation. But we do not need to (in fact, + // cannot) emit an invoke targeting block inside a different LLVM function. + if (catchLlvmBlock->getParent() != getCurrentLlvmFunction()) + { + catchLlvmBlock = nullptr; + } + } + + llvm::CallBase* callInst; if (catchLlvmBlock != nullptr) { llvm::BasicBlock* nextLlvmBlock = createInlineLlvmBlock(); - callValue = _builder.CreateInvoke(callee, nextLlvmBlock, catchLlvmBlock, args); + callInst = _builder.CreateInvoke(callee, nextLlvmBlock, catchLlvmBlock, args); _builder.SetInsertPoint(nextLlvmBlock); } else { - callValue = _builder.CreateCall(callee, args); + callInst = _builder.CreateCall(callee, args); } - return callValue; + return callInst; } FunctionType* Llvm::getFunctionType() @@ -2319,6 +2312,48 @@ llvm::DILocation* Llvm::createDebugFunctionAndDiLocation(DebugMetadata debugMeta return llvm::DILocation::get(_llvmContext, lineNo, 0, _debugFunction); } +llvm::BasicBlock* Llvm::getCurrentLlvmBlock() const +{ + return getCurrentLlvmBlocks()->LastBlock; +} + +void Llvm::setCurrentEmitContextForBlock(BasicBlock* block) +{ + LlvmBlockRange* llvmBlocks = getLlvmBlocksForBlock(block); + unsigned tryIndex = block->hasTryIndex() ? block->getTryIndex() : EHblkDsc::NO_ENCLOSING_INDEX; + + setCurrentEmitContext(llvmBlocks, tryIndex); + m_currentBlock = block; +} + +void Llvm::setCurrentEmitContext(LlvmBlockRange* llvmBlocks, unsigned tryIndex) +{ + _builder.SetInsertPoint(llvmBlocks->LastBlock); + m_currentLlvmBlocks = llvmBlocks; + m_currentProtectedRegionIndex = tryIndex; + + // "Raw" emission contexts do not have a current IR block. + m_currentBlock = nullptr; +} + +LlvmBlockRange* Llvm::getCurrentLlvmBlocks() const +{ + assert(m_currentLlvmBlocks != nullptr); + return m_currentLlvmBlocks; +} + +//------------------------------------------------------------------------ +// getCurrentProtectedRegionIndex: Get the current protected region's index. +// +// Return Value: +// Index of the EH descriptor for the (innermost) protected region ("try") +// enclosing code in the current emit context. +// +unsigned Llvm::getCurrentProtectedRegionIndex() const +{ + return m_currentProtectedRegionIndex; +} + Function* Llvm::getRootLlvmFunction() { return getLlvmFunctionForIndex(ROOT_FUNC_IDX); @@ -2326,7 +2361,7 @@ Function* Llvm::getRootLlvmFunction() Function* Llvm::getCurrentLlvmFunction() { - return getLlvmFunctionForIndex(_compiler->compCurrFuncIdx); + return getCurrentLlvmBlock()->getParent(); } Function* Llvm::getLlvmFunctionForIndex(unsigned funcIdx) @@ -2357,62 +2392,67 @@ unsigned Llvm::getLlvmFunctionIndexForBlock(BasicBlock* block) return funcIdx; } -void Llvm::setCurrentLlvmFunctionForBlock(BasicBlock* block) +unsigned Llvm::getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex) { - _compiler->funSetCurrentFunc(getLlvmFunctionIndexForBlock(block)); + unsigned funcIdx = ROOT_FUNC_IDX; + if (tryIndex != EHblkDsc::NO_ENCLOSING_INDEX) + { + EHblkDsc* ehDsc = _compiler->ehGetDsc(tryIndex); + if (ehDsc->ebdEnclosingHndIndex != EHblkDsc::NO_ENCLOSING_INDEX) + { + // Note here we will correctly get the "filter handler" part of filter. + // There can be no protected regions in the "filter" parts of filters. + funcIdx = _compiler->ehGetDsc(ehDsc->ebdEnclosingHndIndex)->ebdFuncIndex; + } + } + + return funcIdx; } llvm::BasicBlock* Llvm::createInlineLlvmBlock() { - BasicBlock* currentBlock = CurrentBlock(); - llvm::BasicBlock* insertBefore = getLastLlvmBlockForBlock(currentBlock)->getNextNode(); - llvm::BasicBlock* inlineLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "", getCurrentLlvmFunction(), insertBefore); + Function* llvmFunc = getCurrentLlvmFunction(); + LlvmBlockRange* llvmBlocks = getCurrentLlvmBlocks(); + llvm::BasicBlock* insertBefore = llvmBlocks->LastBlock->getNextNode(); + llvm::BasicBlock* inlineLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "", llvmFunc, insertBefore); - setLastLlvmBlockForBlock(currentBlock, inlineLlvmBlock); - return inlineLlvmBlock; -} - -llvm::BasicBlock* Llvm::getEHDispatchLlvmBlockForBlock(BasicBlock* block) -{ - if (!block->hasTryIndex()) +#ifdef DEBUG + llvm::StringRef blocksName = llvmBlocks->FirstBlock->getName(); + if (llvmBlocks->Count == 1) { - return nullptr; + llvmBlocks->FirstBlock->setName(blocksName + ".1"); } - - llvm::BasicBlock* ehDispatchLlvmBlock = m_EHDispatchLlvmBlocks[block->getTryIndex()]; - assert(ehDispatchLlvmBlock != nullptr); - - // Blocks inside funclets retain their original protected region indices, however, we won't have a dispatch - // block for the top-level ones (thus any exceptions out of funclets will propagate to the EH dispatch routine). - if (ehDispatchLlvmBlock->getParent() != getCurrentLlvmFunction()) + else { - return nullptr; + blocksName = blocksName.take_front(blocksName.find_last_of('.')); } - return ehDispatchLlvmBlock; + inlineLlvmBlock->setName(blocksName + "." + llvm::Twine(++llvmBlocks->Count)); +#endif // DEBUG + + llvmBlocks->LastBlock = inlineLlvmBlock; + return inlineLlvmBlock; } -llvm::BasicBlock* Llvm::getFirstLlvmBlockForBlock(BasicBlock* block) +LlvmBlockRange* Llvm::getLlvmBlocksForBlock(BasicBlock* block) { assert(block != nullptr); - llvm::BasicBlock* llvmBlock; - LlvmBlockRange llvmBlockRange; - if (!_blkToLlvmBlksMap.Lookup(block, &llvmBlockRange)) + LlvmBlockRange* llvmBlockRange = _blkToLlvmBlksMap.LookupPointer(block); + if (llvmBlockRange == nullptr) { - unsigned bbNum = block->bbNum; - Function* llvmFunc = getCurrentLlvmFunction(); - llvmBlock = llvm::BasicBlock::Create( - _llvmContext, (bbNum >= 10) ? ("BB" + llvm::Twine(bbNum)) : ("BB0" + llvm::Twine(bbNum)), llvmFunc); + Function* llvmFunc = getLlvmFunctionForIndex(getLlvmFunctionIndexForBlock(block)); + llvm::BasicBlock* llvmBlock = llvm::BasicBlock::Create(_llvmContext, BBNAME("BB", block->bbNum), llvmFunc); - _blkToLlvmBlksMap.Set(block, {llvmBlock, llvmBlock}); - } - else - { - llvmBlock = llvmBlockRange.FirstBlock; + llvmBlockRange = _blkToLlvmBlksMap.Emplace(block, llvmBlock); } - return llvmBlock; + return llvmBlockRange; +} + +llvm::BasicBlock* Llvm::getFirstLlvmBlockForBlock(BasicBlock* block) +{ + return getLlvmBlocksForBlock(block)->FirstBlock; } //------------------------------------------------------------------------ @@ -2430,12 +2470,7 @@ llvm::BasicBlock* Llvm::getFirstLlvmBlockForBlock(BasicBlock* block) // llvm::BasicBlock* Llvm::getLastLlvmBlockForBlock(BasicBlock* block) { - return _blkToLlvmBlksMap[block].LastBlock; -} - -void Llvm::setLastLlvmBlockForBlock(BasicBlock* block, llvm::BasicBlock* llvmBlock) -{ - _blkToLlvmBlksMap[block].LastBlock = llvmBlock; + return getLlvmBlocksForBlock(block)->LastBlock; } Value* Llvm::getLocalAddr(unsigned lclNum) diff --git a/src/coreclr/jit/llvmlower.cpp b/src/coreclr/jit/llvmlower.cpp index f350f2d8ed66..0fc16d845790 100644 --- a/src/coreclr/jit/llvmlower.cpp +++ b/src/coreclr/jit/llvmlower.cpp @@ -342,12 +342,12 @@ void Llvm::lowerBlocks() block->bbFlags &= ~BBF_MARKED; } - _currentBlock = nullptr; + m_currentBlock = nullptr; } void Llvm::lowerBlock(BasicBlock* block) { - _currentBlock = block; + m_currentBlock = block; _currentRange = &LIR::AsRange(block); for (GenTree* node : CurrentRange()) From 9115602a7263db8cd1d77884009eef05219425d7 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Sun, 1 Jan 2023 16:13:19 +0300 Subject: [PATCH 09/13] Disable EH logging --- src/coreclr/nativeaot/Bootstrap/main.cpp | 2 -- .../System.Private.CoreLib/src/System.Private.CoreLib.csproj | 1 - 2 files changed, 3 deletions(-) diff --git a/src/coreclr/nativeaot/Bootstrap/main.cpp b/src/coreclr/nativeaot/Bootstrap/main.cpp index dbe7c5f0fd35..142c6dcd3afa 100644 --- a/src/coreclr/nativeaot/Bootstrap/main.cpp +++ b/src/coreclr/nativeaot/Bootstrap/main.cpp @@ -5,7 +5,6 @@ #if defined HOST_WASM #include -#include #endif // HOST_WASM // @@ -168,7 +167,6 @@ class ManagedExceptionWrapper : std::exception extern "C" void RhpThrowEx(void * pEx) { - printf("WASM EH: exception thrown\n"); throw ManagedExceptionWrapper(pEx); } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj index 09c6fac862e1..20b4f1a2bdcb 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj @@ -23,7 +23,6 @@ SYSTEM_PRIVATE_CORELIB;FEATURE_MANAGED_ETW_CHANNELS;FEATURE_MANAGED_ETW;$(DefineConstants) - ENABLE_NOISY_WASM_EH_LOG;$(DefineConstants) true true ..\..\Runtime.Base\src\ From 7261e42c7cb2567a3f9db43377729a1e28c07872 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Wed, 28 Dec 2022 18:57:34 +0300 Subject: [PATCH 10/13] Implement dispatch codegen Handler frequency statistics, collected via SPMI: benchmarks.pmi Total clauses : 2514 Finally : 281 11.2% Fault : 1413 56.2% Single catch : 724 28.8% Single filter : 49 1.9% Mutually protecting : 47 1.9% Among them, catches : 108 86.4% Among them, filters : 17 13.6% libraries.pmi Total clauses : 19568 Finally : 1937 9.9% Fault : 11399 58.3% Single catch : 5373 27.5% Single filter : 349 1.8% Mutually protecting : 510 2.6% Among them, catches : 1240 95.0% Among them, filters : 65 5.0% Passing (with RyuJit codegen) tests: - baseservices\exceptions\simple\finally.csproj - baseservices\exceptions\simple\fault.ilproj - baseservices\exceptions\unittests\baseclass.csproj - baseservices\exceptions\unittests\Baadbaad.csproj - baseservices\exceptions\unittests\TryCatchInFinallyTest.csproj - baseservices\exceptions\unittests\ThrowInFinallyNestedInTry.csproj - baseservices\exceptions\unittests\ThrowInFinallyTest.csproj - baseservices\exceptions\unittests\ThrowInCatch.csproj - baseservices\exceptions\unittests\ReturnFromCatch.csproj - baseservices\exceptions\unittests\RethrowAndFinally.csproj - baseservices\exceptions\unittests\RecursiveThrowNew.csproj - baseservices\exceptions\unittests\RecursiveRethrow.csproj - baseservices\exceptions\unittests\Recurse.csproj - baseservices\exceptions\unittests\Pending.csproj - baseservices\exceptions\unittests\InnerFinallyAndCatch.csproj - baseservices\exceptions\unittests\InnerFinally.csproj - baseservices\exceptions\unittests\GoryNativePast.csproj - baseservices\exceptions\unittests\GoryManagedPresent.csproj - baseservices\exceptions\unittests\CollidedUnwind.csproj --- src/coreclr/jit/compiler.h | 8 +- src/coreclr/jit/compiler.hpp | 6 +- src/coreclr/jit/fgdiagnostic.cpp | 2 + src/coreclr/jit/jiteh.h | 3 +- src/coreclr/jit/llvm.cpp | 18 +- src/coreclr/jit/llvm.h | 56 +- src/coreclr/jit/llvmcodegen.cpp | 867 +++++++++++++++--- src/coreclr/jit/llvmlower.cpp | 106 ++- .../IL/ILImporter.Scanner.cs | 3 + .../Compiler/LLVMCodegenCompilation.cs | 19 +- .../JitInterface/CorInfoImpl.Llvm.cs | 44 +- 11 files changed, 932 insertions(+), 200 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index ec8883a75706..f63bc8152333 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4362,6 +4362,8 @@ class Compiler bool isExplicitTailCall, IL_OFFSET ilOffset = BAD_IL_OFFSET); + void impResolveToken(const BYTE* addr, CORINFO_RESOLVED_TOKEN* pResolvedToken, CorInfoTokenKind kind); + //========================================================================= // PROTECTED //========================================================================= @@ -4382,8 +4384,6 @@ class Compiler bool impIsPrimitive(CorInfoType type); bool impILConsumesAddr(const BYTE* codeAddr); - void impResolveToken(const BYTE* addr, CORINFO_RESOLVED_TOKEN* pResolvedToken, CorInfoTokenKind kind); - void impPushOnStack(GenTree* tree, typeInfo ti); void impPushNullObjRefOnStack(); StackEntry impPopStack(); @@ -6635,6 +6635,8 @@ class Compiler AddCodeDsc* fgFindExcptnTarget(SpecialCodeKind kind, unsigned refData); + bool fgIsThrowHlpBlk(BasicBlock* block); + bool fgUseThrowHelperBlocks(); AddCodeDsc* fgGetAdditionalCodeDescriptors() @@ -6645,8 +6647,6 @@ class Compiler private: bool fgIsCodeAdded(); - bool fgIsThrowHlpBlk(BasicBlock* block); - #if !FEATURE_FIXED_OUT_ARGS unsigned fgThrowHlpBlkStkLevel(BasicBlock* block); #endif // !FEATURE_FIXED_OUT_ARGS diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 27b8cc0ed583..388c9b2f66e2 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -2705,7 +2705,8 @@ inline bool Compiler::fgIsThrowHlpBlk(BasicBlock* block) (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROWDIVZERO)) || (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROW_ARGUMENTEXCEPTION)) || (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROW_ARGUMENTOUTOFRANGEEXCEPTION)) || - (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_OVERFLOW)))) + (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_OVERFLOW)) || + (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROWNULLREF)))) { return false; } @@ -2719,7 +2720,8 @@ inline bool Compiler::fgIsThrowHlpBlk(BasicBlock* block) if (block == add->acdDstBlk) { return add->acdKind == SCK_RNGCHK_FAIL || add->acdKind == SCK_DIV_BY_ZERO || add->acdKind == SCK_OVERFLOW || - add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN; + add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN || + add->acdKind == SCK_NULL_REF_EXCPN; } } diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index 066ef8bfae19..7b5d6f3077c5 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -1660,6 +1660,8 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) case EH_HANDLER_FAULT_WAS_FINALLY: handlerType = "fault-was-finally"; break; + default: + unreached(); } sprintf_s(name, sizeof(name), "EH#%u %s", XTnum, handlerType); rgnGraph.Insert(name, RegionGraph::RegionType::EH, ehDsc->ebdHndBeg, ehDsc->ebdHndLast); diff --git a/src/coreclr/jit/jiteh.h b/src/coreclr/jit/jiteh.h index 51dcad7d9094..cddfab4ef3f7 100644 --- a/src/coreclr/jit/jiteh.h +++ b/src/coreclr/jit/jiteh.h @@ -27,7 +27,8 @@ enum EHHandlerType EH_HANDLER_FILTER, EH_HANDLER_FAULT, EH_HANDLER_FINALLY, - EH_HANDLER_FAULT_WAS_FINALLY + EH_HANDLER_FAULT_WAS_FINALLY, + EH_HANDLER_COUNT }; // ToCORINFO_EH_CLAUSE_FLAGS: Convert an internal EHHandlerType to a CORINFO_EH_CLAUSE_FLAGS value diff --git a/src/coreclr/jit/llvm.cpp b/src/coreclr/jit/llvm.cpp index 41e218d7c8c2..e2b40a9d9880 100644 --- a/src/coreclr/jit/llvm.cpp +++ b/src/coreclr/jit/llvm.cpp @@ -23,6 +23,7 @@ enum class EEApiId { GetMangledMethodName, GetSymbolMangledName, + GetEHDispatchFunctionName, GetTypeName, AddCodeReloc, IsRuntimeImport, @@ -102,7 +103,6 @@ Llvm::Llvm(Compiler* compiler) _info(compiler->info), _sigInfo(compiler->info.compMethodInfo->args), _builder(_llvmContext), - _prologBuilder(_llvmContext), _blkToLlvmBlksMap(compiler->getAllocator(CMK_Codegen)), _sdsuMap(compiler->getAllocator(CMK_Codegen)), _localsMap(compiler->getAllocator(CMK_Codegen)), @@ -167,6 +167,7 @@ GCInfo* Llvm::getGCInfo() { _gcInfo = new (_compiler->getAllocator(CMK_GC)) GCInfo(_compiler); } + return _gcInfo; } @@ -369,8 +370,8 @@ const HelperFuncInfo& Llvm::getHelperFuncInfo(CorInfoHelpFunc helperFunc) // For WASM, currently implemented in the bootstrapper... { FUNC(CORINFO_HELP_THROW) CORINFO_TYPE_VOID, { CORINFO_TYPE_PTR } }, - // (Not) implemented in "Runtime\EHHelpers.cpp" - { FUNC(CORINFO_HELP_RETHROW) CORINFO_TYPE_VOID, { } }, + // Implemented in "Runtime.Base\src\System\Runtime\ExceptionHandling.wasm.cs". + { FUNC(CORINFO_HELP_RETHROW) CORINFO_TYPE_VOID, { CORINFO_TYPE_PTR } }, // Implemented in "Runtime\MiscHelpers.cpp". { FUNC(CORINFO_HELP_USER_BREAKPOINT) CORINFO_TYPE_VOID, { } }, @@ -641,11 +642,11 @@ unsigned Llvm::padNextOffset(CorInfoType corInfoType, CORINFO_CLASS_HANDLE struc [[noreturn]] void Llvm::failFunctionCompilation() { - for (Function* llvmFunc : m_functions) + for (FunctionInfo& funcInfo : m_functions) { - if (llvmFunc != nullptr) + if (funcInfo.LlvmFunction != nullptr) { - llvmFunc->deleteBody(); + funcInfo.LlvmFunction->deleteBody(); } } @@ -668,6 +669,11 @@ const char* Llvm::GetMangledSymbolName(void* symbol) return CallEEApi(symbol); } +const char* Llvm::GetEHDispatchFunctionName(CORINFO_EH_CLAUSE_FLAGS handlerType) +{ + return CallEEApi(handlerType); +} + const char* Llvm::GetTypeName(CORINFO_CLASS_HANDLE typeHandle) { return CallEEApi(typeHandle); diff --git a/src/coreclr/jit/llvm.h b/src/coreclr/jit/llvm.h index 996edf519c3c..b3d071e55630 100644 --- a/src/coreclr/jit/llvm.h +++ b/src/coreclr/jit/llvm.h @@ -110,6 +110,17 @@ struct LlvmBlockRange } }; +typedef JitHashTable, llvm::AllocaInst*> AllocaMap; + +struct FunctionInfo +{ + Function* LlvmFunction; + union { + llvm::AllocaInst** Allocas; // Dense "lclNum -> Alloca*" mapping used for the main function. + AllocaMap* AllocaMap; // Sparse "lclNum -> Alloca*" mapping used for funclets. + }; +}; + // TODO: We should create a Static... class to manage the globals and their lifetimes. // Note we declare all statics here, and define them in llvm.cpp, for documentation and // visibility purposes even as some are only needed in other compilation units. @@ -133,16 +144,15 @@ class Llvm BasicBlock* m_currentBlock; DebugInfo _currentOffset; llvm::IRBuilder<> _builder; - llvm::IRBuilder<> _prologBuilder; JitHashTable, LlvmBlockRange> _blkToLlvmBlksMap; JitHashTable, Value*> _sdsuMap; JitHashTable _localsMap; std::vector _phiPairs; - std::vector m_allocas; - std::vector m_functions; + std::vector m_functions; std::vector m_EHDispatchLlvmBlocks; // Codegen emit context. + unsigned m_currentLlvmFunctionIndex = ROOT_FUNC_IDX; unsigned m_currentProtectedRegionIndex = EHblkDsc::NO_ENCLOSING_INDEX; LlvmBlockRange* m_currentLlvmBlocks = nullptr; @@ -198,6 +208,7 @@ class Llvm // const char* GetMangledMethodName(CORINFO_METHOD_HANDLE methodHandle); const char* GetMangledSymbolName(void* symbol); + const char* GetEHDispatchFunctionName(CORINFO_EH_CLAUSE_FLAGS handlerType); const char* GetTypeName(CORINFO_CLASS_HANDLE typeHandle); void AddCodeReloc(void* handle); bool IsRuntimeImport(CORINFO_METHOD_HANDLE methodHandle); @@ -247,6 +258,8 @@ class Llvm void lowerFieldOfDependentlyPromotedStruct(GenTree* node); void ConvertShadowStackLocalNode(GenTreeLclVarCommon* node); void lowerCall(GenTreeCall* callNode); + void lowerRethrow(GenTreeCall* callNode); + void lowerCatchArg(GenTree* catchArgNode); void lowerIndir(GenTreeIndir* indirNode); void lowerStoreBlk(GenTreeBlk* storeBlkNode); void lowerDivMod(GenTreeOp* divModNode); @@ -265,6 +278,11 @@ class Llvm bool isShadowFrameLocal(LclVarDsc* varDsc) const; bool isFuncletParameter(unsigned lclNum) const; + unsigned getCurrentShadowFrameSize() const; + unsigned getShadowFrameSize(unsigned hndIndex) const; + unsigned getOriginalShadowFrameSize() const; + unsigned getCatchArgOffset() const; + // ================================================================================================================ // | Codegen | // ================================================================================================================ @@ -273,14 +291,14 @@ class Llvm void Compile(); private: - const int ROOT_FUNC_IDX = 0; + const unsigned ROOT_FUNC_IDX = 0; bool initializeFunctions(); void generateProlog(); void initializeLocals(); - void generateFuncletProlog(unsigned funcIdx); void generateBlock(BasicBlock* block); void generateEHDispatch(); + Value* generateEHDispatchTable(Function* llvmFunc, unsigned innerEHIndex, unsigned outerEHIndex); void fillPhis(); Value* getGenTreeValue(GenTree* node); @@ -315,14 +333,12 @@ class Llvm void buildBinaryOperation(GenTree* node); void buildShift(GenTreeOp* node); void buildReturn(GenTree* node); - void buildCatchArg(GenTree* node); void buildJTrue(GenTree* node); void buildSwitch(GenTreeUnOp* switchNode); void buildNullCheck(GenTreeIndir* nullCheckNode); void buildBoundsCheck(GenTreeBoundsChk* boundsCheck); void buildCallFinally(BasicBlock* block); - void buildCatchReturn(BasicBlock* block); void storeObjAtAddress(Value* baseAddress, Value* data, StructDesc* structDesc); unsigned buildMemCpy(Value* baseAddress, unsigned startOffset, unsigned endOffset, Value* srcAddress); @@ -332,14 +348,17 @@ class Llvm void emitNullCheckForIndir(GenTreeIndir* indir, Value* addrValue); Value* emitCheckedArithmeticOperation(llvm::Intrinsic::ID intrinsicId, Value* op1Value, Value* op2Value); Value* emitHelperCall(CorInfoHelpFunc helperFunc, ArrayRef sigArgs = { }); - Value* emitCallOrInvoke(llvm::FunctionCallee callee, ArrayRef args); + llvm::CallBase* emitCallOrInvoke(llvm::FunctionCallee callee, ArrayRef args); FunctionType* getFunctionType(); Function* getOrCreateLlvmFunction(const char* symbolName, GenTreeCall* call); FunctionType* createFunctionTypeForCall(GenTreeCall* call); FunctionType* createFunctionTypeForHelper(CorInfoHelpFunc helperFunc); - Value* getOrCreateExternalSymbol(const char* symbolName, Type* symbolType = nullptr); + llvm::GlobalVariable* getOrCreateExternalSymbol(const char* symbolName, Type* symbolType = nullptr); + llvm::GlobalVariable* getOrCreateSymbol(CORINFO_GENERIC_HANDLE symbolHandle, Type* symbolType = nullptr); + Value* emitSymbolRef(CORINFO_GENERIC_HANDLE symbolHandle); + CORINFO_GENERIC_HANDLE getSymbolHandleForClassToken(mdToken token); Instruction* getCast(Value* source, Type* targetType); Value* castIfNecessary(Value* source, Type* targetType, llvm::IRBuilder<>* builder = nullptr); @@ -350,27 +369,32 @@ class Llvm DebugMetadata getOrCreateDebugMetadata(const char* documentFileName); llvm::DILocation* createDebugFunctionAndDiLocation(struct DebugMetadata debugMetadata, unsigned int lineNo); + llvm::DILocation* getArtificialDebugLocation(); void setCurrentEmitContextForBlock(BasicBlock* block); - void setCurrentEmitContext(LlvmBlockRange* llvmBlock, unsigned tryIndex); - LlvmBlockRange* getCurrentLlvmBlocks() const; + void setCurrentEmitContext(unsigned funcIdx, unsigned tryIndex, LlvmBlockRange* llvmBlock); + unsigned getCurrentLlvmFunctionIndex() const; unsigned getCurrentProtectedRegionIndex() const; + LlvmBlockRange* getCurrentLlvmBlocks() const; Function* getRootLlvmFunction(); Function* getCurrentLlvmFunction(); Function* getLlvmFunctionForIndex(unsigned funcIdx); - unsigned getLlvmFunctionIndexForBlock(BasicBlock* block); - unsigned getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex); + FunctionInfo& getLlvmFunctionInfoForIndex(unsigned funcIdx); + unsigned getLlvmFunctionIndexForBlock(BasicBlock* block) const; + unsigned getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex) const; llvm::BasicBlock* createInlineLlvmBlock(); llvm::BasicBlock* getCurrentLlvmBlock() const; LlvmBlockRange* getLlvmBlocksForBlock(BasicBlock* block); llvm::BasicBlock* getFirstLlvmBlockForBlock(BasicBlock* block); llvm::BasicBlock* getLastLlvmBlockForBlock(BasicBlock* block); + llvm::BasicBlock* getOrCreatePrologLlvmBlockForFunction(unsigned funcIdx); + + bool isReachable(BasicBlock* block) const; + BasicBlock* getFirstBlockForFunction(unsigned funcIdx) const; Value* getLocalAddr(unsigned lclNum); - unsigned getTotalLocalOffset(); + Value* getOrCreateAllocaForLocalInFunclet(unsigned lclNum); }; - - #endif /* End of _LLVM_H_ */ diff --git a/src/coreclr/jit/llvmcodegen.cpp b/src/coreclr/jit/llvmcodegen.cpp index 161d7b710e27..a807cdea0ae1 100644 --- a/src/coreclr/jit/llvmcodegen.cpp +++ b/src/coreclr/jit/llvmcodegen.cpp @@ -20,7 +20,7 @@ void Llvm::Compile() if ((_info.compFlags & CORINFO_FLG_SYNCH) != 0) { - // TODO-LLVM-EH: enable. + // TODO-LLVM: enable. failFunctionCompilation(); } @@ -86,13 +86,13 @@ void Llvm::Compile() JITDUMP("LLVM IR for %s after codegen:\n", _compiler->info.compFullName); JITDUMP("-------------------------------------------------------------------------------------------------------------------\n\n"); - for (Function* llvmFunc : m_functions) + for (FunctionInfo& funcInfo : m_functions) { - JITDUMPEXEC(llvmFunc->dump()); - - if (llvm::verifyFunction(*llvmFunc, &llvm::errs())) + Function* llvmFunc = funcInfo.LlvmFunction; + if (llvmFunc != nullptr) { - printf("Function failed: '%s'\n", llvmFunc->getName().data()); + JITDUMPEXEC(llvmFunc->dump()); + assert(!llvm::verifyFunction(*llvmFunc, &llvm::errs())); } } #endif @@ -122,8 +122,8 @@ bool Llvm::initializeFunctions() } // First functions is always the root. - m_functions = std::vector(_compiler->compFuncCount()); - m_functions[ROOT_FUNC_IDX] = rootLlvmFunction; + m_functions = std::vector(_compiler->compFuncCount()); + m_functions[ROOT_FUNC_IDX] = {rootLlvmFunction}; m_EHDispatchLlvmBlocks = std::vector(_compiler->compHndBBtabCount); @@ -134,48 +134,50 @@ bool Llvm::initializeFunctions() unsigned ehIndex = funcInfo->funEHIndex; EHblkDsc* ehDsc = _compiler->ehGetDsc(ehIndex); - // Filter and catch handler funclets return int32. "HasCatchHandler" handles both cases. - Type* retLlvmType; - if (ehDsc->HasCatchHandler()) - { - retLlvmType = Type::getInt32Ty(_llvmContext); - } - else + // We won't generate code for unreachable handlers so we will not create functions for them. + // + if (isReachable(getFirstBlockForFunction(funcIdx))) { - retLlvmType = Type::getVoidTy(_llvmContext); - } + // Filter and catch handler funclets return int32. "HasCatchHandler" handles both cases. + Type* retLlvmType = + ehDsc->HasCatchHandler() ? Type::getInt32Ty(_llvmContext) : Type::getVoidTy(_llvmContext); - // All funclets two arguments: original and actual shadow stacks. - Type* ptrLlvmType = Type::getInt8PtrTy(_llvmContext); - FunctionType* llvmFuncType = FunctionType::get(retLlvmType, {ptrLlvmType, ptrLlvmType}, /* isVarArg */ false); + // All funclets have two arguments: original and actual shadow stacks. + Type* ptrLlvmType = Type::getInt8PtrTy(_llvmContext); + FunctionType* llvmFuncType = + FunctionType::get(retLlvmType, {ptrLlvmType, ptrLlvmType}, /* isVarArg */ false); - const char* kindName; - switch (ehDsc->ebdHandlerType) - { - case EH_HANDLER_CATCH: - kindName = "Catch"; - break; - case EH_HANDLER_FILTER: - kindName = (funcInfo->funKind == FUNC_FILTER) ? "Filter" : "FilteredCatch"; - break; - case EH_HANDLER_FAULT: - case EH_HANDLER_FAULT_WAS_FINALLY: - kindName = "Fault"; - break; - case EH_HANDLER_FINALLY: - kindName = "Finally"; - break; - default: - unreached(); - } + const char* kindName; + switch (ehDsc->ebdHandlerType) + { + case EH_HANDLER_CATCH: + kindName = "Catch"; + break; + case EH_HANDLER_FILTER: + kindName = (funcInfo->funKind == FUNC_FILTER) ? "Filter" : "FilteredCatch"; + break; + case EH_HANDLER_FAULT: + case EH_HANDLER_FAULT_WAS_FINALLY: + kindName = "Fault"; + break; + case EH_HANDLER_FINALLY: + kindName = "Finally"; + break; + default: + unreached(); + } - Function* llvmFunc = - Function::Create(llvmFuncType, Function::InternalLinkage, - mangledName + llvm::Twine("$F") + llvm::Twine(funcIdx) + "_" + kindName, _module); + Function* llvmFunc = + Function::Create(llvmFuncType, Function::InternalLinkage, + mangledName + llvm::Twine("$F") + llvm::Twine(funcIdx) + "_" + kindName, _module); + + m_functions[funcIdx] = {llvmFunc}; + } // Note that "mutually-protect" handlers will share the same dispatch block. We only need to associate - // one dispatch block with one protected region, and so simply skip the logic for filter funclets. - if (funcInfo->funKind != FUNC_FILTER) + // one dispatch block with one protected region, and so simply skip the logic for filter funclets. We + // also leave blocks for unreachable dispatches null. + if ((funcInfo->funKind == FUNC_HANDLER) && isReachable(ehDsc->ExFlowBlock())) { llvm::BasicBlock* dispatchLlvmBlock = nullptr; @@ -200,8 +202,6 @@ bool Llvm::initializeFunctions() m_EHDispatchLlvmBlocks[ehIndex] = dispatchLlvmBlock; } - - m_functions[funcIdx] = llvmFunc; } return false; @@ -211,20 +211,15 @@ void Llvm::generateProlog() { JITDUMP("\n=============== Generating prolog:\n"); - llvm::BasicBlock* prologLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "Prolog", getRootLlvmFunction()); - _prologBuilder.SetInsertPoint(prologLlvmBlock); + llvm::BasicBlock* prologLlvmBlock = getOrCreatePrologLlvmBlockForFunction(ROOT_FUNC_IDX); + _builder.SetInsertPoint(prologLlvmBlock->getTerminator()); initializeLocals(); - - llvm::BasicBlock* firstLlvmBlock = getFirstLlvmBlockForBlock(_compiler->fgFirstBB); - _prologBuilder.CreateBr(firstLlvmBlock); - - _builder.SetInsertPoint(firstLlvmBlock); } void Llvm::initializeLocals() { - m_allocas = std::vector(_compiler->lvaCount, nullptr); + llvm::AllocaInst** allocas = new (_compiler->getAllocator(CMK_Codegen)) llvm::AllocaInst*[_compiler->lvaCount]; for (unsigned lclNum = 0; lclNum < _compiler->lvaCount; lclNum++) { @@ -306,7 +301,7 @@ void Llvm::initializeLocals() if (initValue == nullptr) { initValue = llvm::UndefValue::get(lclLlvmType); - initValue = _prologBuilder.CreateFreeze(initValue); + initValue = _builder.CreateFreeze(initValue); JITDUMPEXEC(initValue->dump()); } @@ -318,19 +313,16 @@ void Llvm::initializeLocals() } else { - AllocaInst* allocaInst = _prologBuilder.CreateAlloca(lclLlvmType); - m_allocas[lclNum] = allocaInst; + llvm::AllocaInst* allocaInst = _builder.CreateAlloca(lclLlvmType); + allocas[lclNum] = allocaInst; JITDUMPEXEC(allocaInst->dump()); - Instruction* storeInst = _prologBuilder.CreateStore(initValue, allocaInst); + Instruction* storeInst = _builder.CreateStore(initValue, allocaInst); JITDUMPEXEC(storeInst->dump()); } } -} -void Llvm::generateFuncletProlog(unsigned funcIdx) -{ - // TODO-LLVM-EH: create allocas for all tracked locals that are not in SSA or the shadow stack. + getLlvmFunctionInfoForIndex(ROOT_FUNC_IDX).Allocas = allocas; } void Llvm::generateBlock(BasicBlock* block) @@ -368,9 +360,6 @@ void Llvm::generateBlock(BasicBlock* block) _builder.CreateRetVoid(); } break; - case BBJ_EHCATCHRET: - buildCatchReturn(block); - break; default: // TODO-LLVM: other jump kinds. break; @@ -388,7 +377,8 @@ void Llvm::generateEHDispatch() // Recover the C++ personality function. Type* ptrLlvmType = Type::getInt8PtrTy(_llvmContext); Type* int32LlvmType = Type::getInt32Ty(_llvmContext); - Type* cppExcTupleLlvmType = llvm::StructType::get(_llvmContext, {ptrLlvmType, int32LlvmType}); + Type* cppExcTupleLlvmType = llvm::StructType::get(ptrLlvmType, int32LlvmType); + llvm::StructType* dispatchDataLlvmType = llvm::StructType::get(cppExcTupleLlvmType, ptrLlvmType); static const char* const GXX_PERSONALITY_NAME = "__gxx_personality_v0"; Function* gxxPersonalityLlvmFunc = _module->getFunction(GXX_PERSONALITY_NAME); @@ -400,54 +390,501 @@ void Llvm::generateEHDispatch() Function::Create(gxxPersonalityLlvmFuncType, Function::ExternalLinkage, GXX_PERSONALITY_NAME, _module); } - JITDUMP("\n"); + // Recover the runtime helper routines. TODO-LLVM: use proper "CorInfoHelpFunc"s for these. + Function* dispatchLlvmFuncs[EH_HANDLER_COUNT]{}; + auto getDispatchLlvmFunc = [&](EHblkDsc* hndDsc) { + Function** pDispatchLlvmFunc; + if (hndDsc == nullptr) + { + static_assert_no_msg(EH_HANDLER_CATCH == 1); // We rely on zero being invalid. + pDispatchLlvmFunc = &dispatchLlvmFuncs[0]; + } + else + { + pDispatchLlvmFunc = &dispatchLlvmFuncs[hndDsc->ebdHandlerType]; + } + + Function* dispatchLlvmFunc = *pDispatchLlvmFunc; + if (dispatchLlvmFunc == nullptr) + { + CORINFO_EH_CLAUSE_FLAGS eeHndType = (hndDsc == nullptr) ? CORINFO_EH_CLAUSE_SAMETRY + : ToCORINFO_EH_CLAUSE_FLAGS(hndDsc->ebdHandlerType); + const char* dispatchFuncName = GetEHDispatchFunctionName(eeHndType); + dispatchLlvmFunc = _module->getFunction(dispatchFuncName); + if (dispatchLlvmFunc == nullptr) + { + FunctionType* dispatchLlvmFuncType; + switch (eeHndType) + { + case CORINFO_EH_CLAUSE_NONE: + case CORINFO_EH_CLAUSE_FILTER: + dispatchLlvmFuncType = FunctionType::get( + int32LlvmType, {ptrLlvmType, ptrLlvmType, ptrLlvmType, ptrLlvmType, ptrLlvmType}, + /* isVarArg */ false); + break; + case CORINFO_EH_CLAUSE_SAMETRY: + dispatchLlvmFuncType = FunctionType::get( + int32LlvmType, {ptrLlvmType, ptrLlvmType, ptrLlvmType, ptrLlvmType}, /* isVarArg */ false); + break; + case CORINFO_EH_CLAUSE_FAULT: + case CORINFO_EH_CLAUSE_FINALLY: + dispatchLlvmFuncType = FunctionType::get( + Type::getVoidTy(_llvmContext), {ptrLlvmType, ptrLlvmType, ptrLlvmType, ptrLlvmType}, + /* isVarArg */ false); + break; + default: + unreached(); + } + + dispatchLlvmFunc = + Function::Create(dispatchLlvmFuncType, Function::ExternalLinkage, dispatchFuncName, _module); + *pDispatchLlvmFunc = dispatchLlvmFunc; + } + } + + return dispatchLlvmFunc; + }; - for (unsigned ehIndex = 0; ehIndex < _compiler->compHndBBtabCount; ehIndex++) + BitVecTraits blockVecTraits(_compiler->fgBBNumMax + 1, _compiler); + + struct DispatchData + { + llvm::SwitchInst* DispatchSwitchInst; + BitVec DispatchSwitchTargets; + unsigned LastDispatchSwitchTargetIndex; + llvm::BasicBlock* ResumeLlvmBlock; + Value* DispatchDataRefValue; + + llvm::BasicBlock* GetDispatchSwitchLlvmBlock() const + { + return (DispatchSwitchInst != nullptr) ? DispatchSwitchInst->getParent() : nullptr; + } + }; + + // There is no meaningful source location we can attach to the dispatch blocks. None of them are "user" code. + llvm::DebugLoc dispatchDebugLoc = getArtificialDebugLocation(); + std::vector functionData(_compiler->compFuncCount()); + + // Note the iteration order: outer -> inner. + for (unsigned ehIndex = _compiler->compHndBBtabCount - 1; ehIndex != -1; ehIndex--) { EHblkDsc* ehDsc = _compiler->ehGetDsc(ehIndex); - llvm::BasicBlock* llvmBlock = m_EHDispatchLlvmBlocks[ehIndex]; + llvm::BasicBlock* dispatchPadLlvmBlock = m_EHDispatchLlvmBlocks[ehIndex]; - if (!llvmBlock->empty()) + if (dispatchPadLlvmBlock == nullptr) { - // We've already generated code for this dispatch shared between "mutual protect" handlers. + // Would have been unreachable. continue; } - JITDUMP("=============== Generating EH dispatch block %s:\n", llvmBlock->getName().data()); - - if (llvmBlock->hasNPredecessors(0)) + if (!dispatchPadLlvmBlock->empty()) { - // During codegen we have discovered there are no (EH) edges to this dispatch. Remove it. - JITDUMP("\nUnreachable; skipped\n"); - llvmBlock->eraseFromParent(); + // We've already generated code for this dispatch shared between "mutual protect" handlers. continue; } - Function* llvmFunc = llvmBlock->getParent(); + unsigned funcIdx = getLlvmFunctionIndexForProtectedRegion(ehIndex); + Function* llvmFunc = getLlvmFunctionForIndex(funcIdx); if (!llvmFunc->hasPersonalityFn()) { llvmFunc->setPersonalityFn(gxxPersonalityLlvmFunc); } - // Move it after the last block in the corresponding protected region. - llvmBlock->moveAfter(getLastLlvmBlockForBlock(ehDsc->ebdTryLast)); + // The code we will generate effectively inlines the usual runtime dispatch logic. The main reason this + // scheme was chosen is the fact (re)throwing funclets are handled by it semlessly and efficiently. The + // downside to it is the code size overhead of the calls made for each protected region. + // + // DISPATCH_PAD_INNER: + // dispatchData.CppExceptionTuple = landingPadInst + // dispatchData.DispatcherData = null + // goto DISPATCH_INNER; + // + // DISPATCH_INNER: + // dispatchDest = DispatchFunction(FuncletShadowStack(), &dispatchData, &HandlerFunclet, ...) + // unwind to DISPATCH_PAD_OUTER + // if (dispatchDest == 0) + // goto DISPATCH_OUTER; // For nested regions; top-level ones will use the "switch". + // goto UNIFIED_DISPATCH; + // + // UNIFIED_DISPATCH: + // switch (dispatchDest) { + // case 0: goto RESUME; + // case 1: goto BB01; + // case 2: goto BB02; + // ... + // default: goto FAIL_FAST; + // } + // + // RESUME: + // resume(dispatchData.CppExceptionTuple); // Rethrow the exception and unwind to caller. + // + // FAIL_FAST: + // FailFast(); + // + // What is the possibe set of dispatch destinations (aka why have "UNIFIED_DISPATCH")? + // + // We consider the tree of active protected regions above this one, that are also contained in the same funclet. + // For each region with a (possibly filtered) catch handler, we consider successors of all "catchret" blocks. + // The union of these will form the set of all possible dispatch destinations for the current protected region. + // However, we do not actually emit the "switch" code for each individual region, as it would mean quadratic + // code size growth (number of dispatch destinations X number of protected regions) for deeply nested EH trees. + // Instead, we create one "universal" dispatch block for each funclet, and jump to it from each dispatch. Note + // that thanks to the step blocks inserted by "impImportLeave", we do not need to consider cases where a jump + // from a funclet to its caller would be required. + + // Create the dispatch data alloca. Its structure is a contract between codegen and runtime. The runtime may + // not modify the part where codegen stores the landing pad value, while the other part will be solely under + // runtime's control (currently, this is just one pointer-sized field). + DispatchData& funcDispatchData = functionData[funcIdx]; + Value* dispatchDataRefValue = funcDispatchData.DispatchDataRefValue; + if (dispatchDataRefValue == nullptr) + { + llvm::BasicBlock* prologLlvmBlock = getOrCreatePrologLlvmBlockForFunction(funcIdx); - _builder.SetInsertPoint(llvmBlock); + _builder.SetInsertPoint(prologLlvmBlock->getTerminator()); + dispatchDataRefValue = _builder.CreateAlloca(dispatchDataLlvmType); + + funcDispatchData.DispatchDataRefValue = dispatchDataRefValue; + } + + // Dispatch blocks, when initially created, are placed at the start of the function. + // Here we move them to a more appropriate place so that the entry block is correct. + if (funcDispatchData.GetDispatchSwitchLlvmBlock() != nullptr) + { + dispatchPadLlvmBlock->moveBefore(funcDispatchData.GetDispatchSwitchLlvmBlock()); + } + else if (funcDispatchData.ResumeLlvmBlock != nullptr) + { + dispatchPadLlvmBlock->moveBefore(funcDispatchData.ResumeLlvmBlock); + } + else + { + dispatchPadLlvmBlock->moveAfter(&llvmFunc->back()); + } + _builder.SetCurrentDebugLocation(dispatchDebugLoc); + + LlvmBlockRange dispatchLlvmBlocks(dispatchPadLlvmBlock); + setCurrentEmitContext(funcIdx, ehDsc->ebdEnclosingTryIndex, &dispatchLlvmBlocks); llvm::LandingPadInst* landingPadInst = _builder.CreateLandingPad(cppExcTupleLlvmType, 1); landingPadInst->addClause(llvm::Constant::getNullValue(ptrLlvmType)); // Catch all C++ exceptions. - // TODO-LLVM-EH: implement. - _builder.CreateResume(landingPadInst); + _builder.CreateStore(landingPadInst, dispatchDataRefValue); + + // The dispatchers rely on this being set to null to detect whether the ongoing dispatch is already "active". + unsigned dispatcherDataFieldOffset = + _module->getDataLayout().getStructLayout(dispatchDataLlvmType)->getElementOffset(1); + Value* dispatchDataFieldRefValue = gepOrAddr(dispatchDataRefValue, dispatcherDataFieldOffset); + _builder.CreateStore(llvm::Constant::getNullValue(ptrLlvmType), dispatchDataFieldRefValue); + + // The "actual" dispatch block. Nested dispatches (if any) will branch to it. + llvm::BasicBlock* dispatchLlvmBlock = createInlineLlvmBlock(); + _builder.CreateBr(dispatchLlvmBlock); + _builder.SetInsertPoint(dispatchLlvmBlock); + + // The dispatcher uses the passed-in shadow stack pointer to call funclets. All funclets (no matter how + // nested) share the same original shadow frame, thus we need to pass the original shadow stack in case + // the exception is being dispatched out of a funclet. + Value* dispatcherShadowStackValue = getShadowStackForCallee(); + Value* funcletShadowStackValue = getOriginalShadowStack(); + + // Do we only have one (catch) handler? We will use specialized dispatchers for this case as an optimization: + // about 2/3 of all EH handlers in optimized code are finallys/faults, ~28% - single catches, with the rest + // (single filters / 2+ mutually protecting handlers) comprising less than 5% of cases. We could drop the + // specialized filter dispatcher here, but it doesn't cost us much to have one, and it is considerably more + // efficient than the general table-based one (and more than 4/5 of all filters are "single"). + // + unsigned innerEHIndex = ehIndex; + while ((innerEHIndex > 0) && ehDsc->ebdIsSameTry(_compiler, innerEHIndex - 1)) + { + innerEHIndex--; + } + + llvm::CallBase* dispatchDestValue = nullptr; + if (innerEHIndex == ehIndex) + { + Function* dispatchLlvmFunc = getDispatchLlvmFunc(ehDsc); + Value* handlerValue = getLlvmFunctionForIndex(ehDsc->ebdFuncIndex); -#ifdef DEBUG - llvm::BasicBlock* lastLlvmBlock = llvmBlock; - for (; llvmBlock != lastLlvmBlock->getNextNode(); llvmBlock = llvmBlock->getNextNode()) + if (ehDsc->ebdHandlerType == EH_HANDLER_CATCH) + { + Value* typeSymbolRefValue = emitSymbolRef(getSymbolHandleForClassToken(ehDsc->ebdTyp)); + dispatchDestValue = + emitCallOrInvoke(dispatchLlvmFunc, {dispatcherShadowStackValue, funcletShadowStackValue, + dispatchDataRefValue, handlerValue, typeSymbolRefValue}); + } + else if (ehDsc->ebdHandlerType == EH_HANDLER_FILTER) + { + Value* filterValue = getLlvmFunctionForIndex(ehDsc->ebdFuncIndex - 1); + dispatchDestValue = + emitCallOrInvoke(dispatchLlvmFunc, {dispatcherShadowStackValue, funcletShadowStackValue, + dispatchDataRefValue, handlerValue, filterValue}); + } + else + { + dispatchDestValue = emitCallOrInvoke(dispatchLlvmFunc, {dispatcherShadowStackValue, + funcletShadowStackValue, dispatchDataRefValue, handlerValue}); + } + } + else { - JITDUMPEXEC(llvmBlock->dump()); + Function* dispatchLlvmFunc = getDispatchLlvmFunc(nullptr); + Value* dispatchTableRefValue = generateEHDispatchTable(llvmFunc, innerEHIndex, ehIndex); + dispatchDestValue = + emitCallOrInvoke(dispatchLlvmFunc, {dispatcherShadowStackValue, funcletShadowStackValue, + dispatchDataRefValue, dispatchTableRefValue}); } -#endif // DEBUG + + // Generate code for per-funclet dispatch blocks. The dispatch switch block is only needed if we have + // catch handlers. The resume block is always needed. + // + llvm::BasicBlock* resumeLlvmBlock = funcDispatchData.ResumeLlvmBlock; + if (resumeLlvmBlock == nullptr) + { + resumeLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "BBDR", llvmFunc); + + _builder.SetInsertPoint(resumeLlvmBlock); // No need for a full emit context. + Value* resumeOperandValue = _builder.CreateLoad(landingPadInst->getType(), dispatchDataRefValue); + _builder.CreateResume(resumeOperandValue); + + funcDispatchData.ResumeLlvmBlock = resumeLlvmBlock; + } + + const int EH_CONTINUE_SEARCH = 0; + + llvm::BasicBlock* dispatchSwitchLlvmBlock = funcDispatchData.GetDispatchSwitchLlvmBlock(); + if (ehDsc->HasCatchHandler() && (dispatchSwitchLlvmBlock == nullptr)) + { + dispatchSwitchLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "BBDS", llvmFunc, resumeLlvmBlock); + llvm::BasicBlock* failFastLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "BBFF", llvmFunc); + + LlvmBlockRange dispatchSwitchLlvmBlocks(dispatchSwitchLlvmBlock); + setCurrentEmitContext(funcIdx, EHblkDsc::NO_ENCLOSING_INDEX, &dispatchSwitchLlvmBlocks); + + llvm::SwitchInst* switchInst = _builder.CreateSwitch(dispatchDestValue, failFastLlvmBlock); + switchInst->addCase(_builder.getInt32(EH_CONTINUE_SEARCH), resumeLlvmBlock); + + LlvmBlockRange failFastLlvmBlocks(failFastLlvmBlock); + setCurrentEmitContext(funcIdx, EHblkDsc::NO_ENCLOSING_INDEX, &failFastLlvmBlocks); + + emitHelperCall(CORINFO_HELP_FAIL_FAST); + _builder.CreateUnreachable(); + + funcDispatchData.DispatchSwitchInst = switchInst; + funcDispatchData.DispatchSwitchTargets = BitVecOps::MakeEmpty(&blockVecTraits); + } + + llvm::BasicBlock* outerDispatchLlvmBlock = nullptr; + if (llvm::isa(dispatchDestValue)) + { + // This will be the "dispatch pad" block. Since we're generating dispatches from outer to inner, we already + // have the "actual" dispatch block; it will be the next one. + outerDispatchLlvmBlock = llvm::cast(dispatchDestValue)->getUnwindDest(); + outerDispatchLlvmBlock = outerDispatchLlvmBlock->getNextNode(); + assert(outerDispatchLlvmBlock != nullptr); + } + + // Reset context back to the dispatch block. + setCurrentEmitContext(funcIdx, ehDsc->ebdEnclosingTryIndex, &dispatchLlvmBlocks); + + // For inner dispatch, jump to the outer one if the handler returned "continue search". Faults / finallys cannot + // satisfy the first-pass search and so for them this jump is unconditional. + llvm::BasicBlock* lastDispatchLlvmBlock = dispatchLlvmBlocks.LastBlock; + if (ehDsc->HasCatchHandler()) + { + Value* unifiedDispatchDestValue = funcDispatchData.DispatchSwitchInst->getOperand(0); + if (unifiedDispatchDestValue != dispatchDestValue) + { + llvm::PHINode* phiNode = llvm::dyn_cast(unifiedDispatchDestValue); + if (phiNode == nullptr) + { + phiNode = + llvm::PHINode::Create(dispatchDestValue->getType(), 2, "", funcDispatchData.DispatchSwitchInst); + phiNode->addIncoming(unifiedDispatchDestValue, dispatchSwitchLlvmBlock->getUniquePredecessor()); + + funcDispatchData.DispatchSwitchInst->setOperand(0, phiNode); + } + + phiNode->addIncoming(dispatchDestValue, lastDispatchLlvmBlock); + } + + if (outerDispatchLlvmBlock != nullptr) + { + Value* doContinueSearchValue = + _builder.CreateICmpEQ(dispatchDestValue, _builder.getInt32(EH_CONTINUE_SEARCH)); + _builder.CreateCondBr(doContinueSearchValue, outerDispatchLlvmBlock, dispatchSwitchLlvmBlock); + } + else + { + _builder.CreateBr(dispatchSwitchLlvmBlock); + } + } + else + { + if (outerDispatchLlvmBlock != nullptr) + { + _builder.CreateBr(outerDispatchLlvmBlock); + } + else + { + _builder.CreateBr(resumeLlvmBlock); + } + } + + // Finally, add in the possible "catchret" destinations. Do not forget to consider all of the mutally protecting + // handlers, since there is only one dispatch block for all of them. Note how we are only doing linear work here + // because the funclet creating process will hoist nested handlers, "flattening" the basic block list. Also, we + // check for the reachability of the handler here, even as we've already checked for whether the dispatch itself + // is reachable. The reason for this is a possibility of a dispatch with a reachable filter but an unreachable + // handler (where the filter always returns false). This is currently, technically, redundant, because RyuJit + // doesn't perform flow optimizations which would expose the handler as unreachable. We choose to be resilient + // against this anyway. + // + if (ehDsc->HasCatchHandler() && isReachable(ehDsc->ebdHndBeg)) + { + llvm::SwitchInst* switchInst = funcDispatchData.DispatchSwitchInst; + BitVec& dispatchSwitchTargets = funcDispatchData.DispatchSwitchTargets; + for (unsigned hndIndex = innerEHIndex; hndIndex <= ehIndex; hndIndex++) + { + EHblkDsc* hndDsc = _compiler->ehGetDsc(hndIndex); + for (BasicBlock* hndBlock : _compiler->Blocks(hndDsc->ebdHndBeg, hndDsc->ebdHndLast)) + { + assert((hndDsc->HasCatchHandler()) && (hndBlock->getHndIndex() == hndIndex)); + if (hndBlock->bbJumpKind == BBJ_EHCATCHRET) + { + BasicBlock* destBlock = hndBlock->bbJumpDest; + llvm::BasicBlock* destLlvmBlock = getFirstLlvmBlockForBlock(destBlock); + assert(destLlvmBlock->getParent() == llvmFunc); // No jumping out of a funclet. + + // We use a bitset to avoid quadratic behavior associated with checking if we have already added + // this dispatch destination - multiple sets of "catchret"s may target the same set of blocks. + unsigned destBlockNum = destBlock->bbNum; + if (!BitVecOps::IsMember(&blockVecTraits, dispatchSwitchTargets, destBlockNum)) + { + unsigned destIndex = ++funcDispatchData.LastDispatchSwitchTargetIndex; + llvm::ConstantInt* destIndexValue = _builder.getInt32(destIndex); + + switchInst->addCase(destIndexValue, destLlvmBlock); + + // Complete the catch return blocks (this one and all the others with the same target). + for (BasicBlock* predBlock : destBlock->PredBlocks()) + { + if (predBlock->bbJumpKind == BBJ_EHCATCHRET) + { + llvm::BasicBlock* catchRetLlvmBlock = getLastLlvmBlockForBlock(predBlock); + llvm::ReturnInst::Create(_llvmContext, destIndexValue, catchRetLlvmBlock); + } + } + + BitVecOps::AddElemD(&blockVecTraits, dispatchSwitchTargets, destBlockNum); + } + } + } + } + } + } +} + +Value* Llvm::generateEHDispatchTable(Function* llvmFunc, unsigned innerEHIndex, unsigned outerEHIndex) +{ + // We only generate this table for a run of mutually protecting handlers. + assert(outerEHIndex > innerEHIndex); + + // The table will have the following format: + // + // [2 (4) bytes: size of table in pointer units] (Means we don't support > ~2^15 clauses) + // [2 (4) bytes: bitmap of clause kinds, 0 - typed, 1 - filter] + // [up to 16 (32) clauses: { void* "Data", void* "Handler" }] + // + // - "Data": exception type symbol pointer / filter handler. + // - "Handler": pointer to the handler + // + // [4 (8) bytes: bitmap of clause kinds] [32 (64) clauses], ... + // + // This is "optimal" for the purposes of targeting WASM, where we cannot encode funclet pointers + // more efficiently using native code offsets. + // + const int LARGE_SECTION_CLAUSE_COUNT = TARGET_POINTER_SIZE * BITS_PER_BYTE; + const int FIRST_SECTION_CLAUSE_COUNT = LARGE_SECTION_CLAUSE_COUNT / 2; + + Type* firstClauseMaskType = Type::getIntNTy(_llvmContext, FIRST_SECTION_CLAUSE_COUNT); + Type* largeClauseMaskType = getIntPtrLlvmType(); + + unsigned clauseCount = outerEHIndex - innerEHIndex + 1; + ArrayStack data(_compiler->getAllocator(CMK_Codegen)); + + data.Push(nullptr); // Placeholder for size. + data.Push(nullptr); // Placeholder for the first mask. + + target_size_t clauseKindMask = 0; + unsigned baseSectionIndex = 0; + unsigned nextSectionIndex = FIRST_SECTION_CLAUSE_COUNT; + for (unsigned index = 0; index < clauseCount; index++) + { + EHblkDsc* ehDsc = _compiler->ehGetDsc(innerEHIndex + index); + unsigned clauseIndex = index - baseSectionIndex; + + llvm::Constant* dataValue; + if (ehDsc->HasFilter()) + { + clauseKindMask |= (target_size_t(1) << clauseIndex); + dataValue = getLlvmFunctionForIndex(ehDsc->ebdFuncIndex - 1); + } + else + { + // Otherwise we need a type symbol reference. + CORINFO_GENERIC_HANDLE typeSymbolHandle = getSymbolHandleForClassToken(ehDsc->ebdTyp); + dataValue = getOrCreateSymbol(typeSymbolHandle); + } + + data.Push(dataValue); + data.Push(getLlvmFunctionForIndex(ehDsc->ebdFuncIndex)); + + // Is this the last entry in the current section? Initialize the mask if so. + bool isEndOfTable = (index + 1) == clauseCount; + bool isEndOfSection = (index + 1) == nextSectionIndex; + if (isEndOfTable || isEndOfSection) + { + Type* clauseMaskType = (baseSectionIndex == 0) ? firstClauseMaskType : largeClauseMaskType; + data.TopRef(2 * (clauseIndex + 1)) = llvm::ConstantInt::get(clauseMaskType, clauseKindMask); + + // Start the next section if needed. + if (!isEndOfTable) + { + clauseKindMask = 0; + data.Push(nullptr); + + baseSectionIndex = nextSectionIndex; + nextSectionIndex += LARGE_SECTION_CLAUSE_COUNT; + } + } + } + + data.BottomRef(0) = llvm::ConstantInt::get(firstClauseMaskType, data.Height() - 1); + + ArrayStack llvmTypeBuilder(_compiler->getAllocator(CMK_Codegen), data.Height()); + for (size_t i = 0; i < data.Height(); i++) + { + llvmTypeBuilder.Push(data.Bottom(i)->getType()); + } + llvm::StructType* tableLlvmType = llvm::StructType::get(_llvmContext, {&llvmTypeBuilder.BottomRef(0), + static_cast(llvmTypeBuilder.Height())}); + llvm::Constant* tableValue = llvm::ConstantStruct::get(tableLlvmType, {&data.BottomRef(0), + static_cast(data.Height())}); + + llvm::GlobalVariable* tableRef = new llvm::GlobalVariable(*_module, tableLlvmType, /* isConstant */ true, + llvm::GlobalVariable::InternalLinkage, tableValue, + llvmFunc->getName() + "__EHTable"); + tableRef->setAlignment(llvm::MaybeAlign(TARGET_POINTER_SIZE)); + + JITDUMP("\nGenerated EH dispatch table for mutually protecting handlers:\n", innerEHIndex, outerEHIndex); + for (unsigned ehIndex = innerEHIndex; ehIndex <= outerEHIndex; ehIndex++) + { + JITDUMPEXEC(_compiler->ehGetDsc(ehIndex)->DispEntry(ehIndex)); } + JITDUMPEXEC(tableRef->dump()); + + return tableRef; } void Llvm::fillPhis() @@ -774,9 +1211,6 @@ void Llvm::visitNode(GenTree* node) case GT_RETFILT: buildReturn(node); break; - case GT_CATCH_ARG: - buildCatchArg(node); - break; case GT_STORE_LCL_VAR: buildStoreLocalVar(node->AsLclVar()); break; @@ -1376,10 +1810,8 @@ void Llvm::buildCnsInt(GenTree* node) if (node->IsIconHandle(GTF_ICON_TOKEN_HDL) || node->IsIconHandle(GTF_ICON_CLASS_HDL) || node->IsIconHandle(GTF_ICON_METHOD_HDL) || node->IsIconHandle(GTF_ICON_FIELD_HDL)) { - const char* symbolName = GetMangledSymbolName((void*)(node->AsIntCon()->IconValue())); - AddCodeReloc((void*)node->AsIntCon()->IconValue()); - mapGenTreeToValue(node, _builder.CreateLoad(llvm::PointerType::getUnqual(_llvmContext), - getOrCreateExternalSymbol(symbolName))); + CORINFO_GENERIC_HANDLE symbolHandle = CORINFO_GENERIC_HANDLE(node->AsIntCon()->IconValue()); + mapGenTreeToValue(node, emitSymbolRef(symbolHandle)); } else { @@ -1398,10 +1830,8 @@ void Llvm::buildCnsInt(GenTree* node) ssize_t intCon = node->AsIntCon()->gtIconVal; if (node->IsIconHandle(GTF_ICON_STR_HDL)) { - const char* symbolName = GetMangledSymbolName((void *)(node->AsIntCon()->IconValue())); - AddCodeReloc((void*)node->AsIntCon()->IconValue()); - mapGenTreeToValue(node, _builder.CreateLoad(Type::getInt8PtrTy(_llvmContext), - getOrCreateExternalSymbol(symbolName))); + CORINFO_GENERIC_HANDLE symbolHandle = CORINFO_GENERIC_HANDLE(node->AsIntCon()->IconValue()); + mapGenTreeToValue(node, emitSymbolRef(symbolHandle)); return; } // TODO: delete this check, just handling string constants and null ptr stores for now, other TYP_REFs not implemented yet @@ -1757,17 +2187,6 @@ void Llvm::buildReturn(GenTree* node) _builder.CreateRet(retValValue); } -void Llvm::buildCatchArg(GenTree* node) -{ - // The exception object is passed as the first shadow parameter to funclets. - Type* excLlvmType = getLlvmTypeForVarType(node->TypeGet()); - // TODO-LLVM: delete once we move to opaque pointers. - Value* shadowStackValue = _builder.CreateBitCast(getShadowStack(), excLlvmType->getPointerTo()); - Value* excObjValue = _builder.CreateLoad(excLlvmType, shadowStackValue); - - mapGenTreeToValue(node, excObjValue); -} - void Llvm::buildJTrue(GenTree* node) { Value* condValue = getGenTreeValue(node->gtGetOp1()); @@ -1842,12 +2261,12 @@ void Llvm::buildCallFinally(BasicBlock* block) // Other backends will simply skip generating the second block, while we will branch to it. // Function* finallyLlvmFunc = getLlvmFunctionForIndex(getLlvmFunctionIndexForBlock(block->bbJumpDest)); - Value* shadowStackValue = getShadowStack(); - - // For a locally invoked finally, both the original and actual shadow stacks have the same value. - emitCallOrInvoke(finallyLlvmFunc, {shadowStackValue, shadowStackValue}); + emitCallOrInvoke(finallyLlvmFunc, {getShadowStackForCallee(), getOriginalShadowStack()}); - if ((block->bbFlags & BBF_RETLESS_CALL) != 0) + // Some tricky EH flow configurations can make the ALWAYS part of the pair unreachable without + // marking "block" "BBF_RETLESS_CALL". Detect this case by checking if the next block is reachable + // at all. + if (((block->bbFlags & BBF_RETLESS_CALL) != 0) || !isReachable(block->bbNext)) { _builder.CreateUnreachable(); } @@ -1858,14 +2277,6 @@ void Llvm::buildCallFinally(BasicBlock* block) } } -void Llvm::buildCatchReturn(BasicBlock* block) -{ - assert(block->bbJumpKind == BBJ_EHCATCHRET); - - // TODO-LLVM-EH: return the appropriate value. - _builder.CreateRet(_builder.getInt32(0)); -} - void Llvm::storeObjAtAddress(Value* baseAddress, Value* data, StructDesc* structDesc) { unsigned fieldCount = structDesc->getFieldCount(); @@ -2153,21 +2564,54 @@ FunctionType* Llvm::createFunctionTypeForHelper(CorInfoHelpFunc helperFunc) return llvmFuncType; } -Value* Llvm::getOrCreateExternalSymbol(const char* symbolName, Type* symbolType) +llvm::GlobalVariable* Llvm::getOrCreateExternalSymbol(const char* symbolName, Type* symbolType) { if (symbolType == nullptr) { symbolType = Type::getInt32PtrTy(_llvmContext); } - Value* symbol = _module->getGlobalVariable(symbolName); + llvm::GlobalVariable* symbol = _module->getGlobalVariable(symbolName); if (symbol == nullptr) { - symbol = new llvm::GlobalVariable(*_module, symbolType, false, llvm::GlobalValue::LinkageTypes::ExternalLinkage, (llvm::Constant*)nullptr, symbolName); + symbol = new llvm::GlobalVariable(*_module, symbolType, false, llvm::GlobalValue::LinkageTypes::ExternalLinkage, + nullptr, symbolName); } return symbol; } +llvm::GlobalVariable* Llvm::getOrCreateSymbol(CORINFO_GENERIC_HANDLE symbolHandle, Type* symbolType) +{ + const char* symbolName = GetMangledSymbolName(symbolHandle); + AddCodeReloc(symbolHandle); + llvm::GlobalVariable* symbol = getOrCreateExternalSymbol(symbolName, symbolType); + + return symbol; +} + +Value* Llvm::emitSymbolRef(CORINFO_GENERIC_HANDLE symbolHandle) +{ + // The symbols handled here represent addresses of the relevant entities. + // So e. g. "symbolRefAddrValue" for a type handle would represent TypeHandle*. + Value* symbolRefAddrValue = getOrCreateSymbol(symbolHandle); + Value* symbolRefValue = _builder.CreateLoad(llvm::PointerType::getUnqual(_llvmContext), symbolRefAddrValue); + + return symbolRefValue; +} + +CORINFO_GENERIC_HANDLE Llvm::getSymbolHandleForClassToken(mdToken token) +{ + // The importer call here relies on RyuJit not inlining EH (which it currently does not). + CORINFO_RESOLVED_TOKEN resolvedToken; + _compiler->impResolveToken((BYTE*)&token, &resolvedToken, CORINFO_TOKENKIND_Class); + + void* pIndirection = nullptr; + CORINFO_CLASS_HANDLE typeSymbolHandle = _info.compCompHnd->embedClassHandle(resolvedToken.hClass, &pIndirection); + assert(pIndirection == nullptr); + + return CORINFO_GENERIC_HANDLE(typeSymbolHandle); +} + Instruction* Llvm::getCast(Value* source, Type* targetType) { Type* sourceType = source->getType(); @@ -2242,14 +2686,21 @@ Value* Llvm::getShadowStack() // Shadow stack moved up to avoid overwriting anything on the stack in the compiling method Value* Llvm::getShadowStackForCallee() { - // Note that funclets have the shadow stack arg in the same position (0) as the main function. - return gepOrAddr(getShadowStack(), getTotalLocalOffset()); + unsigned funcIdx = getCurrentLlvmFunctionIndex(); + unsigned hndIndex = + (funcIdx == ROOT_FUNC_IDX) ? EHblkDsc::NO_ENCLOSING_INDEX : _compiler->funGetFunc(funcIdx)->funEHIndex; + + return gepOrAddr(getShadowStack(), getShadowFrameSize(hndIndex)); } Value* Llvm::getOriginalShadowStack() { + if (getCurrentLlvmFunctionIndex() == ROOT_FUNC_IDX) + { + return getShadowStack(); + } + // The original shadow stack pointer is the second funclet parameter. - assert(getCurrentLlvmFunction() != getRootLlvmFunction()); return getCurrentLlvmFunction()->getArg(1); } @@ -2306,12 +2757,23 @@ llvm::DILocation* Llvm::createDebugFunctionAndDiLocation(DebugMetadata debugMeta functionMetaType, lineNumber, llvm::DINode::DIFlags::FlagZero, llvm::DISubprogram::DISPFlags::SPFlagDefinition | llvm::DISubprogram::DISPFlags::SPFlagLocalToUnit); - // TODO-LLVM-EH: broken for funclets. + // TODO-LLVM-EH: debugging in funclets. getRootLlvmFunction()->setSubprogram(_debugFunction); } return llvm::DILocation::get(_llvmContext, lineNo, 0, _debugFunction); } +llvm::DILocation* Llvm::getArtificialDebugLocation() +{ + if (_debugFunction == nullptr) + { + return nullptr; + } + + // Line number "0" is used to represent non-user code in DWARF. + return llvm::DILocation::get(_llvmContext, 0, 0, _debugFunction); +} + llvm::BasicBlock* Llvm::getCurrentLlvmBlock() const { return getCurrentLlvmBlocks()->LastBlock; @@ -2319,27 +2781,30 @@ llvm::BasicBlock* Llvm::getCurrentLlvmBlock() const void Llvm::setCurrentEmitContextForBlock(BasicBlock* block) { - LlvmBlockRange* llvmBlocks = getLlvmBlocksForBlock(block); + unsigned funcIdx = getLlvmFunctionIndexForBlock(block); unsigned tryIndex = block->hasTryIndex() ? block->getTryIndex() : EHblkDsc::NO_ENCLOSING_INDEX; + LlvmBlockRange* llvmBlocks = getLlvmBlocksForBlock(block); - setCurrentEmitContext(llvmBlocks, tryIndex); + setCurrentEmitContext(funcIdx, tryIndex, llvmBlocks); m_currentBlock = block; } -void Llvm::setCurrentEmitContext(LlvmBlockRange* llvmBlocks, unsigned tryIndex) +void Llvm::setCurrentEmitContext(unsigned funcIdx, unsigned tryIndex, LlvmBlockRange* llvmBlocks) { + assert(getLlvmFunctionForIndex(funcIdx) == llvmBlocks->LastBlock->getParent()); + _builder.SetInsertPoint(llvmBlocks->LastBlock); - m_currentLlvmBlocks = llvmBlocks; + m_currentLlvmFunctionIndex = funcIdx; m_currentProtectedRegionIndex = tryIndex; + m_currentLlvmBlocks = llvmBlocks; // "Raw" emission contexts do not have a current IR block. m_currentBlock = nullptr; } -LlvmBlockRange* Llvm::getCurrentLlvmBlocks() const +unsigned Llvm::getCurrentLlvmFunctionIndex() const { - assert(m_currentLlvmBlocks != nullptr); - return m_currentLlvmBlocks; + return m_currentLlvmFunctionIndex; } //------------------------------------------------------------------------ @@ -2354,6 +2819,12 @@ unsigned Llvm::getCurrentProtectedRegionIndex() const return m_currentProtectedRegionIndex; } +LlvmBlockRange* Llvm::getCurrentLlvmBlocks() const +{ + assert(m_currentLlvmBlocks != nullptr); + return m_currentLlvmBlocks; +} + Function* Llvm::getRootLlvmFunction() { return getLlvmFunctionForIndex(ROOT_FUNC_IDX); @@ -2361,18 +2832,23 @@ Function* Llvm::getRootLlvmFunction() Function* Llvm::getCurrentLlvmFunction() { - return getCurrentLlvmBlock()->getParent(); + return getLlvmFunctionForIndex(getCurrentLlvmFunctionIndex()); } Function* Llvm::getLlvmFunctionForIndex(unsigned funcIdx) { - Function* llvmFunc = m_functions[funcIdx]; - assert(llvmFunc != nullptr); + return getLlvmFunctionInfoForIndex(funcIdx).LlvmFunction; +} - return llvmFunc; +FunctionInfo& Llvm::getLlvmFunctionInfoForIndex(unsigned funcIdx) +{ + FunctionInfo& funcInfo = m_functions[funcIdx]; + assert(funcInfo.LlvmFunction != nullptr); + + return funcInfo; } -unsigned Llvm::getLlvmFunctionIndexForBlock(BasicBlock* block) +unsigned Llvm::getLlvmFunctionIndexForBlock(BasicBlock* block) const { unsigned funcIdx = ROOT_FUNC_IDX; @@ -2392,7 +2868,7 @@ unsigned Llvm::getLlvmFunctionIndexForBlock(BasicBlock* block) return funcIdx; } -unsigned Llvm::getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex) +unsigned Llvm::getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex) const { unsigned funcIdx = ROOT_FUNC_IDX; if (tryIndex != EHblkDsc::NO_ENCLOSING_INDEX) @@ -2436,7 +2912,8 @@ llvm::BasicBlock* Llvm::createInlineLlvmBlock() LlvmBlockRange* Llvm::getLlvmBlocksForBlock(BasicBlock* block) { - assert(block != nullptr); + // We should never be asking for unreachable blocks here since we won't generate code for them. + assert(isReachable(block) || (block == _compiler->fgFirstBB) || _compiler->fgIsThrowHlpBlk(block)); LlvmBlockRange* llvmBlockRange = _blkToLlvmBlksMap.LookupPointer(block); if (llvmBlockRange == nullptr) @@ -2473,16 +2950,108 @@ llvm::BasicBlock* Llvm::getLastLlvmBlockForBlock(BasicBlock* block) return getLlvmBlocksForBlock(block)->LastBlock; } +llvm::BasicBlock* Llvm::getOrCreatePrologLlvmBlockForFunction(unsigned funcIdx) +{ + const char* const PROLOG_BLOCK_NAME = "BB00"; + + BasicBlock* firstUserBlock = getFirstBlockForFunction(funcIdx); + llvm::BasicBlock* firstLlvmUserBlock = getFirstLlvmBlockForBlock(firstUserBlock); + llvm::BasicBlock* prologLlvmBlock = firstLlvmUserBlock->getPrevNode(); + if ((prologLlvmBlock == nullptr) || !prologLlvmBlock->getName().startswith(PROLOG_BLOCK_NAME)) + { + Function* llvmFunc = firstLlvmUserBlock->getParent(); + prologLlvmBlock = llvm::BasicBlock::Create(_llvmContext, PROLOG_BLOCK_NAME, llvmFunc, firstLlvmUserBlock); + + // Eagerly insert jump to the user block to simplify calling code. + llvm::BranchInst::Create(firstLlvmUserBlock, prologLlvmBlock); + } + + return prologLlvmBlock; +} + +//------------------------------------------------------------------------ +// isReachable: Does this block have an immediate dominator? +// +// Arguments: +// block - The block to check +// +// Return Value: +// Whether "block" has an immediate dominator, i. e. is statically +// reachable, not the first block, and not a throw helper block. +// +bool Llvm::isReachable(BasicBlock* block) const +{ + return block->bbIDom != nullptr; +} + +BasicBlock* Llvm::getFirstBlockForFunction(unsigned funcIdx) const +{ + if (funcIdx == ROOT_FUNC_IDX) + { + return _compiler->fgFirstBB; + } + + FuncInfoDsc* funcInfo = _compiler->funGetFunc(funcIdx); + EHblkDsc* ehDsc = _compiler->ehGetDsc(funcInfo->funEHIndex); + return (funcInfo->funKind == FUNC_FILTER) ? ehDsc->ebdFilter : ehDsc->ebdHndBeg; +} + Value* Llvm::getLocalAddr(unsigned lclNum) { - AllocaInst* addrValue = m_allocas[lclNum]; - assert(addrValue != nullptr); + Value* addrValue; + if (getCurrentLlvmFunctionIndex() == ROOT_FUNC_IDX) + { + addrValue = getLlvmFunctionInfoForIndex(ROOT_FUNC_IDX).Allocas[lclNum]; + } + else + { + addrValue = getOrCreateAllocaForLocalInFunclet(lclNum); + } + assert(addrValue != nullptr); return addrValue; } -unsigned int Llvm::getTotalLocalOffset() +//------------------------------------------------------------------------ +// getOrCreateAllocaForLocalInFunclet: Get an address for a funclet local. +// +// For a local to be (locally) live on the LLVM frame in a funclet, it has +// to be tracked and have its address taken (but not exposed!). Such locals +// are rare, and it is not cheap to indentify their set precisely before +// the code has been generated. We therefore use a lazy strategy for their +// materialization in the funclet prologs. +// +// Arguments: +// lclNum - The local for which to get the allocated home +// +// Return Value: +// Address on the LLVM frame "lclNum" has in the current funclet. +// +Value* Llvm::getOrCreateAllocaForLocalInFunclet(unsigned lclNum) { - assert((_shadowStackLocalsSize % TARGET_POINTER_SIZE) == 0); - return _shadowStackLocalsSize; + LclVarDsc* varDsc = _compiler->lvaGetDesc(lclNum); + assert(varDsc->lvTracked); // Untracked locals in functions with funclets live on the shadow frame. + + unsigned funcIdx = getCurrentLlvmFunctionIndex(); + assert(funcIdx != ROOT_FUNC_IDX); // The root's prolog is generated eagerly. + assert(!VarSetOps::IsMember(_compiler, getFirstBlockForFunction(funcIdx)->bbLiveIn, varDsc->lvVarIndex)); + + FunctionInfo& funcInfo = getLlvmFunctionInfoForIndex(funcIdx); + AllocaMap* allocaMap = funcInfo.AllocaMap; + if (allocaMap == nullptr) + { + allocaMap = new (_compiler->getAllocator(CMK_Codegen)) AllocaMap(_compiler->getAllocator(CMK_Codegen)); + funcInfo.AllocaMap = allocaMap; + } + + llvm::AllocaInst* allocaInst; + if (!allocaMap->Lookup(lclNum, &allocaInst)) + { + llvm::BasicBlock* prologLlvmBlock = getOrCreatePrologLlvmBlockForFunction(funcIdx); + allocaInst = new llvm::AllocaInst(getLlvmTypeForLclVar(varDsc), 0, "", prologLlvmBlock->getTerminator()); + + allocaMap->Set(lclNum, allocaInst); + } + + return allocaInst; } diff --git a/src/coreclr/jit/llvmlower.cpp b/src/coreclr/jit/llvmlower.cpp index 0fc16d845790..54d85b7db009 100644 --- a/src/coreclr/jit/llvmlower.cpp +++ b/src/coreclr/jit/llvmlower.cpp @@ -383,6 +383,10 @@ void Llvm::lowerBlock(BasicBlock* block) lowerCall(node->AsCall()); break; + case GT_CATCH_ARG: + lowerCatchArg(node); + break; + case GT_IND: case GT_STOREIND: case GT_OBJ: @@ -555,6 +559,11 @@ void Llvm::lowerCall(GenTreeCall* callNode) { failUnsupportedCalls(callNode); + if (callNode->IsHelperCall(_compiler, CORINFO_HELP_RETHROW)) + { + lowerRethrow(callNode); + } + lowerCallToShadowStack(callNode); // If there is a no return, or always throw call, delete the dead code so we can add unreachable @@ -568,6 +577,40 @@ void Llvm::lowerCall(GenTreeCall* callNode) } } +void Llvm::lowerRethrow(GenTreeCall* callNode) +{ + assert(callNode->IsHelperCall(_compiler, CORINFO_HELP_RETHROW)); + + // Language in ECMA 335 I.12.4.2.8.2.2 clearly states that rethrows nested inside finallys are + // legal, however, neither C# nor the old verification system allow this. CoreCLR behavior was + // not tested. Implementing this would imply saving the exception object to the "original" shadow + // frame shared between funclets. For now we punt. + if (!_compiler->ehGetDsc(CurrentBlock()->getHndIndex())->HasCatchHandler()) + { + IMPL_LIMITATION("Nested rethrow"); + } + + // A rethrow is a special throw that preserves the stack trace. Our helper we use for rethrow has + // the equivalent of a managed signature "void (object*)", i. e. takes the exception object address + // explicitly. Add it here, before the general call lowering. + assert(callNode->gtCallArgs == nullptr); + callNode->ResetArgInfo(); + + GenTree* excObjAddr = insertShadowStackAddr(callNode, getCatchArgOffset(), _shadowStackLclNum); + callNode->gtCallArgs = _compiler->gtNewCallArgs(excObjAddr); + + _compiler->fgInitArgInfo(callNode); +} + +void Llvm::lowerCatchArg(GenTree* catchArgNode) +{ + GenTree* excObjAddr = insertShadowStackAddr(catchArgNode, getCatchArgOffset(), _shadowStackLclNum); + + catchArgNode->ChangeOper(GT_IND); + catchArgNode->gtFlags |= GTF_IND_NONFAULTING; + catchArgNode->AsIndir()->SetAddr(excObjAddr); +} + void Llvm::lowerIndir(GenTreeIndir* indirNode) { if ((indirNode->gtFlags & GTF_IND_NONFAULTING) == 0) @@ -746,7 +789,7 @@ void Llvm::lowerCallToShadowStack(GenTreeCall* callNode) GenTreeCall::Use* lastArg = nullptr; if (callHasShadowStackArg(callNode)) { - GenTree* calleeShadowStack = insertShadowStackAddr(callNode, _shadowStackLocalsSize, _shadowStackLclNum); + GenTree* calleeShadowStack = insertShadowStackAddr(callNode, getCurrentShadowFrameSize(), _shadowStackLclNum); GenTreePutArgType* calleeShadowStackPutArg = _compiler->gtNewPutArgType(calleeShadowStack, CORINFO_TYPE_PTR, NO_CLASS_HANDLE); @@ -844,13 +887,14 @@ void Llvm::lowerCallToShadowStack(GenTreeCall* callNode) shadowStackUseOffest = padOffset(corInfoType, clsHnd, shadowStackUseOffest); } + unsigned shadowFrameSize = getCurrentShadowFrameSize(); if (argNode->OperIs(GT_FIELD_LIST)) { for (GenTreeFieldList::Use& use : argNode->AsFieldList()->Uses()) { assert(use.GetType() != TYP_STRUCT); - unsigned fieldOffsetValue = _shadowStackLocalsSize + shadowStackUseOffest + use.GetOffset(); + unsigned fieldOffsetValue = shadowFrameSize + shadowStackUseOffest + use.GetOffset(); GenTree* fieldSlotAddr = insertShadowStackAddr(callNode, fieldOffsetValue, _shadowStackLclNum); GenTree* fieldStoreNode = createShadowStackStoreNode(use.GetType(), fieldSlotAddr, use.GetNode()); @@ -861,7 +905,7 @@ void Llvm::lowerCallToShadowStack(GenTreeCall* callNode) } else { - unsigned offsetValue = _shadowStackLocalsSize + shadowStackUseOffest; + unsigned offsetValue = shadowFrameSize + shadowStackUseOffest; GenTree* slotAddr = insertShadowStackAddr(callNode, offsetValue, _shadowStackLclNum); GenTree* storeNode = createShadowStackStoreNode(argNode->TypeGet(), slotAddr, argNode); @@ -971,9 +1015,7 @@ GenTreeCall::Use* Llvm::lowerCallReturn(GenTreeCall* callNode, GenTreeCall::Use* if (needsReturnStackSlot(_compiler, callNode)) { // replace the "CALL ref" with a "CALL void" that takes a return address as the first argument - GenTreeLclVar* shadowStackVar = _compiler->gtNewLclvNode(_shadowStackLclNum, TYP_I_IMPL); - GenTreeIntCon* offset = _compiler->gtNewIconNode(_shadowStackLocalsSize, TYP_I_IMPL); - GenTree* returnValueAddress = _compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, shadowStackVar, offset); + GenTree* returnValueAddress = insertShadowStackAddr(callNode, getCurrentShadowFrameSize(), _shadowStackLclNum); // create temp for the return address unsigned returnTempNum = _compiler->lvaGrabTemp(false DEBUGARG("return value address")); @@ -1017,8 +1059,7 @@ GenTreeCall::Use* Llvm::lowerCallReturn(GenTreeCall* callNode, GenTreeCall::Use* callNode->gtCorInfoType = CORINFO_TYPE_VOID; callNode->ChangeType(TYP_VOID); - CurrentRange().InsertBefore(callNode, shadowStackVar, offset, returnValueAddress, addrStore); - CurrentRange().InsertAfter(addrStore, returnAddrLcl, putArg); + CurrentRange().InsertBefore(callNode, addrStore, returnAddrLcl, putArg); CurrentRange().InsertAfter(callNode, returnAddrLclAfterCall, indirNode); } else @@ -1190,3 +1231,52 @@ bool Llvm::isFuncletParameter(unsigned lclNum) const { return (lclNum == _shadowStackLclNum) || (lclNum == _originalShadowStackLclNum); } + +unsigned Llvm::getCurrentShadowFrameSize() const +{ + assert(CurrentBlock() != nullptr); + unsigned hndIndex = CurrentBlock()->hasHndIndex() ? CurrentBlock()->getHndIndex() : EHblkDsc::NO_ENCLOSING_INDEX; + return getShadowFrameSize(hndIndex); +} + +//------------------------------------------------------------------------ +// getShadowFrameSize: What is the size of a function's shadow frame? +// +// Arguments: +// hndIndex - Handler index representing the function, NO_ENCLOSING_INDEX +// is used for the root +// +// Return Value: +// The size of the shadow frame for the given function. We term this +// the value by which the shadow stack pointer must be offset before +// calling managed code such that the caller will not clobber anything +// live on the frame. Note that funclets do not have any shadow state +// of their own and use the "original" frame from the parent function, +// with one exception: catch handlers and filters have one readonly +// pointer-sized argument representing the exception. +// +unsigned Llvm::getShadowFrameSize(unsigned hndIndex) const +{ + if (hndIndex == EHblkDsc::NO_ENCLOSING_INDEX) + { + return getOriginalShadowFrameSize(); + } + if (_compiler->ehGetDsc(hndIndex)->HasCatchHandler()) + { + // For the implicit (readonly) exception object argument. + return TARGET_POINTER_SIZE; + } + + return 0; +} + +unsigned Llvm::getOriginalShadowFrameSize() const +{ + assert((_shadowStackLocalsSize % TARGET_POINTER_SIZE) == 0); + return _shadowStackLocalsSize; +} + +unsigned Llvm::getCatchArgOffset() const +{ + return 0; +} diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs index c2cb3eaecf6e..81b81612b337 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs @@ -177,6 +177,9 @@ public DependencyList Import() _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); // TODO-LLVM: make these into normal "ReadyToRunHelper" instead of hardcoding things here. + helperMethod = helperType.GetKnownMethod("HandleExceptionWasmMutuallyProtectingCatches", null); + _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); + helperMethod = helperType.GetKnownMethod("HandleExceptionWasmFilteredCatch", null); _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); diff --git a/src/coreclr/tools/aot/ILCompiler.LLVM/Compiler/LLVMCodegenCompilation.cs b/src/coreclr/tools/aot/ILCompiler.LLVM/Compiler/LLVMCodegenCompilation.cs index 074d756e6eee..5433a67ffcf2 100644 --- a/src/coreclr/tools/aot/ILCompiler.LLVM/Compiler/LLVMCodegenCompilation.cs +++ b/src/coreclr/tools/aot/ILCompiler.LLVM/Compiler/LLVMCodegenCompilation.cs @@ -90,7 +90,6 @@ protected override void CompileInternal(string outputFile, ObjectDumper dumper) LLVMObjectWriter.EmitObject(outputFile, nodes, NodeFactory, this, dumper); - Console.WriteLine($"RyuJIT compilation results, total methods {totalMethodCount} RyuJit Methods {ryuJitMethodCount} {((decimal)ryuJitMethodCount * 100 / totalMethodCount):n4}%"); } @@ -159,27 +158,27 @@ private unsafe void CompileSingleMethod(CorInfoImpl corInfo, LLVMMethodCodeNode methodCodeNodeNeedingCode.CompilationCompleted = true; return; } - - if (methodIL.GetExceptionRegions().Length == 0 && !_disableRyuJit) + if (!_disableRyuJit) { - var mangledName = NodeFactory.NameMangler.GetMangledMethodName(method).ToString(); - var sig = method.Signature; - corInfo.RegisterLlvmCallbacks((IntPtr)Unsafe.AsPointer(ref corInfo), _outputFile, - Module.Target, - Module.DataLayout); + corInfo.RegisterLlvmCallbacks((IntPtr)Unsafe.AsPointer(ref corInfo), _outputFile, Module.Target, Module.DataLayout); corInfo.CompileMethod(methodCodeNodeNeedingCode); methodCodeNodeNeedingCode.CompilationCompleted = true; + // TODO: delete this external function when old module is gone - LLVMValueRef externFunc = ILImporter.GetOrCreateLLVMFunction(Module, mangledName, GetLLVMSignatureForMethod(sig, method.RequiresInstArg())); + var mangledName = NodeFactory.NameMangler.GetMangledMethodName(method).ToString(); + LLVMValueRef externFunc = ILImporter.GetOrCreateLLVMFunction(Module, mangledName, GetLLVMSignatureForMethod(method.Signature, method.RequiresInstArg())); externFunc.Linkage = LLVMLinkage.LLVMExternalLinkage; ILImporter.GenerateRuntimeExportThunk(this, method, externFunc); ryuJitMethodCount++; } - else ILImporter.CompileMethod(this, methodCodeNodeNeedingCode); + else + { + ILImporter.CompileMethod(this, methodCodeNodeNeedingCode); + } } catch (CodeGenerationFailedException) { diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.Llvm.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.Llvm.cs index e8b2858d3156..60c6599cdc17 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.Llvm.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.Llvm.cs @@ -26,8 +26,13 @@ public unsafe sealed partial class CorInfoImpl public static void addCodeReloc(IntPtr thisHandle, void* handle) { var _this = GetThis(thisHandle); - var obj = _this.HandleToObject((IntPtr)handle); + + AddCodeRelocImpl(_this, obj); + } + + private static void AddCodeRelocImpl(CorInfoImpl _this, object obj) + { ISymbolNode node; if (obj is ISymbolNode symbolNode) { @@ -47,7 +52,6 @@ public static void addCodeReloc(IntPtr thisHandle, void* handle) MethodDesc virtualCallTarget = (MethodDesc)helperNode.Target; _this._codeRelocs.Add(new Relocation(RelocType.IMAGE_REL_BASED_REL32, 0, _this._compilation.NodeFactory.MethodEntrypoint(virtualCallTarget))); - return; } @@ -108,10 +112,15 @@ private static byte[] AppendNullByte(byte[] inputArray) public static byte* getMangledMethodName(IntPtr thisHandle, CORINFO_METHOD_STRUCT_* ftn) { var _this = GetThis(thisHandle); - MethodDesc method = _this.HandleToObject(ftn); - return (byte*)_this.GetPin(AppendNullByte(_this._compilation.NameMangler.GetMangledMethodName(method).UnderlyingArray)); + return GetMangledMethodNameImpl(_this, method); + } + + private static byte* GetMangledMethodNameImpl(CorInfoImpl _this, MethodDesc method) + { + Utf8String mangledName = _this._compilation.NameMangler.GetMangledMethodName(method); + return (byte*)_this.GetPin(AppendNullByte(mangledName.UnderlyingArray)); } [UnmanagedCallersOnly] @@ -131,6 +140,31 @@ private static byte[] AppendNullByte(byte[] inputArray) return (byte*)_this.GetPin(sb.UnderlyingArray); } + [UnmanagedCallersOnly] + public static byte* getEHDispatchFunctionName(IntPtr thisHandle, CORINFO_EH_CLAUSE_FLAGS handlerType) + { + var _this = GetThis(thisHandle); + string dispatchMethodName = handlerType switch + { + CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_SAMETRY => "HandleExceptionWasmMutuallyProtectingCatches", + CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_NONE => "HandleExceptionWasmCatch", + CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_FILTER => "HandleExceptionWasmFilteredCatch", + CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_FINALLY or CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_FAULT => "HandleExceptionWasmFault", + _ => throw new NotSupportedException() + }; + + // TODO-LLVM: we are breaking the abstraction here. Compiler is not allowed to access methods from the + // managed runtime directly and assume they are compiled into CoreLib. The dispatch routine should be + // made into a RuntimeExport once we solve the issues around calling convention mismatch for them. + MetadataType ehType = _this._compilation.TypeSystemContext.SystemModule.GetKnownType("System.Runtime", "EH"); + MethodDesc dispatchMethod = ehType.GetKnownMethod(dispatchMethodName, null); + + // Codegen is asking for the dispatcher; assume it'll use it. + AddCodeRelocImpl(_this, dispatchMethod); + + return GetMangledMethodNameImpl(_this, dispatchMethod); + } + // IL backend does not use the mangled name. The unmangled name is easier to read. [UnmanagedCallersOnly] public static byte* getTypeName(IntPtr thisHandle, CORINFO_CLASS_STRUCT_* structHnd) @@ -312,6 +346,7 @@ enum EEApiId { GetMangledMethodName, GetSymbolMangledName, + GetEHDispatchFunctionName, GetTypeName, AddCodeReloc, IsRuntimeImport, @@ -333,6 +368,7 @@ public void RegisterLlvmCallbacks(IntPtr corInfoPtr, string outputFileName, stri void** callbacks = stackalloc void*[(int)EEApiId.Count + 1]; callbacks[(int)EEApiId.GetMangledMethodName] = (delegate* unmanaged)&getMangledMethodName; callbacks[(int)EEApiId.GetSymbolMangledName] = (delegate* unmanaged)&getSymbolMangledName; + callbacks[(int)EEApiId.GetEHDispatchFunctionName] = (delegate* unmanaged)&getEHDispatchFunctionName; callbacks[(int)EEApiId.GetTypeName] = (delegate* unmanaged)&getTypeName; callbacks[(int)EEApiId.AddCodeReloc] = (delegate* unmanaged)&addCodeReloc; callbacks[(int)EEApiId.IsRuntimeImport] = (delegate* unmanaged)&isRuntimeImport; From d5e9d29132ad3caaad6148c75678688f6de800c8 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Fri, 13 Jan 2023 20:53:50 +0300 Subject: [PATCH 11/13] Opaque pointers and a typo --- src/coreclr/jit/llvmcodegen.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/llvmcodegen.cpp b/src/coreclr/jit/llvmcodegen.cpp index a807cdea0ae1..4600836827b9 100644 --- a/src/coreclr/jit/llvmcodegen.cpp +++ b/src/coreclr/jit/llvmcodegen.cpp @@ -143,7 +143,7 @@ bool Llvm::initializeFunctions() ehDsc->HasCatchHandler() ? Type::getInt32Ty(_llvmContext) : Type::getVoidTy(_llvmContext); // All funclets have two arguments: original and actual shadow stacks. - Type* ptrLlvmType = Type::getInt8PtrTy(_llvmContext); + Type* ptrLlvmType = getPtrLlvmType(); FunctionType* llvmFuncType = FunctionType::get(retLlvmType, {ptrLlvmType, ptrLlvmType}, /* isVarArg */ false); @@ -375,7 +375,7 @@ void Llvm::generateEHDispatch() } // Recover the C++ personality function. - Type* ptrLlvmType = Type::getInt8PtrTy(_llvmContext); + Type* ptrLlvmType = getPtrLlvmType(); Type* int32LlvmType = Type::getInt32Ty(_llvmContext); Type* cppExcTupleLlvmType = llvm::StructType::get(ptrLlvmType, int32LlvmType); llvm::StructType* dispatchDataLlvmType = llvm::StructType::get(cppExcTupleLlvmType, ptrLlvmType); @@ -491,7 +491,7 @@ void Llvm::generateEHDispatch() } // The code we will generate effectively inlines the usual runtime dispatch logic. The main reason this - // scheme was chosen is the fact (re)throwing funclets are handled by it semlessly and efficiently. The + // scheme was chosen is the fact (re)throwing funclets are handled by it seamlessly and efficiently. The // downside to it is the code size overhead of the calls made for each protected region. // // DISPATCH_PAD_INNER: From 1a21442ffb00a4d75afde9db75a1b781288446bb Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Fri, 13 Jan 2023 21:11:21 +0300 Subject: [PATCH 12/13] Use proper pointer types in dispatchers --- .../System/Runtime/ExceptionHandling.wasm.cs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs index ecfb2f9cf53f..dd3e4fe69450 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs @@ -28,7 +28,7 @@ private struct ManagedExceptionWrapper private struct DispatchData { public readonly CppExceptionTuple CppExceptionTuple; // Owned by codegen. - public void* DispatcherData; // Owned by runtime. + public ManagedExceptionWrapper* DispatcherData; // Owned by runtime. // We consider a dispatch to be "active" if it has already visited nested handlers in the same LLVM frame. // Codegen will initialize "DispatcherData" to null on entry to the native handler. @@ -44,11 +44,11 @@ private struct CppExceptionTuple // These per-clause handlers are invoked by RyuJit-generated LLVM code. TODO-LLVM: once we move to opaque // pointers, we can change the signatures of these functions to use precise pointer types instead of "void*"s. // - private static int HandleExceptionWasmMutuallyProtectingCatches(void* pShadowFrame, void* pDispatchData, void* pEHTable) + private static int HandleExceptionWasmMutuallyProtectingCatches(void* pShadowFrame, DispatchData* pDispatchData, void** pEHTable) { - object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_UNUSED, pEHTable, pShadowFrame, (DispatchData*)pDispatchData); + object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_UNUSED, pEHTable, pShadowFrame, pDispatchData); - EHRyuJitClauseIteratorWasm clauseIter = new EHRyuJitClauseIteratorWasm((void**)pEHTable); + EHRyuJitClauseIteratorWasm clauseIter = new EHRyuJitClauseIteratorWasm(pEHTable); EHRyuJitClauseWasm clause; while (clauseIter.Next(&clause)) { @@ -80,9 +80,9 @@ private static int HandleExceptionWasmMutuallyProtectingCatches(void* pShadowFra return DispatchContinueSearch(exception, pDispatchData); } - private static int HandleExceptionWasmFilteredCatch(void* pShadowFrame, void* pDispatchData, void* pHandler, void* pFilter) + private static int HandleExceptionWasmFilteredCatch(void* pShadowFrame, DispatchData* pDispatchData, void* pHandler, void* pFilter) { - object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_FILTER, null, pShadowFrame, (DispatchData*)pDispatchData); + object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_FILTER, null, pShadowFrame, pDispatchData); if (CallFilterFunclet(pFilter, exception, pShadowFrame, ryuJitAbi: true)) { @@ -93,11 +93,11 @@ private static int HandleExceptionWasmFilteredCatch(void* pShadowFrame, void* pD return DispatchContinueSearch(exception, pDispatchData); } - private static int HandleExceptionWasmCatch(void* pShadowFrame, void* pDispatchData, void* pHandler, void* pClauseType) + private static int HandleExceptionWasmCatch(void* pShadowFrame, DispatchData* pDispatchData, void* pHandler, MethodTable* pClauseType) { - object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_TYPED, pClauseType, pShadowFrame, (DispatchData*)pDispatchData); + object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_TYPED, pClauseType, pShadowFrame, pDispatchData); - if (ShouldTypedClauseCatchThisException(exception, (MethodTable*)pClauseType)) + if (ShouldTypedClauseCatchThisException(exception, pClauseType)) { __cxa_end_catch(); return CallCatchFunclet(pHandler, exception, pShadowFrame); @@ -106,14 +106,14 @@ private static int HandleExceptionWasmCatch(void* pShadowFrame, void* pDispatchD return DispatchContinueSearch(exception, pDispatchData); } - private static void HandleExceptionWasmFault(void* pShadowFrame, void* pDispatchData, void* pHandler) + private static void HandleExceptionWasmFault(void* pShadowFrame, DispatchData* pDispatchData, void* pHandler) { // TODO-LLVM-EH: for compatiblity with the IL backend we will invoke faults/finallys even if we do not find // a suitable catch in a frame. A correct implementation of this will require us to keep a stack of pending // faults/finallys (inside the native exception), to be invoked at the point we find the handling frame. We // should also fail fast instead of invoking the second pass handlers if the exception goes unhandled. // - object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_FAULT, null, pShadowFrame, (DispatchData*)pDispatchData); + object exception = BeginSingleDispatch(RhEHClauseKind.RH_EH_CLAUSE_FAULT, null, pShadowFrame, pDispatchData); CallFinallyFunclet(pHandler, pShadowFrame, ryuJitAbi: true); DispatchContinueSearch(exception, pDispatchData); } @@ -133,7 +133,7 @@ private static object BeginSingleDispatch(RhEHClauseKind kind, void* data, void* } else { - pCppException = (ManagedExceptionWrapper*)pDispatchData->DispatcherData; + pCppException = pDispatchData->DispatcherData; } object exception = Unsafe.Read(&pCppException->ManagedException); @@ -144,13 +144,13 @@ private static object BeginSingleDispatch(RhEHClauseKind kind, void* data, void* return exception; } - private static int DispatchContinueSearch(object exception, void* pDispatchData) + private static int DispatchContinueSearch(object exception, DispatchData* pDispatchData) { // GC may have invalidated the exception object in the native exception; make sure it is up-to-date before // rethrowing or jumping to an upstream dispatcher. fixed (void* pKeepAlive = &exception.GetRawData()) { - Unsafe.Write(&((ManagedExceptionWrapper*)((DispatchData*)pDispatchData)->DispatcherData)->ManagedException, exception); + Unsafe.Write(&pDispatchData->DispatcherData->ManagedException, exception); } return 0; From f73a449e65fc180bbbcd009f0dcf3b5fb3fe9167 Mon Sep 17 00:00:00 2001 From: SingleAccretion <62474226+SingleAccretion@users.noreply.github.com> Date: Sat, 14 Jan 2023 01:18:09 +0300 Subject: [PATCH 13/13] Delete the TODO as well --- .../Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs index dd3e4fe69450..83b3d98fde04 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.wasm.cs @@ -41,8 +41,7 @@ private struct CppExceptionTuple public int Selector; } - // These per-clause handlers are invoked by RyuJit-generated LLVM code. TODO-LLVM: once we move to opaque - // pointers, we can change the signatures of these functions to use precise pointer types instead of "void*"s. + // These per-clause handlers are invoked by RyuJit-generated LLVM code. // private static int HandleExceptionWasmMutuallyProtectingCatches(void* pShadowFrame, DispatchData* pDispatchData, void** pEHTable) {