From aa181f944db08df336ad3513fb2ecb3998a1bc33 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Wed, 28 Dec 2022 18:57:34 +0300 Subject: [PATCH] Implement dispatch codegen Handler frequency statistics, collected via SPMI: benchmarks.pmi Total clauses : 2514 Finally : 281 11.2% Fault : 1413 56.2% Single catch : 724 28.8% Single filter : 49 1.9% Mutually protecting : 47 1.9% Among them, catches : 108 86.4% Among them, filters : 17 13.6% libraries.pmi Total clauses : 19568 Finally : 1937 9.9% Fault : 11399 58.3% Single catch : 5373 27.5% Single filter : 349 1.8% Mutually protecting : 510 2.6% Among them, catches : 1240 95.0% Among them, filters : 65 5.0% Passing (with RyuJit codegen) tests: - baseservices\exceptions\simple\finally.csproj - baseservices\exceptions\simple\fault.ilproj - baseservices\exceptions\unittests\baseclass.csproj - baseservices\exceptions\unittests\Baadbaad.csproj - baseservices\exceptions\unittests\TryCatchInFinallyTest.csproj - baseservices\exceptions\unittests\ThrowInFinallyNestedInTry.csproj - baseservices\exceptions\unittests\ThrowInFinallyTest.csproj - baseservices\exceptions\unittests\ThrowInCatch.csproj - baseservices\exceptions\unittests\ReturnFromCatch.csproj - baseservices\exceptions\unittests\RethrowAndFinally.csproj - baseservices\exceptions\unittests\RecursiveThrowNew.csproj - baseservices\exceptions\unittests\RecursiveRethrow.csproj - baseservices\exceptions\unittests\Recurse.csproj - baseservices\exceptions\unittests\Pending.csproj - baseservices\exceptions\unittests\InnerFinallyAndCatch.csproj - baseservices\exceptions\unittests\InnerFinally.csproj - baseservices\exceptions\unittests\GoryNativePast.csproj - baseservices\exceptions\unittests\GoryManagedPresent.csproj - baseservices\exceptions\unittests\CollidedUnwind.csproj --- src/coreclr/jit/compiler.h | 8 +- src/coreclr/jit/compiler.hpp | 6 +- src/coreclr/jit/fgdiagnostic.cpp | 2 + src/coreclr/jit/jiteh.h | 3 +- src/coreclr/jit/llvm.cpp | 18 +- src/coreclr/jit/llvm.h | 57 +- src/coreclr/jit/llvmcodegen.cpp | 884 +++++++++++++++--- src/coreclr/jit/llvmlower.cpp | 106 ++- src/coreclr/jit/llvmtypes.cpp | 9 + .../IL/ILImporter.Scanner.cs | 3 + .../Compiler/LLVMCodegenCompilation.cs | 18 +- .../JitInterface/CorInfoImpl.Llvm.cs | 44 +- 12 files changed, 959 insertions(+), 199 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index dc1c85223700..e00cf386e202 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4337,6 +4337,8 @@ class Compiler bool isExplicitTailCall, IL_OFFSET ilOffset = BAD_IL_OFFSET); + void impResolveToken(const BYTE* addr, CORINFO_RESOLVED_TOKEN* pResolvedToken, CorInfoTokenKind kind); + //========================================================================= // PROTECTED //========================================================================= @@ -4357,8 +4359,6 @@ class Compiler bool impIsPrimitive(CorInfoType type); bool impILConsumesAddr(const BYTE* codeAddr); - void impResolveToken(const BYTE* addr, CORINFO_RESOLVED_TOKEN* pResolvedToken, CorInfoTokenKind kind); - void impPushOnStack(GenTree* tree, typeInfo ti); void impPushNullObjRefOnStack(); StackEntry impPopStack(); @@ -6555,6 +6555,8 @@ class Compiler AddCodeDsc* fgFindExcptnTarget(SpecialCodeKind kind, unsigned refData); + bool fgIsThrowHlpBlk(BasicBlock* block); + bool fgUseThrowHelperBlocks(); AddCodeDsc* fgGetAdditionalCodeDescriptors() @@ -6565,8 +6567,6 @@ class Compiler private: bool fgIsCodeAdded(); - bool fgIsThrowHlpBlk(BasicBlock* block); - #if !FEATURE_FIXED_OUT_ARGS unsigned fgThrowHlpBlkStkLevel(BasicBlock* block); #endif // !FEATURE_FIXED_OUT_ARGS diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 6cb5ad0fac64..c9443fafcc8a 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -2705,7 +2705,8 @@ inline bool Compiler::fgIsThrowHlpBlk(BasicBlock* block) (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROWDIVZERO)) || (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROW_ARGUMENTEXCEPTION)) || (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROW_ARGUMENTOUTOFRANGEEXCEPTION)) || - (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_OVERFLOW)))) + (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_OVERFLOW)) || + (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROWNULLREF)))) { return false; } @@ -2719,7 +2720,8 @@ inline bool Compiler::fgIsThrowHlpBlk(BasicBlock* block) if (block == add->acdDstBlk) { return add->acdKind == SCK_RNGCHK_FAIL || add->acdKind == SCK_DIV_BY_ZERO || add->acdKind == SCK_OVERFLOW || - add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN; + add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN || + add->acdKind == SCK_NULL_REF_EXCPN; } } diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index 2df6d000419b..b5f622e7b2ec 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -1660,6 +1660,8 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) case EH_HANDLER_FAULT_WAS_FINALLY: handlerType = "fault-was-finally"; break; + default: + unreached(); } sprintf_s(name, sizeof(name), "EH#%u %s", XTnum, handlerType); rgnGraph.Insert(name, RegionGraph::RegionType::EH, ehDsc->ebdHndBeg, ehDsc->ebdHndLast); diff --git a/src/coreclr/jit/jiteh.h b/src/coreclr/jit/jiteh.h index 51dcad7d9094..cddfab4ef3f7 100644 --- a/src/coreclr/jit/jiteh.h +++ b/src/coreclr/jit/jiteh.h @@ -27,7 +27,8 @@ enum EHHandlerType EH_HANDLER_FILTER, EH_HANDLER_FAULT, EH_HANDLER_FINALLY, - EH_HANDLER_FAULT_WAS_FINALLY + EH_HANDLER_FAULT_WAS_FINALLY, + EH_HANDLER_COUNT }; // ToCORINFO_EH_CLAUSE_FLAGS: Convert an internal EHHandlerType to a CORINFO_EH_CLAUSE_FLAGS value diff --git a/src/coreclr/jit/llvm.cpp b/src/coreclr/jit/llvm.cpp index 7eb43f91d8cf..85c8a3a56499 100644 --- a/src/coreclr/jit/llvm.cpp +++ b/src/coreclr/jit/llvm.cpp @@ -23,6 +23,7 @@ enum class EEApiId { GetMangledMethodName, GetSymbolMangledName, + GetEHDispatchFunctionName, GetTypeName, AddCodeReloc, IsRuntimeImport, @@ -104,7 +105,6 @@ Llvm::Llvm(Compiler* compiler) _info(compiler->info), _sigInfo(compiler->info.compMethodInfo->args), _builder(_llvmContext), - _prologBuilder(_llvmContext), _blkToLlvmBlksMap(compiler->getAllocator(CMK_Codegen)), _sdsuMap(compiler->getAllocator(CMK_Codegen)), _localsMap(compiler->getAllocator(CMK_Codegen)), @@ -169,6 +169,7 @@ GCInfo* Llvm::getGCInfo() { _gcInfo = new (_compiler->getAllocator(CMK_GC)) GCInfo(_compiler); } + return _gcInfo; } @@ -377,8 +378,8 @@ const HelperFuncInfo& Llvm::getHelperFuncInfo(CorInfoHelpFunc helperFunc) // For WASM, currently implemented in the bootstrapper... { FUNC(CORINFO_HELP_THROW) CORINFO_TYPE_VOID, { CORINFO_TYPE_PTR } }, - // (Not) implemented in "Runtime\EHHelpers.cpp" - { FUNC(CORINFO_HELP_RETHROW) CORINFO_TYPE_VOID, { } }, + // Implemented in "Runtime.Base\src\System\Runtime\ExceptionHandling.wasm.cs". + { FUNC(CORINFO_HELP_RETHROW) CORINFO_TYPE_VOID, { CORINFO_TYPE_PTR } }, // Implemented in "Runtime\MiscHelpers.cpp". { FUNC(CORINFO_HELP_USER_BREAKPOINT) CORINFO_TYPE_VOID, { } }, @@ -654,11 +655,11 @@ unsigned Llvm::padNextOffset(CorInfoType corInfoType, CORINFO_CLASS_HANDLE struc [[noreturn]] void Llvm::failFunctionCompilation() { - for (Function* llvmFunc : m_functions) + for (FunctionInfo& funcInfo : m_functions) { - if (llvmFunc != nullptr) + if (funcInfo.LlvmFunction != nullptr) { - llvmFunc->deleteBody(); + funcInfo.LlvmFunction->deleteBody(); } } @@ -681,6 +682,11 @@ const char* Llvm::GetMangledSymbolName(void* symbol) return CallEEApi(symbol); } +const char* Llvm::GetEHDispatchFunctionName(CORINFO_EH_CLAUSE_FLAGS handlerType) +{ + return CallEEApi(handlerType); +} + const char* Llvm::GetTypeName(CORINFO_CLASS_HANDLE typeHandle) { return CallEEApi(typeHandle); diff --git a/src/coreclr/jit/llvm.h b/src/coreclr/jit/llvm.h index f35b68e3473e..cdedc53a4219 100644 --- a/src/coreclr/jit/llvm.h +++ b/src/coreclr/jit/llvm.h @@ -109,6 +109,17 @@ struct LlvmBlockRange } }; +typedef JitHashTable, llvm::AllocaInst*> AllocaMap; + +struct FunctionInfo +{ + Function* LlvmFunction; + union { + llvm::AllocaInst** Allocas; // Dense "lclNum -> Alloca*" mapping used for the main function. + AllocaMap* AllocaMap; // Sparse "lclNum -> Alloca*" mapping used for funclets. + }; +}; + // TODO: We should create a Static... class to manage the globals and their lifetimes. // Note we declare all statics here, and define them in llvm.cpp, for documentation and // visibility purposes even as some are only needed in other compilation units. @@ -132,16 +143,15 @@ class Llvm BasicBlock* m_currentBlock; DebugInfo _currentOffset; llvm::IRBuilder<> _builder; - llvm::IRBuilder<> _prologBuilder; JitHashTable, LlvmBlockRange> _blkToLlvmBlksMap; JitHashTable, Value*> _sdsuMap; JitHashTable _localsMap; std::vector _phiPairs; - std::vector m_allocas; - std::vector m_functions; + std::vector m_functions; std::vector m_EHDispatchLlvmBlocks; // Codegen emit context. + unsigned m_currentLlvmFunctionIndex = ROOT_FUNC_IDX; unsigned m_currentProtectedRegionIndex = EHblkDsc::NO_ENCLOSING_INDEX; LlvmBlockRange* m_currentLlvmBlocks = nullptr; @@ -198,6 +208,7 @@ class Llvm // const char* GetMangledMethodName(CORINFO_METHOD_HANDLE methodHandle); const char* GetMangledSymbolName(void* symbol); + const char* GetEHDispatchFunctionName(CORINFO_EH_CLAUSE_FLAGS handlerType); const char* GetTypeName(CORINFO_CLASS_HANDLE typeHandle); void AddCodeReloc(void* handle); bool IsRuntimeImport(CORINFO_METHOD_HANDLE methodHandle); @@ -223,6 +234,7 @@ class Llvm Type* getLlvmTypeForLclVar(LclVarDsc* varDsc); Type* getLlvmTypeForCorInfoType(CorInfoType corInfoType, CORINFO_CLASS_HANDLE classHnd); Type* getLlvmTypeForParameterType(CORINFO_CLASS_HANDLE classHnd); + Type* getIntPtrLlvmType(); unsigned getElementSize(CORINFO_CLASS_HANDLE fieldClassHandle, CorInfoType corInfoType); void addPaddingFields(unsigned paddingSize, std::vector& llvmFields); @@ -247,6 +259,8 @@ class Llvm void lowerFieldOfDependentlyPromotedStruct(GenTree* node); void ConvertShadowStackLocalNode(GenTreeLclVarCommon* node); void lowerCall(GenTreeCall* callNode); + void lowerRethrow(GenTreeCall* callNode); + void lowerCatchArg(GenTree* catchArgNode); void lowerIndir(GenTreeIndir* indirNode); void lowerStoreBlk(GenTreeBlk* storeBlkNode); void lowerDivMod(GenTreeOp* divModNode); @@ -265,6 +279,11 @@ class Llvm bool isShadowFrameLocal(LclVarDsc* varDsc) const; bool isFuncletParameter(unsigned lclNum) const; + unsigned getCurrentShadowFrameSize() const; + unsigned getShadowFrameSize(unsigned hndIndex) const; + unsigned getOriginalShadowFrameSize() const; + unsigned getCatchArgOffset() const; + // ================================================================================================================ // | Codegen | // ================================================================================================================ @@ -273,14 +292,14 @@ class Llvm void Compile(); private: - const int ROOT_FUNC_IDX = 0; + const unsigned ROOT_FUNC_IDX = 0; bool initializeFunctions(); void generateProlog(); void initializeLocals(); - void generateFuncletProlog(unsigned funcIdx); void generateBlock(BasicBlock* block); void generateEHDispatch(); + Value* generateEHDispatchTable(Function* llvmFunc, unsigned innerEHIndex, unsigned outerEHIndex); void fillPhis(); Value* getGenTreeValue(GenTree* node); @@ -313,13 +332,11 @@ class Llvm void buildBinaryOperation(GenTree* node); void buildShift(GenTreeOp* node); void buildReturn(GenTree* node); - void buildCatchArg(GenTree* node); void buildJTrue(GenTree* node); void buildNullCheck(GenTreeIndir* nullCheckNode); void buildBoundsCheck(GenTreeBoundsChk* boundsCheck); void buildCallFinally(BasicBlock* block); - void buildCatchReturn(BasicBlock* block); void storeObjAtAddress(Value* baseAddress, Value* data, StructDesc* structDesc); unsigned buildMemCpy(Value* baseAddress, unsigned startOffset, unsigned endOffset, Value* srcAddress); @@ -328,14 +345,17 @@ class Llvm void emitJumpToThrowHelper(Value* jumpCondValue, SpecialCodeKind throwKind); void emitNullCheckForIndir(GenTreeIndir* indir, Value* addrValue); Value* emitHelperCall(CorInfoHelpFunc helperFunc, ArrayRef sigArgs = { }); - Value* emitCallOrInvoke(llvm::FunctionCallee callee, ArrayRef args); + llvm::CallBase* emitCallOrInvoke(llvm::FunctionCallee callee, ArrayRef args); FunctionType* getFunctionType(); Function* getOrCreateLlvmFunction(const char* symbolName, GenTreeCall* call); FunctionType* createFunctionTypeForCall(GenTreeCall* call); FunctionType* createFunctionTypeForHelper(CorInfoHelpFunc helperFunc); - Value* getOrCreateExternalSymbol(const char* symbolName, Type* symbolType = nullptr); + llvm::GlobalVariable* getOrCreateExternalSymbol(const char* symbolName, Type* symbolType = nullptr); + llvm::GlobalVariable* getOrCreateSymbol(CORINFO_GENERIC_HANDLE symbolHandle, Type* symbolType = nullptr); + Value* emitSymbolRef(CORINFO_GENERIC_HANDLE symbolHandle); + CORINFO_GENERIC_HANDLE getSymbolHandleForClassToken(mdToken token); Instruction* getCast(Value* source, Type* targetType); Value* castIfNecessary(Value* source, Type* targetType, llvm::IRBuilder<>* builder = nullptr); @@ -346,27 +366,32 @@ class Llvm DebugMetadata getOrCreateDebugMetadata(const char* documentFileName); llvm::DILocation* createDebugFunctionAndDiLocation(struct DebugMetadata debugMetadata, unsigned int lineNo); + llvm::DILocation* getArtificialDebugLocation(); void setCurrentEmitContextForBlock(BasicBlock* block); - void setCurrentEmitContext(LlvmBlockRange* llvmBlock, unsigned tryIndex); - LlvmBlockRange* getCurrentLlvmBlocks() const; + void setCurrentEmitContext(unsigned funcIdx, unsigned tryIndex, LlvmBlockRange* llvmBlock); + unsigned getCurrentLlvmFunctionIndex() const; unsigned getCurrentProtectedRegionIndex() const; + LlvmBlockRange* getCurrentLlvmBlocks() const; Function* getRootLlvmFunction(); Function* getCurrentLlvmFunction(); Function* getLlvmFunctionForIndex(unsigned funcIdx); - unsigned getLlvmFunctionIndexForBlock(BasicBlock* block); - unsigned getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex); + FunctionInfo& getLlvmFunctionInfoForIndex(unsigned funcIdx); + unsigned getLlvmFunctionIndexForBlock(BasicBlock* block) const; + unsigned getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex) const; llvm::BasicBlock* createInlineLlvmBlock(); llvm::BasicBlock* getCurrentLlvmBlock() const; LlvmBlockRange* getLlvmBlocksForBlock(BasicBlock* block); llvm::BasicBlock* getFirstLlvmBlockForBlock(BasicBlock* block); llvm::BasicBlock* getLastLlvmBlockForBlock(BasicBlock* block); + llvm::BasicBlock* getOrCreatePrologLlvmBlockForFunction(unsigned funcIdx); + + bool isReachable(BasicBlock* block) const; + BasicBlock* getFirstBlockForFunction(unsigned funcIdx) const; Value* getLocalAddr(unsigned lclNum); - unsigned getTotalLocalOffset(); + Value* getOrCreateAllocaForLocalInFunclet(unsigned lclNum); }; - - #endif /* End of _LLVM_H_ */ diff --git a/src/coreclr/jit/llvmcodegen.cpp b/src/coreclr/jit/llvmcodegen.cpp index db6b898feb57..52851d3e53aa 100644 --- a/src/coreclr/jit/llvmcodegen.cpp +++ b/src/coreclr/jit/llvmcodegen.cpp @@ -20,7 +20,7 @@ void Llvm::Compile() if ((_info.compFlags & CORINFO_FLG_SYNCH) != 0) { - // TODO-LLVM-EH: enable. + // TODO-LLVM: enable. failFunctionCompilation(); } @@ -86,13 +86,13 @@ void Llvm::Compile() JITDUMP("LLVM IR for %s after codegen:\n", _compiler->info.compFullName); JITDUMP("-------------------------------------------------------------------------------------------------------------------\n\n"); - for (Function* llvmFunc : m_functions) + for (FunctionInfo& funcInfo : m_functions) { - JITDUMPEXEC(llvmFunc->dump()); - - if (llvm::verifyFunction(*llvmFunc, &llvm::errs())) + Function* llvmFunc = funcInfo.LlvmFunction; + if (llvmFunc != nullptr) { - printf("Function failed: '%s'\n", llvmFunc->getName().data()); + JITDUMPEXEC(llvmFunc->dump()); + assert(!llvm::verifyFunction(*llvmFunc, &llvm::errs())); } } #endif @@ -122,60 +122,61 @@ bool Llvm::initializeFunctions() } // First functions is always the root. - m_functions = std::vector(_compiler->compFuncCount()); - m_functions[ROOT_FUNC_IDX] = rootLlvmFunction; + m_functions = std::vector(_compiler->compFuncCount()); + m_functions[ROOT_FUNC_IDX] = {rootLlvmFunction}; m_EHDispatchLlvmBlocks = std::vector(_compiler->compHndBBtabCount); - // Note the iteration order: outer -> inner. for (unsigned funcIdx = _compiler->compFuncCount() - 1; funcIdx >= 1; funcIdx--) { FuncInfoDsc* funcInfo = _compiler->funGetFunc(funcIdx); unsigned ehIndex = funcInfo->funEHIndex; EHblkDsc* ehDsc = _compiler->ehGetDsc(ehIndex); - // Filter and catch handler funclets return int32. "HasCatchHandler" handles both cases. - Type* retLlvmType; - if (ehDsc->HasCatchHandler()) - { - retLlvmType = Type::getInt32Ty(_llvmContext); - } - else + // We won't generate code for unreachable handlers so we will not create functions for them. + // + if (isReachable(getFirstBlockForFunction(funcIdx))) { - retLlvmType = Type::getVoidTy(_llvmContext); - } + // Filter and catch handler funclets return int32. "HasCatchHandler" handles both cases. + Type* retLlvmType = + ehDsc->HasCatchHandler() ? Type::getInt32Ty(_llvmContext) : Type::getVoidTy(_llvmContext); - // All funclets two arguments: original and actual shadow stacks. - Type* ptrLlvmType = Type::getInt8PtrTy(_llvmContext); - FunctionType* llvmFuncType = FunctionType::get(retLlvmType, {ptrLlvmType, ptrLlvmType}, /* isVarArg */ false); + // All funclets have two arguments: original and actual shadow stacks. + Type* ptrLlvmType = Type::getInt8PtrTy(_llvmContext); + FunctionType* llvmFuncType = + FunctionType::get(retLlvmType, {ptrLlvmType, ptrLlvmType}, /* isVarArg */ false); - const char* kindName; - switch (ehDsc->ebdHandlerType) - { - case EH_HANDLER_CATCH: - kindName = "Catch"; - break; - case EH_HANDLER_FILTER: - kindName = (funcInfo->funKind == FUNC_FILTER) ? "Filter" : "FilteredCatch"; - break; - case EH_HANDLER_FAULT: - case EH_HANDLER_FAULT_WAS_FINALLY: - kindName = "Fault"; - break; - case EH_HANDLER_FINALLY: - kindName = "Finally"; - break; - default: - unreached(); - } + const char* kindName; + switch (ehDsc->ebdHandlerType) + { + case EH_HANDLER_CATCH: + kindName = "Catch"; + break; + case EH_HANDLER_FILTER: + kindName = (funcInfo->funKind == FUNC_FILTER) ? "Filter" : "FilteredCatch"; + break; + case EH_HANDLER_FAULT: + case EH_HANDLER_FAULT_WAS_FINALLY: + kindName = "Fault"; + break; + case EH_HANDLER_FINALLY: + kindName = "Finally"; + break; + default: + unreached(); + } - Function* llvmFunc = - Function::Create(llvmFuncType, Function::InternalLinkage, - mangledName + llvm::Twine("$F") + llvm::Twine(funcIdx) + "_" + kindName, _module); + Function* llvmFunc = + Function::Create(llvmFuncType, Function::InternalLinkage, + mangledName + llvm::Twine("$F") + llvm::Twine(funcIdx) + "_" + kindName, _module); + + m_functions[funcIdx] = {llvmFunc}; + } // Note that "mutually-protect" handlers will share the same dispatch block. We only need to associate - // one dispatch block with one protected region, and so simply skip the logic for filter funclets. - if (funcInfo->funKind != FUNC_FILTER) + // one dispatch block with one protected region, and so simply skip the logic for filter funclets. We + // also leave blocks for unreachable dispatches null. + if ((funcInfo->funKind == FUNC_HANDLER) && isReachable(ehDsc->ExFlowBlock())) { llvm::BasicBlock* dispatchLlvmBlock = nullptr; @@ -200,8 +201,6 @@ bool Llvm::initializeFunctions() m_EHDispatchLlvmBlocks[ehIndex] = dispatchLlvmBlock; } - - m_functions[funcIdx] = llvmFunc; } return false; @@ -211,20 +210,15 @@ void Llvm::generateProlog() { JITDUMP("\n=============== Generating prolog:\n"); - llvm::BasicBlock* prologLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "Prolog", getRootLlvmFunction()); - _prologBuilder.SetInsertPoint(prologLlvmBlock); + llvm::BasicBlock* prologLlvmBlock = getOrCreatePrologLlvmBlockForFunction(ROOT_FUNC_IDX); + _builder.SetInsertPoint(prologLlvmBlock->getTerminator()); initializeLocals(); - - llvm::BasicBlock* firstLlvmBlock = getFirstLlvmBlockForBlock(_compiler->fgFirstBB); - _prologBuilder.CreateBr(firstLlvmBlock); - - _builder.SetInsertPoint(firstLlvmBlock); } void Llvm::initializeLocals() { - m_allocas = std::vector(_compiler->lvaCount, nullptr); + llvm::AllocaInst** allocas = new (_compiler->getAllocator(CMK_Codegen)) llvm::AllocaInst*[_compiler->lvaCount]; for (unsigned lclNum = 0; lclNum < _compiler->lvaCount; lclNum++) { @@ -306,7 +300,7 @@ void Llvm::initializeLocals() if (initValue == nullptr) { initValue = llvm::UndefValue::get(lclLlvmType); - initValue = _prologBuilder.CreateFreeze(initValue); + initValue = _builder.CreateFreeze(initValue); JITDUMPEXEC(initValue->dump()); } @@ -318,19 +312,16 @@ void Llvm::initializeLocals() } else { - Instruction* allocaInst = _prologBuilder.CreateAlloca(lclLlvmType); - m_allocas[lclNum] = allocaInst; + llvm::AllocaInst* allocaInst = _builder.CreateAlloca(lclLlvmType); + allocas[lclNum] = allocaInst; JITDUMPEXEC(allocaInst->dump()); - Instruction* storeInst = _prologBuilder.CreateStore(initValue, allocaInst); + Instruction* storeInst = _builder.CreateStore(initValue, allocaInst); JITDUMPEXEC(storeInst->dump()); } } -} -void Llvm::generateFuncletProlog(unsigned funcIdx) -{ - // TODO-LLVM-EH: create allocas for all tracked locals that are not in SSA or the shadow stack. + getLlvmFunctionInfoForIndex(ROOT_FUNC_IDX).Allocas = allocas; } void Llvm::generateBlock(BasicBlock* block) @@ -368,9 +359,6 @@ void Llvm::generateBlock(BasicBlock* block) _builder.CreateRetVoid(); } break; - case BBJ_EHCATCHRET: - buildCatchReturn(block); - break; default: // TODO-LLVM: other jump kinds. break; @@ -388,7 +376,8 @@ void Llvm::generateEHDispatch() // Recover the C++ personality function. Type* ptrLlvmType = Type::getInt8PtrTy(_llvmContext); Type* int32LlvmType = Type::getInt32Ty(_llvmContext); - Type* cppExcTupleLlvmType = llvm::StructType::get(_llvmContext, {ptrLlvmType, int32LlvmType}); + Type* cppExcTupleLlvmType = llvm::StructType::get(ptrLlvmType, int32LlvmType); + llvm::StructType* dispatchDataLlvmType = llvm::StructType::get(cppExcTupleLlvmType, ptrLlvmType); static const char* const GXX_PERSONALITY_NAME = "__gxx_personality_v0"; Function* gxxPersonalityLlvmFunc = _module->getFunction(GXX_PERSONALITY_NAME); @@ -400,56 +389,519 @@ void Llvm::generateEHDispatch() Function::Create(gxxPersonalityLlvmFuncType, Function::ExternalLinkage, GXX_PERSONALITY_NAME, _module); } - JITDUMP("\n"); + // Recover the runtime helper routines. TODO-LLVM: use proper "CorInfoHelpFunc"s for these. + Function* dispatchLlvmFuncs[EH_HANDLER_COUNT]{}; + auto getDispatchLlvmFunc = [&](EHblkDsc* hndDsc) { + Function** pDispatchLlvmFunc; + if (hndDsc == nullptr) + { + static_assert_no_msg(EH_HANDLER_CATCH == 1); // We rely on zero being invalid. + pDispatchLlvmFunc = &dispatchLlvmFuncs[0]; + } + else + { + pDispatchLlvmFunc = &dispatchLlvmFuncs[hndDsc->ebdHandlerType]; + } - for (unsigned ehIndex = 0; ehIndex < _compiler->compHndBBtabCount; ehIndex++) + Function* dispatchLlvmFunc = *pDispatchLlvmFunc; + if (dispatchLlvmFunc == nullptr) + { + CORINFO_EH_CLAUSE_FLAGS eeHndType = (hndDsc == nullptr) ? CORINFO_EH_CLAUSE_SAMETRY + : ToCORINFO_EH_CLAUSE_FLAGS(hndDsc->ebdHandlerType); + const char* dispatchFuncName = GetEHDispatchFunctionName(eeHndType); + dispatchLlvmFunc = _module->getFunction(dispatchFuncName); + if (dispatchLlvmFunc == nullptr) + { + FunctionType* dispatchLlvmFuncType; + switch (eeHndType) + { + case CORINFO_EH_CLAUSE_NONE: + case CORINFO_EH_CLAUSE_FILTER: + dispatchLlvmFuncType = FunctionType::get( + int32LlvmType, {ptrLlvmType, ptrLlvmType, ptrLlvmType, ptrLlvmType, ptrLlvmType}, + /* isVarArg */ false); + break; + case CORINFO_EH_CLAUSE_SAMETRY: + dispatchLlvmFuncType = FunctionType::get( + int32LlvmType, {ptrLlvmType, ptrLlvmType, ptrLlvmType, ptrLlvmType}, /* isVarArg */ false); + break; + case CORINFO_EH_CLAUSE_FAULT: + case CORINFO_EH_CLAUSE_FINALLY: + dispatchLlvmFuncType = FunctionType::get( + Type::getVoidTy(_llvmContext), {ptrLlvmType, ptrLlvmType, ptrLlvmType, ptrLlvmType}, + /* isVarArg */ false); + break; + default: + unreached(); + } + + dispatchLlvmFunc = + Function::Create(dispatchLlvmFuncType, Function::ExternalLinkage, dispatchFuncName, _module); + *pDispatchLlvmFunc = dispatchLlvmFunc; + } + } + + return dispatchLlvmFunc; + }; + + BitVecTraits blockVecTraits(_compiler->fgBBNumMax + 1, _compiler); + + struct DispatchData + { + llvm::SwitchInst* DispatchSwitchInst; + BitVec DispatchSwitchTargets; + unsigned LastDispatchSwitchTargetIndex; + llvm::BasicBlock* ResumeLlvmBlock; + Value* DispatchDataRefValue; + + llvm::BasicBlock* GetDispatchSwitchLlvmBlock() const + { + return (DispatchSwitchInst != nullptr) ? DispatchSwitchInst->getParent() : nullptr; + } + }; + + // There is no meaningful source location we can attach to the dispatch blocks. None of them are "user" code. + llvm::DebugLoc dispatchDebugLoc = getArtificialDebugLocation(); + std::vector functionData(_compiler->compFuncCount()); + + // Note the iteration order: inner -> outer. + for (unsigned ehIndex = _compiler->compHndBBtabCount - 1; ehIndex != -1; ehIndex--) { EHblkDsc* ehDsc = _compiler->ehGetDsc(ehIndex); - llvm::BasicBlock* llvmBlock = m_EHDispatchLlvmBlocks[ehIndex]; + llvm::BasicBlock* dispatchPadLlvmBlock = m_EHDispatchLlvmBlocks[ehIndex]; - if (!llvmBlock->empty()) + if (dispatchPadLlvmBlock == nullptr) { - // We've already generated code for this dispatch shared between "mutual protect" handlers. + // Would have been unreachable. continue; } - JITDUMP("=============== Generating EH dispatch block %s:\n", llvmBlock->getName().data()); - - if (llvmBlock->hasNPredecessors(0)) + if (!dispatchPadLlvmBlock->empty()) { - // During codegen we have discovered there are no (EH) edges to this dispatch. Remove it. - JITDUMP("\nUnreachable; skipped\n"); - llvmBlock->eraseFromParent(); + // We've already generated code for this dispatch shared between "mutual protect" handlers. continue; } - Function* llvmFunc = llvmBlock->getParent(); + unsigned funcIdx = getLlvmFunctionIndexForProtectedRegion(ehIndex); + Function* llvmFunc = getLlvmFunctionForIndex(funcIdx); if (!llvmFunc->hasPersonalityFn()) { llvmFunc->setPersonalityFn(gxxPersonalityLlvmFunc); } - // Move it after the last block in the corresponding protected region. - llvmBlock->moveAfter(getLastLlvmBlockForBlock(ehDsc->ebdTryLast)); + // The code we will generate effectively inlines the usual runtime dispatch logic. The main reason this + // scheme was chosen is the fact (re)throwing funclets are handled by it semlessly and efficiently. The + // downside to it is the code size overhead of the calls made for each protected region. + // + // DISPATCH_PAD_INNER: + // dispatchData.CppExceptionTuple = landingPadInst + // dispatchData.DispatcherData = null + // goto DISPATCH_INNER; + // + // DISPATCH_INNER: + // dispatchDest = DispatchFunction(FuncletShadowStack(), &dispatchData, &HandlerFunclet, ...) + // unwind to DISPATCH_PAD_OUTER + // if (dispatchDest == 0) + // goto DISPATCH_OUTER; // For nested regions; top-level ones will use the "switch". + // goto UNIFIED_DISPATCH; + // + // UNIFIED_DISPATCH: + // switch (dispatchDest) { + // case 0: goto RESUME; + // case 1: goto BB01; + // case 2: goto BB02; + // ... + // default: goto FAIL_FAST; + // } + // + // RESUME: + // resume(dispatchData.CppExceptionTuple); // Rethrow the exception and unwind to caller. + // + // FAIL_FAST: + // FailFast(); + // + // What is the possibe set of dispatch destinations (aka why have "UNIFIED_DISPATCH")? + // + // We consider the tree of active protected regions above this one, that are also contained in the same funclet. + // For each region with a (possibly filtered) catch handler, we consider successors of all "catchret" blocks. + // The union of these will form the set of all possible dispatch destinations for the current protected region. + // However, we do not actually emit the "switch" code for each individual region, as it would mean quadratic + // code size growth (number of dispatch destinations X number of protected regions) for deeply nested EH trees. + // Instead, we create one "universal" dispatch block for each funclet, and jump to it from each dispatch. Note + // that thanks to the step blocks inserted by "impImportLeave", we do not need to consider cases where a jump + // from a funclet to its caller would be required. + + // Create the dispatch data alloca. Its structure is a contract between codegen and runtime. The runtime may + // not modify the part where codegen stores the landing pad value, while the other part will be solely under + // runtime's control (currently, this is just one pointer-sized field). + DispatchData& funcDispatchData = functionData[funcIdx]; + Value* dispatchDataRefValue = funcDispatchData.DispatchDataRefValue; + if (dispatchDataRefValue == nullptr) + { + llvm::BasicBlock* prologLlvmBlock = getOrCreatePrologLlvmBlockForFunction(funcIdx); + + _builder.SetInsertPoint(prologLlvmBlock->getTerminator()); + dispatchDataRefValue = _builder.CreateAlloca(dispatchDataLlvmType); - _builder.SetInsertPoint(llvmBlock); + funcDispatchData.DispatchDataRefValue = dispatchDataRefValue; + } + + // Dispatch blocks, when initially created, are placed at the start of the function. + // Here we move them to a more appropriate place so that the entry block is correct. + if (funcDispatchData.GetDispatchSwitchLlvmBlock() != nullptr) + { + dispatchPadLlvmBlock->moveBefore(funcDispatchData.GetDispatchSwitchLlvmBlock()); + } + else if (funcDispatchData.ResumeLlvmBlock != nullptr) + { + dispatchPadLlvmBlock->moveBefore(funcDispatchData.ResumeLlvmBlock); + } + else + { + dispatchPadLlvmBlock->moveAfter(&llvmFunc->back()); + } + _builder.SetCurrentDebugLocation(dispatchDebugLoc); + + LlvmBlockRange dispatchLlvmBlocks(dispatchPadLlvmBlock); + setCurrentEmitContext(funcIdx, ehDsc->ebdEnclosingTryIndex, &dispatchLlvmBlocks); llvm::LandingPadInst* landingPadInst = _builder.CreateLandingPad(cppExcTupleLlvmType, 1); landingPadInst->addClause(llvm::Constant::getNullValue(ptrLlvmType)); // Catch all C++ exceptions. - // TODO-LLVM-EH: implement. - _builder.CreateResume(landingPadInst); + // TODO-LLVM: delete the bitcasts once we move to opaque pointers. + _builder.CreateStore(landingPadInst, + _builder.CreateBitCast(dispatchDataRefValue, landingPadInst->getType()->getPointerTo())); + + // The dispatchers rely on this being set to null to detect whether the ongoing dispatch is already "active". + unsigned dispatcherDataFieldOffset = + _module->getDataLayout().getStructLayout(dispatchDataLlvmType)->getElementOffset(1); + Value* dispatchDataFieldRefValue = _builder.CreateBitCast(dispatchDataRefValue, ptrLlvmType); + dispatchDataFieldRefValue = gepOrAddr(dispatchDataFieldRefValue, dispatcherDataFieldOffset); + _builder.CreateStore(llvm::Constant::getNullValue(ptrLlvmType), + _builder.CreateBitCast(dispatchDataFieldRefValue, ptrLlvmType->getPointerTo())); + + // The "actual" dispatch block. Nested dispatches (if any) will branch to it. + llvm::BasicBlock* dispatchLlvmBlock = createInlineLlvmBlock(); + _builder.CreateBr(dispatchLlvmBlock); + _builder.SetInsertPoint(dispatchLlvmBlock); + + // The dispatcher uses the passed-in shadow stack pointer to call funclets. All funclets (no matter how + // nested) share the same original shadow frame, thus we need to pass the original shadow stack in case + // the exception is being dispatched out of a funclet. + Value* dispatcherShadowStackValue = getShadowStackForCallee(); + Value* funcletShadowStackValue = getOriginalShadowStack(); + // TODO-LLVM: delete the bitcasts once we move to opaque pointers. + Value* castDispatchDataRefValue = _builder.CreateBitCast(dispatchDataRefValue, ptrLlvmType); + + // Do we only have one (catch) handler? We will use specialized dispatchers for this case as an optimization: + // about 2/3 of all EH handlers in optimized code are finallys/faults, ~28% - single catches, with the rest + // (single filters / 2+ mutually protecting handlers) comprising less than 5% of cases. We could drop the + // specialized filter dispatcher here, but it doesn't cost us much to have one, and it is considerably more + // efficient than the general table-based one (and more than 4/5 of all filters are "single"). + // + unsigned innerEHIndex = ehIndex; + while ((innerEHIndex > 0) && ehDsc->ebdIsSameTry(_compiler, innerEHIndex - 1)) + { + innerEHIndex--; + } -#ifdef DEBUG - llvm::BasicBlock* lastLlvmBlock = llvmBlock; - for (; llvmBlock != lastLlvmBlock->getNextNode(); llvmBlock = llvmBlock->getNextNode()) + llvm::CallBase* dispatchDestValue = nullptr; + if (innerEHIndex == ehIndex) { - JITDUMPEXEC(llvmBlock->dump()); + Function* dispatchLlvmFunc = getDispatchLlvmFunc(ehDsc); + Value* handlerValue = _builder.CreateBitCast(getLlvmFunctionForIndex(ehDsc->ebdFuncIndex), ptrLlvmType); + + if (ehDsc->ebdHandlerType == EH_HANDLER_CATCH) + { + Value* typeSymbolRefValue = emitSymbolRef(getSymbolHandleForClassToken(ehDsc->ebdTyp)); + // TODO-LLVM: delete the bitcast once we move to opaque pointers. + typeSymbolRefValue = _builder.CreateBitCast(typeSymbolRefValue, ptrLlvmType); + + dispatchDestValue = emitCallOrInvoke(dispatchLlvmFunc, {dispatcherShadowStackValue, funcletShadowStackValue, + castDispatchDataRefValue, handlerValue, typeSymbolRefValue}); + } + else if (ehDsc->ebdHandlerType == EH_HANDLER_FILTER) + { + // TODO-LLVM: delete the bitcast once we move to opaque pointers. + Value* filterValue = + _builder.CreateBitCast(getLlvmFunctionForIndex(ehDsc->ebdFuncIndex - 1), ptrLlvmType); + + dispatchDestValue = + emitCallOrInvoke(dispatchLlvmFunc, {dispatcherShadowStackValue, funcletShadowStackValue, + castDispatchDataRefValue, handlerValue, filterValue}); + } + else + { + dispatchDestValue = emitCallOrInvoke(dispatchLlvmFunc, {dispatcherShadowStackValue, + funcletShadowStackValue, castDispatchDataRefValue, handlerValue}); + } + } + else + { + Function* dispatchLlvmFunc = getDispatchLlvmFunc(nullptr); + Value* dispatchTableRefValue = generateEHDispatchTable(llvmFunc, innerEHIndex, ehIndex); + // TODO-LLVM: delete the bitcast once we move to opaque pointers. + dispatchTableRefValue = _builder.CreateBitCast(dispatchTableRefValue, ptrLlvmType); + + dispatchDestValue = + emitCallOrInvoke(dispatchLlvmFunc, {dispatcherShadowStackValue, funcletShadowStackValue, + castDispatchDataRefValue, dispatchTableRefValue}); + } + + // Generate code for per-funclet dispatch blocks. The dispatch switch block is only needed if we have + // catch handlers. The resume block is always needed. + // + llvm::BasicBlock* resumeLlvmBlock = funcDispatchData.ResumeLlvmBlock; + if (resumeLlvmBlock == nullptr) + { + resumeLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "BBDR", llvmFunc); + + _builder.SetInsertPoint(resumeLlvmBlock); // No need for a full emit context. + Value* castDispatchDataRefValue = // TODO-LLVM: delete the bitcast once we move to opaque pointers. + _builder.CreateBitCast(dispatchDataRefValue, landingPadInst->getType()->getPointerTo()); + Value* resumeOperandValue = _builder.CreateLoad(landingPadInst->getType(), castDispatchDataRefValue); + _builder.CreateResume(resumeOperandValue); + + funcDispatchData.ResumeLlvmBlock = resumeLlvmBlock; + } + + const int EH_CONTINUE_SEARCH = 0; + + llvm::BasicBlock* dispatchSwitchLlvmBlock = funcDispatchData.GetDispatchSwitchLlvmBlock(); + if (ehDsc->HasCatchHandler() && (dispatchSwitchLlvmBlock == nullptr)) + { + dispatchSwitchLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "BBDS", llvmFunc, resumeLlvmBlock); + llvm::BasicBlock* failFastLlvmBlock = llvm::BasicBlock::Create(_llvmContext, "BBFF", llvmFunc); + + LlvmBlockRange dispatchSwitchLlvmBlocks(dispatchSwitchLlvmBlock); + setCurrentEmitContext(funcIdx, EHblkDsc::NO_ENCLOSING_INDEX, &dispatchSwitchLlvmBlocks); + + llvm::SwitchInst* switchInst = _builder.CreateSwitch(dispatchDestValue, failFastLlvmBlock); + switchInst->addCase(_builder.getInt32(EH_CONTINUE_SEARCH), resumeLlvmBlock); + + LlvmBlockRange failFastLlvmBlocks(failFastLlvmBlock); + setCurrentEmitContext(funcIdx, EHblkDsc::NO_ENCLOSING_INDEX, &failFastLlvmBlocks); + + emitHelperCall(CORINFO_HELP_FAIL_FAST); + _builder.CreateUnreachable(); + + funcDispatchData.DispatchSwitchInst = switchInst; + funcDispatchData.DispatchSwitchTargets = BitVecOps::MakeEmpty(&blockVecTraits); + } + + llvm::BasicBlock* outerDispatchLlvmBlock = nullptr; + if (llvm::isa(dispatchDestValue)) + { + // This will be the "dispatch pad" block. Since we're generating dispatches from outer to inner, we already + // have the "actual" dispatch block; it will be the next one. + outerDispatchLlvmBlock = llvm::cast(dispatchDestValue)->getUnwindDest(); + outerDispatchLlvmBlock = outerDispatchLlvmBlock->getNextNode(); + assert(outerDispatchLlvmBlock != nullptr); + } + + // Reset context back to the dispatch block. + setCurrentEmitContext(funcIdx, ehDsc->ebdEnclosingTryIndex, &dispatchLlvmBlocks); + + // For inner dispatch, jump to the outer one if the handler returned "continue search". Faults / finallys cannot + // satisfy the first-pass search and so for them this jump is unconditional. + llvm::BasicBlock* lastDispatchLlvmBlock = dispatchLlvmBlocks.LastBlock; + if (ehDsc->HasCatchHandler()) + { + Value* unifiedDispatchDestValue = funcDispatchData.DispatchSwitchInst->getOperand(0); + if (unifiedDispatchDestValue != dispatchDestValue) + { + llvm::PHINode* phiNode = llvm::dyn_cast(unifiedDispatchDestValue); + if (phiNode == nullptr) + { + phiNode = + llvm::PHINode::Create(dispatchDestValue->getType(), 2, "", funcDispatchData.DispatchSwitchInst); + phiNode->addIncoming(unifiedDispatchDestValue, dispatchSwitchLlvmBlock->getUniquePredecessor()); + + funcDispatchData.DispatchSwitchInst->setOperand(0, phiNode); + } + + phiNode->addIncoming(dispatchDestValue, lastDispatchLlvmBlock); + } + + if (outerDispatchLlvmBlock != nullptr) + { + Value* doContinueSearchValue = + _builder.CreateICmpEQ(dispatchDestValue, _builder.getInt32(EH_CONTINUE_SEARCH)); + _builder.CreateCondBr(doContinueSearchValue, outerDispatchLlvmBlock, dispatchSwitchLlvmBlock); + } + else + { + _builder.CreateBr(dispatchSwitchLlvmBlock); + } + } + else + { + if (outerDispatchLlvmBlock != nullptr) + { + _builder.CreateBr(outerDispatchLlvmBlock); + } + else + { + _builder.CreateBr(resumeLlvmBlock); + } + } + + // Finally, add in the possible "catchret" destinations. Do not forget to consider all of the mutally protecting + // handlers, since there is only one dispatch block for all of them. Note how we are only doing linear work here + // because the funclet creating process will hoist nested handlers, "flattening" the basic block list. Also, we + // check for the reachability of the handler here, even as we've already checked for whether the dispatch itself + // is reachable. The reason for this is a possibility of a dispatch with a reachable filter but an unreachable + // handler (where the filter always returns false). This is currently, technically, redundant, because RyuJit + // doesn't perform flow optimizations which would expose the handler as unreachable. We choose to be resilient + // against this anyway. + // + if (ehDsc->HasCatchHandler() && isReachable(ehDsc->ebdHndBeg)) + { + llvm::SwitchInst* switchInst = funcDispatchData.DispatchSwitchInst; + BitVec& dispatchSwitchTargets = funcDispatchData.DispatchSwitchTargets; + for (unsigned hndIndex = innerEHIndex; hndIndex <= ehIndex; hndIndex++) + { + EHblkDsc* hndDsc = _compiler->ehGetDsc(hndIndex); + for (BasicBlock* hndBlock : _compiler->Blocks(hndDsc->ebdHndBeg, hndDsc->ebdHndLast)) + { + assert((hndDsc->HasCatchHandler()) && (hndBlock->getHndIndex() == hndIndex)); + if (hndBlock->bbJumpKind == BBJ_EHCATCHRET) + { + BasicBlock* destBlock = hndBlock->bbJumpDest; + llvm::BasicBlock* destLlvmBlock = getFirstLlvmBlockForBlock(destBlock); + assert(destLlvmBlock->getParent() == llvmFunc); // No jumping out of a funclet. + + // We use a bitset to avoid quadratic behavior associated with checking if we have already added + // this dispatch destination - multiple sets of "catchret"s may target the same set of blocks. + unsigned destBlockNum = destBlock->bbNum; + if (!BitVecOps::IsMember(&blockVecTraits, dispatchSwitchTargets, destBlockNum)) + { + unsigned destIndex = ++funcDispatchData.LastDispatchSwitchTargetIndex; + llvm::ConstantInt* destIndexValue = _builder.getInt32(destIndex); + + switchInst->addCase(destIndexValue, destLlvmBlock); + + // Complete the catch return blocks (this one and all the others with the same target). + for (BasicBlock* predBlock : destBlock->PredBlocks()) + { + if (predBlock->bbJumpKind == BBJ_EHCATCHRET) + { + llvm::BasicBlock* catchRetLlvmBlock = getLastLlvmBlockForBlock(predBlock); + llvm::ReturnInst::Create(_llvmContext, destIndexValue, catchRetLlvmBlock); + } + } + + BitVecOps::AddElemD(&blockVecTraits, dispatchSwitchTargets, destBlockNum); + } + } + } + } } -#endif // DEBUG } } +Value* Llvm::generateEHDispatchTable(Function* llvmFunc, unsigned innerEHIndex, unsigned outerEHIndex) +{ + // We only generate this table for a run of mutually protecting handlers. + assert(outerEHIndex > innerEHIndex); + + // The table will have the following format: + // + // [2 (4) bytes: size of table in pointer units] (Means we don't support > ~2^15 clauses) + // [2 (4) bytes: bitmap of clause kinds, 0 - typed, 1 - filter] + // [up to 16 (32) clauses: { void* "Data", void* "Handler" }] + // + // - "Data": exception type symbol pointer / filter handler. + // - "Handler": pointer to the handler + // + // [4 (8) bytes: bitmap of clause kinds] [32 (64) clauses], ... + // + // This is "optimal" for the purposes of targeting WASM, where we cannot encode funclet pointers + // more efficiently using native code offsets. + // + const int LARGE_SECTION_CLAUSE_COUNT = TARGET_POINTER_SIZE * BITS_PER_BYTE; + const int FIRST_SECTION_CLAUSE_COUNT = LARGE_SECTION_CLAUSE_COUNT / 2; + + Type* firstClauseMaskType = Type::getIntNTy(_llvmContext, FIRST_SECTION_CLAUSE_COUNT); + Type* largeClauseMaskType = getIntPtrLlvmType(); + + unsigned clauseCount = outerEHIndex - innerEHIndex + 1; + ArrayStack data(_compiler->getAllocator(CMK_Codegen)); + + data.Push(nullptr); // Placeholder for size. + data.Push(nullptr); // Placeholder for the first mask. + + target_size_t clauseKindMask = 0; + unsigned baseSectionIndex = 0; + unsigned nextSectionIndex = FIRST_SECTION_CLAUSE_COUNT; + for (unsigned index = 0; index < clauseCount; index++) + { + EHblkDsc* ehDsc = _compiler->ehGetDsc(innerEHIndex + index); + unsigned clauseIndex = index - baseSectionIndex; + + llvm::Constant* dataValue; + if (ehDsc->HasFilter()) + { + clauseKindMask |= (target_size_t(1) << clauseIndex); + dataValue = getLlvmFunctionForIndex(ehDsc->ebdFuncIndex - 1); + } + else + { + // Otherwise we need a type symbol reference. + CORINFO_GENERIC_HANDLE typeSymbolHandle = getSymbolHandleForClassToken(ehDsc->ebdTyp); + dataValue = getOrCreateSymbol(typeSymbolHandle); + } + + data.Push(dataValue); + data.Push(getLlvmFunctionForIndex(ehDsc->ebdFuncIndex)); + + // Is this the last entry in the current section? Initialize the mask if so. + bool isEndOfTable = (index + 1) == clauseCount; + bool isEndOfSection = (index + 1) == nextSectionIndex; + if (isEndOfTable || isEndOfSection) + { + Type* clauseMaskType = (baseSectionIndex == 0) ? firstClauseMaskType : largeClauseMaskType; + data.TopRef(2 * (clauseIndex + 1)) = llvm::ConstantInt::get(clauseMaskType, clauseKindMask); + + // Start the next section if needed. + if (!isEndOfTable) + { + clauseKindMask = 0; + data.Push(nullptr); + + baseSectionIndex = nextSectionIndex; + nextSectionIndex += LARGE_SECTION_CLAUSE_COUNT; + } + } + } + + data.BottomRef(0) = llvm::ConstantInt::get(firstClauseMaskType, data.Height() - 1); + + ArrayStack llvmTypeBuilder(_compiler->getAllocator(CMK_Codegen), data.Height()); + for (size_t i = 0; i < data.Height(); i++) + { + llvmTypeBuilder.Push(data.Bottom(i)->getType()); + } + llvm::StructType* tableLlvmType = llvm::StructType::get(_llvmContext, {&llvmTypeBuilder.BottomRef(0), + static_cast(llvmTypeBuilder.Height())}); + llvm::Constant* tableValue = llvm::ConstantStruct::get(tableLlvmType, {&data.BottomRef(0), + static_cast(data.Height())}); + + llvm::GlobalVariable* tableRef = new llvm::GlobalVariable(*_module, tableLlvmType, /* isConstant */ true, + llvm::GlobalVariable::InternalLinkage, tableValue, + llvmFunc->getName() + "__EHTable"); + tableRef->setAlignment(llvm::MaybeAlign(TARGET_POINTER_SIZE)); + + JITDUMP("\nGenerated EH dispatch table for mutually protecting handlers:\n", innerEHIndex, outerEHIndex); + for (unsigned ehIndex = innerEHIndex; ehIndex <= outerEHIndex; ehIndex++) + { + JITDUMPEXEC(_compiler->ehGetDsc(ehIndex)->DispEntry(ehIndex)); + } + JITDUMPEXEC(tableRef->dump()); + + return tableRef; +} + void Llvm::fillPhis() { for (PhiPair phiPair : _phiPairs) @@ -715,9 +1167,6 @@ void Llvm::visitNode(GenTree* node) case GT_RETFILT: buildReturn(node); break; - case GT_CATCH_ARG: - buildCatchArg(node); - break; case GT_STORE_LCL_VAR: buildStoreLocalVar(node->AsLclVar()); break; @@ -1253,9 +1702,8 @@ void Llvm::buildCnsInt(GenTree* node) if (node->IsIconHandle(GTF_ICON_TOKEN_HDL) || node->IsIconHandle(GTF_ICON_CLASS_HDL) || node->IsIconHandle(GTF_ICON_METHOD_HDL) || node->IsIconHandle(GTF_ICON_FIELD_HDL)) { - const char* symbolName = GetMangledSymbolName((void*)(node->AsIntCon()->IconValue())); - AddCodeReloc((void*)node->AsIntCon()->IconValue()); - mapGenTreeToValue(node, _builder.CreateLoad(getOrCreateExternalSymbol(symbolName))); + CORINFO_GENERIC_HANDLE symbolHandle = CORINFO_GENERIC_HANDLE(node->AsIntCon()->IconValue()); + mapGenTreeToValue(node, emitSymbolRef(symbolHandle)); } else { @@ -1274,9 +1722,8 @@ void Llvm::buildCnsInt(GenTree* node) ssize_t intCon = node->AsIntCon()->gtIconVal; if (node->IsIconHandle(GTF_ICON_STR_HDL)) { - const char* symbolName = GetMangledSymbolName((void *)(node->AsIntCon()->IconValue())); - AddCodeReloc((void*)node->AsIntCon()->IconValue()); - mapGenTreeToValue(node, _builder.CreateLoad(getOrCreateExternalSymbol(symbolName))); + CORINFO_GENERIC_HANDLE symbolHandle = CORINFO_GENERIC_HANDLE(node->AsIntCon()->IconValue()); + mapGenTreeToValue(node, emitSymbolRef(symbolHandle)); return; } // TODO: delete this check, just handling string constants and null ptr stores for now, other TYP_REFs not implemented yet @@ -1618,17 +2065,6 @@ void Llvm::buildReturn(GenTree* node) _builder.CreateRet(retValValue); } -void Llvm::buildCatchArg(GenTree* node) -{ - // The exception object is passed as the first shadow parameter to funclets. - Type* excLlvmType = getLlvmTypeForVarType(node->TypeGet()); - // TODO-LLVM: delete once we move to opaque pointers. - Value* shadowStackValue = _builder.CreateBitCast(getShadowStack(), excLlvmType->getPointerTo()); - Value* excObjValue = _builder.CreateLoad(excLlvmType, shadowStackValue); - - mapGenTreeToValue(node, excObjValue); -} - void Llvm::buildJTrue(GenTree* node) { assert(CurrentBlock() != nullptr); @@ -1666,12 +2102,12 @@ void Llvm::buildCallFinally(BasicBlock* block) // Other backends will simply skip generating the second block, while we will branch to it. // Function* finallyLlvmFunc = getLlvmFunctionForIndex(getLlvmFunctionIndexForBlock(block->bbJumpDest)); - Value* shadowStackValue = getShadowStack(); + emitCallOrInvoke(finallyLlvmFunc, {getShadowStackForCallee(), getOriginalShadowStack()}); - // For a locally invoked finally, both the original and actual shadow stacks have the same value. - emitCallOrInvoke(finallyLlvmFunc, {shadowStackValue, shadowStackValue}); - - if ((block->bbFlags & BBF_RETLESS_CALL) != 0) + // Some tricky EH flow configurations can make the ALWAYS part of the pair unreachable without + // marking "block" "BBF_RETLESS_CALL". Detect this case by checking if the next block is reachable + // at all. + if (((block->bbFlags & BBF_RETLESS_CALL) != 0) || !isReachable(block)) { _builder.CreateUnreachable(); } @@ -1682,14 +2118,6 @@ void Llvm::buildCallFinally(BasicBlock* block) } } -void Llvm::buildCatchReturn(BasicBlock* block) -{ - assert(block->bbJumpKind == BBJ_EHCATCHRET); - - // TODO-LLVM-EH: return the appropriate value. - _builder.CreateRet(_builder.getInt32(0)); -} - void Llvm::storeObjAtAddress(Value* baseAddress, Value* data, StructDesc* structDesc) { unsigned fieldCount = structDesc->getFieldCount(); @@ -1966,21 +2394,54 @@ FunctionType* Llvm::createFunctionTypeForHelper(CorInfoHelpFunc helperFunc) return llvmFuncType; } -Value* Llvm::getOrCreateExternalSymbol(const char* symbolName, Type* symbolType) +llvm::GlobalVariable* Llvm::getOrCreateExternalSymbol(const char* symbolName, Type* symbolType) { if (symbolType == nullptr) { symbolType = Type::getInt32PtrTy(_llvmContext); } - Value* symbol = _module->getGlobalVariable(symbolName); + llvm::GlobalVariable* symbol = _module->getGlobalVariable(symbolName); if (symbol == nullptr) { - symbol = new llvm::GlobalVariable(*_module, symbolType, false, llvm::GlobalValue::LinkageTypes::ExternalLinkage, (llvm::Constant*)nullptr, symbolName); + symbol = new llvm::GlobalVariable(*_module, symbolType, false, llvm::GlobalValue::LinkageTypes::ExternalLinkage, + nullptr, symbolName); } return symbol; } +llvm::GlobalVariable* Llvm::getOrCreateSymbol(CORINFO_GENERIC_HANDLE symbolHandle, Type* symbolType) +{ + const char* symbolName = GetMangledSymbolName(symbolHandle); + AddCodeReloc(symbolHandle); + llvm::GlobalVariable* symbol = getOrCreateExternalSymbol(symbolName, symbolType); + + return symbol; +} + +Value* Llvm::emitSymbolRef(CORINFO_GENERIC_HANDLE symbolHandle) +{ + // The symbols handled here represent addresses of the relevant entities. + // So e. g. "symbolRefAddrValue" for a type handle would represent TypeHandle*. + Value* symbolRefAddrValue = getOrCreateSymbol(symbolHandle); + Value* symbolRefValue = _builder.CreateLoad(symbolRefAddrValue); + + return symbolRefValue; +} + +CORINFO_GENERIC_HANDLE Llvm::getSymbolHandleForClassToken(mdToken token) +{ + // The importer call here relies on RyuJit not inlining EH (which it currently does not). + CORINFO_RESOLVED_TOKEN resolvedToken; + _compiler->impResolveToken((BYTE*)&token, &resolvedToken, CORINFO_TOKENKIND_Class); + + void* pIndirection = nullptr; + CORINFO_CLASS_HANDLE typeSymbolHandle = _info.compCompHnd->embedClassHandle(resolvedToken.hClass, &pIndirection); + assert(pIndirection == nullptr); + + return CORINFO_GENERIC_HANDLE(typeSymbolHandle); +} + Instruction* Llvm::getCast(Value* source, Type* targetType) { Type* sourceType = source->getType(); @@ -2054,14 +2515,23 @@ Value* Llvm::getShadowStack() // Shadow stack moved up to avoid overwriting anything on the stack in the compiling method Value* Llvm::getShadowStackForCallee() { - // Note that funclets have the shadow stack arg in the same position (0) as the main function. - return gepOrAddr(getShadowStack(), getTotalLocalOffset()); + unsigned hndIndex = EHblkDsc::NO_ENCLOSING_INDEX; + if (getCurrentProtectedRegionIndex() != EHblkDsc::NO_ENCLOSING_INDEX) + { + hndIndex = _compiler->ehGetEnclosingHndIndex(getCurrentProtectedRegionIndex()); + } + + return gepOrAddr(getShadowStack(), getShadowFrameSize(hndIndex)); } Value* Llvm::getOriginalShadowStack() { + if (getCurrentLlvmFunction() == getRootLlvmFunction()) + { + return getShadowStack(); + } + // The original shadow stack pointer is the second funclet parameter. - assert(getCurrentLlvmFunction() != getRootLlvmFunction()); return getCurrentLlvmFunction()->getArg(1); } @@ -2118,12 +2588,23 @@ llvm::DILocation* Llvm::createDebugFunctionAndDiLocation(DebugMetadata debugMeta functionMetaType, lineNumber, llvm::DINode::DIFlags::FlagZero, llvm::DISubprogram::DISPFlags::SPFlagDefinition | llvm::DISubprogram::DISPFlags::SPFlagLocalToUnit); - // TODO-LLVM-EH: broken for funclets. + // TODO-LLVM-EH: debugging in funclets. getRootLlvmFunction()->setSubprogram(_debugFunction); } return llvm::DILocation::get(_llvmContext, lineNo, 0, _debugFunction); } +llvm::DILocation* Llvm::getArtificialDebugLocation() +{ + if (_debugFunction == nullptr) + { + return nullptr; + } + + // Line number "0" is used to represent non-user code in DWARF. + return llvm::DILocation::get(_llvmContext, 0, 0, _debugFunction); +} + llvm::BasicBlock* Llvm::getCurrentLlvmBlock() const { return getCurrentLlvmBlocks()->LastBlock; @@ -2131,27 +2612,30 @@ llvm::BasicBlock* Llvm::getCurrentLlvmBlock() const void Llvm::setCurrentEmitContextForBlock(BasicBlock* block) { - LlvmBlockRange* llvmBlocks = getLlvmBlocksForBlock(block); + unsigned funcIdx = getLlvmFunctionIndexForBlock(block); unsigned tryIndex = block->hasTryIndex() ? block->getTryIndex() : EHblkDsc::NO_ENCLOSING_INDEX; + LlvmBlockRange* llvmBlocks = getLlvmBlocksForBlock(block); - setCurrentEmitContext(llvmBlocks, tryIndex); + setCurrentEmitContext(funcIdx, tryIndex, llvmBlocks); m_currentBlock = block; } -void Llvm::setCurrentEmitContext(LlvmBlockRange* llvmBlocks, unsigned tryIndex) +void Llvm::setCurrentEmitContext(unsigned funcIdx, unsigned tryIndex, LlvmBlockRange* llvmBlocks) { + assert(getLlvmFunctionForIndex(funcIdx) == llvmBlocks->LastBlock->getParent()); + _builder.SetInsertPoint(llvmBlocks->LastBlock); - m_currentLlvmBlocks = llvmBlocks; + m_currentLlvmFunctionIndex = funcIdx; m_currentProtectedRegionIndex = tryIndex; + m_currentLlvmBlocks = llvmBlocks; // "Raw" emission contexts do not have a current IR block. m_currentBlock = nullptr; } -LlvmBlockRange* Llvm::getCurrentLlvmBlocks() const +unsigned Llvm::getCurrentLlvmFunctionIndex() const { - assert(m_currentLlvmBlocks != nullptr); - return m_currentLlvmBlocks; + return m_currentLlvmFunctionIndex; } //------------------------------------------------------------------------ @@ -2166,6 +2650,12 @@ unsigned Llvm::getCurrentProtectedRegionIndex() const return m_currentProtectedRegionIndex; } +LlvmBlockRange* Llvm::getCurrentLlvmBlocks() const +{ + assert(m_currentLlvmBlocks != nullptr); + return m_currentLlvmBlocks; +} + Function* Llvm::getRootLlvmFunction() { return getLlvmFunctionForIndex(ROOT_FUNC_IDX); @@ -2173,18 +2663,23 @@ Function* Llvm::getRootLlvmFunction() Function* Llvm::getCurrentLlvmFunction() { - return getCurrentLlvmBlock()->getParent(); + return getLlvmFunctionForIndex(getCurrentLlvmFunctionIndex()); } Function* Llvm::getLlvmFunctionForIndex(unsigned funcIdx) { - Function* llvmFunc = m_functions[funcIdx]; - assert(llvmFunc != nullptr); + return getLlvmFunctionInfoForIndex(funcIdx).LlvmFunction; +} - return llvmFunc; +FunctionInfo& Llvm::getLlvmFunctionInfoForIndex(unsigned funcIdx) +{ + FunctionInfo& funcInfo = m_functions[funcIdx]; + assert(funcInfo.LlvmFunction != nullptr); + + return funcInfo; } -unsigned Llvm::getLlvmFunctionIndexForBlock(BasicBlock* block) +unsigned Llvm::getLlvmFunctionIndexForBlock(BasicBlock* block) const { unsigned funcIdx = ROOT_FUNC_IDX; @@ -2205,7 +2700,7 @@ unsigned Llvm::getLlvmFunctionIndexForBlock(BasicBlock* block) return funcIdx; } -unsigned Llvm::getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex) +unsigned Llvm::getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex) const { unsigned funcIdx = ROOT_FUNC_IDX; if (tryIndex != EHblkDsc::NO_ENCLOSING_INDEX) @@ -2249,7 +2744,8 @@ llvm::BasicBlock* Llvm::createInlineLlvmBlock() LlvmBlockRange* Llvm::getLlvmBlocksForBlock(BasicBlock* block) { - assert(block != nullptr); + // We should never be asking for unreachable blocks here since we won't generate code for them. + assert(isReachable(block) || (block == _compiler->fgFirstBB) || _compiler->fgIsThrowHlpBlk(block)); LlvmBlockRange* llvmBlockRange = _blkToLlvmBlksMap.LookupPointer(block); if (llvmBlockRange == nullptr) @@ -2286,16 +2782,108 @@ llvm::BasicBlock* Llvm::getLastLlvmBlockForBlock(BasicBlock* block) return getLlvmBlocksForBlock(block)->LastBlock; } +llvm::BasicBlock* Llvm::getOrCreatePrologLlvmBlockForFunction(unsigned funcIdx) +{ + const char* const PROLOG_BLOCK_NAME = "BB00"; + + BasicBlock* firstUserBlock = getFirstBlockForFunction(funcIdx); + llvm::BasicBlock* firstLlvmUserBlock = getFirstLlvmBlockForBlock(firstUserBlock); + llvm::BasicBlock* prologLlvmBlock = firstLlvmUserBlock->getPrevNode(); + if ((prologLlvmBlock == nullptr) || !prologLlvmBlock->getName().equals(PROLOG_BLOCK_NAME)) + { + Function* llvmFunc = firstLlvmUserBlock->getParent(); + prologLlvmBlock = llvm::BasicBlock::Create(_llvmContext, PROLOG_BLOCK_NAME, llvmFunc, firstLlvmUserBlock); + + // Eagerly insert jump to the user block to simplify calling code. + llvm::BranchInst::Create(firstLlvmUserBlock, prologLlvmBlock); + } + + return prologLlvmBlock; +} + +//------------------------------------------------------------------------ +// isReachable: Does this block have an immediate dominator? +// +// Arguments: +// block - The block to check +// +// Return Value: +// Whether "block" has an immediate dominator, i. e. is statically +// reachable, not the first block, and not a throw helper block. +// +bool Llvm::isReachable(BasicBlock* block) const +{ + return block->bbIDom != nullptr; +} + +BasicBlock* Llvm::getFirstBlockForFunction(unsigned funcIdx) const +{ + if (funcIdx == ROOT_FUNC_IDX) + { + return _compiler->fgFirstBB; + } + + FuncInfoDsc* funcInfo = _compiler->funGetFunc(funcIdx); + EHblkDsc* ehDsc = _compiler->ehGetDsc(funcInfo->funEHIndex); + return (funcInfo->funKind == FUNC_FILTER) ? ehDsc->ebdFilter : ehDsc->ebdHndBeg; +} + Value* Llvm::getLocalAddr(unsigned lclNum) { - Value* addrValue = m_allocas[lclNum]; - assert(addrValue != nullptr); + Value* addrValue; + if (getCurrentLlvmFunction() == getRootLlvmFunction()) + { + addrValue = getLlvmFunctionInfoForIndex(ROOT_FUNC_IDX).Allocas[lclNum]; + } + else + { + addrValue = getOrCreateAllocaForLocalInFunclet(lclNum); + } + assert(addrValue != nullptr); return addrValue; } -unsigned int Llvm::getTotalLocalOffset() +//------------------------------------------------------------------------ +// getOrCreateAllocaForLocalInFunclet: Get an address for a funclet local. +// +// For a local to be (locally) live on the LLVM frame in a funclet, it has +// to be tracked and have its address taken (but not exposed!). Such locals +// are rare, and it is not cheap to indentify their set precisely before +// the code has been generated. We therefore use a lazy strategy for their +// materialization in the funclet prologs. +// +// Arguments: +// lclNum - The local for which to get the allocated home +// +// Return Value: +// Address on the LLVM frame "lclNum" has in the current funclet. +// +Value* Llvm::getOrCreateAllocaForLocalInFunclet(unsigned lclNum) { - assert((_shadowStackLocalsSize % TARGET_POINTER_SIZE) == 0); - return _shadowStackLocalsSize; + LclVarDsc* varDsc = _compiler->lvaGetDesc(lclNum); + assert(varDsc->lvTracked); // Untracked locals in functions with funclets live on the shadow frame. + + unsigned funcIdx = getCurrentLlvmFunctionIndex(); + assert(funcIdx != ROOT_FUNC_IDX); // The root's prolog is generated eagerly. + assert(!VarSetOps::IsMember(_compiler, getFirstBlockForFunction(funcIdx)->bbLiveIn, varDsc->lvVarIndex)); + + FunctionInfo& funcInfo = getLlvmFunctionInfoForIndex(funcIdx); + AllocaMap* allocaMap = funcInfo.AllocaMap; + if (allocaMap == nullptr) + { + allocaMap = new (_compiler->getAllocator(CMK_Codegen)) AllocaMap(_compiler->getAllocator(CMK_Codegen)); + funcInfo.AllocaMap = allocaMap; + } + + llvm::AllocaInst* allocaInst; + if (!allocaMap->Lookup(lclNum, &allocaInst)) + { + llvm::BasicBlock* prologLlvmBlock = getOrCreatePrologLlvmBlockForFunction(funcIdx); + allocaInst = new llvm::AllocaInst(getLlvmTypeForLclVar(varDsc), 0, "", prologLlvmBlock->getTerminator()); + + allocaMap->Set(lclNum, allocaInst); + } + + return allocaInst; } diff --git a/src/coreclr/jit/llvmlower.cpp b/src/coreclr/jit/llvmlower.cpp index 26635efc5f8d..5928391bb31b 100644 --- a/src/coreclr/jit/llvmlower.cpp +++ b/src/coreclr/jit/llvmlower.cpp @@ -383,6 +383,10 @@ void Llvm::lowerBlock(BasicBlock* block) lowerCall(node->AsCall()); break; + case GT_CATCH_ARG: + lowerCatchArg(node); + break; + case GT_IND: case GT_STOREIND: case GT_OBJ: @@ -555,6 +559,11 @@ void Llvm::lowerCall(GenTreeCall* callNode) { failUnsupportedCalls(callNode); + if (callNode->IsHelperCall(_compiler, CORINFO_HELP_RETHROW)) + { + lowerRethrow(callNode); + } + lowerCallToShadowStack(callNode); // If there is a no return, or always throw call, delete the dead code so we can add unreachable @@ -568,6 +577,40 @@ void Llvm::lowerCall(GenTreeCall* callNode) } } +void Llvm::lowerRethrow(GenTreeCall* callNode) +{ + assert(callNode->IsHelperCall(_compiler, CORINFO_HELP_RETHROW)); + + // Language in ECMA 335 I.12.4.2.8.2.2 clearly states that rethrows nested inside finallys are + // legal, however, neither C# nor the old verification system allow this. CoreCLR behavior was + // not tested. Implementing this would imply saving the exception object to the "original" shadow + // frame shared between funclets. For now we punt. + if (!_compiler->ehGetDsc(CurrentBlock()->getHndIndex())->HasCatchHandler()) + { + IMPL_LIMITATION("Nested rethrow"); + } + + // A rethrow is a special throw that preserves the stack trace. Our helper we use for rethrow has + // the equivalent of a managed signature "void (object*)", i. e. takes the exception object address + // explicitly. Add it here, before the general call lowering. + assert(callNode->gtCallArgs == nullptr); + callNode->ResetArgInfo(); + + GenTree* excObjAddr = insertShadowStackAddr(callNode, getCatchArgOffset(), _shadowStackLclNum); + callNode->gtCallArgs = _compiler->gtNewCallArgs(excObjAddr); + + _compiler->fgInitArgInfo(callNode); +} + +void Llvm::lowerCatchArg(GenTree* catchArgNode) +{ + GenTree* excObjAddr = insertShadowStackAddr(catchArgNode, getCatchArgOffset(), _shadowStackLclNum); + + catchArgNode->ChangeOper(GT_IND); + catchArgNode->gtFlags |= GTF_IND_NONFAULTING; + catchArgNode->AsIndir()->SetAddr(excObjAddr); +} + void Llvm::lowerIndir(GenTreeIndir* indirNode) { if ((indirNode->gtFlags & GTF_IND_NONFAULTING) == 0) @@ -746,7 +789,7 @@ void Llvm::lowerCallToShadowStack(GenTreeCall* callNode) GenTreeCall::Use* lastArg = nullptr; if (callHasShadowStackArg(callNode)) { - GenTree* calleeShadowStack = insertShadowStackAddr(callNode, _shadowStackLocalsSize, _shadowStackLclNum); + GenTree* calleeShadowStack = insertShadowStackAddr(callNode, getCurrentShadowFrameSize(), _shadowStackLclNum); GenTreePutArgType* calleeShadowStackPutArg = _compiler->gtNewPutArgType(calleeShadowStack, CORINFO_TYPE_PTR, NO_CLASS_HANDLE); @@ -844,13 +887,14 @@ void Llvm::lowerCallToShadowStack(GenTreeCall* callNode) shadowStackUseOffest = padOffset(corInfoType, clsHnd, shadowStackUseOffest); } + unsigned shadowFrameSize = getCurrentShadowFrameSize(); if (argNode->OperIs(GT_FIELD_LIST)) { for (GenTreeFieldList::Use& use : argNode->AsFieldList()->Uses()) { assert(use.GetType() != TYP_STRUCT); - unsigned fieldOffsetValue = _shadowStackLocalsSize + shadowStackUseOffest + use.GetOffset(); + unsigned fieldOffsetValue = shadowFrameSize + shadowStackUseOffest + use.GetOffset(); GenTree* fieldSlotAddr = insertShadowStackAddr(callNode, fieldOffsetValue, _shadowStackLclNum); GenTree* fieldStoreNode = createShadowStackStoreNode(use.GetType(), fieldSlotAddr, use.GetNode()); @@ -861,7 +905,7 @@ void Llvm::lowerCallToShadowStack(GenTreeCall* callNode) } else { - unsigned offsetValue = _shadowStackLocalsSize + shadowStackUseOffest; + unsigned offsetValue = shadowFrameSize + shadowStackUseOffest; GenTree* slotAddr = insertShadowStackAddr(callNode, offsetValue, _shadowStackLclNum); GenTree* storeNode = createShadowStackStoreNode(argNode->TypeGet(), slotAddr, argNode); @@ -965,9 +1009,7 @@ GenTreeCall::Use* Llvm::lowerCallReturn(GenTreeCall* callNode, GenTreeCall::Use* if (needsReturnStackSlot(_compiler, callNode)) { // replace the "CALL ref" with a "CALL void" that takes a return address as the first argument - GenTreeLclVar* shadowStackVar = _compiler->gtNewLclvNode(_shadowStackLclNum, TYP_I_IMPL); - GenTreeIntCon* offset = _compiler->gtNewIconNode(_shadowStackLocalsSize, TYP_I_IMPL); - GenTree* returnValueAddress = _compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, shadowStackVar, offset); + GenTree* returnValueAddress = insertShadowStackAddr(callNode, getCurrentShadowFrameSize(), _shadowStackLclNum); // create temp for the return address unsigned returnTempNum = _compiler->lvaGrabTemp(false DEBUGARG("return value address")); @@ -1011,8 +1053,7 @@ GenTreeCall::Use* Llvm::lowerCallReturn(GenTreeCall* callNode, GenTreeCall::Use* callNode->gtCorInfoType = CORINFO_TYPE_VOID; callNode->ChangeType(TYP_VOID); - CurrentRange().InsertBefore(callNode, shadowStackVar, offset, returnValueAddress, addrStore); - CurrentRange().InsertAfter(addrStore, returnAddrLcl, putArg); + CurrentRange().InsertBefore(callNode, addrStore, returnAddrLcl, putArg); CurrentRange().InsertAfter(callNode, returnAddrLclAfterCall, indirNode); } else @@ -1184,3 +1225,52 @@ bool Llvm::isFuncletParameter(unsigned lclNum) const { return (lclNum == _shadowStackLclNum) || (lclNum == _originalShadowStackLclNum); } + +unsigned Llvm::getCurrentShadowFrameSize() const +{ + assert(CurrentBlock() != nullptr); + unsigned hndIndex = CurrentBlock()->hasHndIndex() ? CurrentBlock()->getHndIndex() : EHblkDsc::NO_ENCLOSING_INDEX; + return getShadowFrameSize(hndIndex); +} + +//------------------------------------------------------------------------ +// getShadowFrameSize: What is the size of a function's shadow frame? +// +// Arguments: +// hndIndex - Handler index representing the function, NO_ENCLOSING_INDEX +// is used for the root +// +// Return Value: +// The size of the shadow frame for the given function. We term this +// the value by which the shadow stack pointer must be offset before +// calling managed code such that the caller will not clobber anything +// live on the frame. Note that funclets do not have any shadow state +// of their own and use the "original" frame from the parent function, +// with one exception: catch handlers and filters have one readonly +// pointer-sized argument representing the exception. +// +unsigned Llvm::getShadowFrameSize(unsigned hndIndex) const +{ + if (hndIndex == EHblkDsc::NO_ENCLOSING_INDEX) + { + return getOriginalShadowFrameSize(); + } + if (_compiler->ehGetDsc(hndIndex)->HasCatchHandler()) + { + // For the implicit (readonly) exception object argument. + return TARGET_POINTER_SIZE; + } + + return 0; +} + +unsigned Llvm::getOriginalShadowFrameSize() const +{ + assert((_shadowStackLocalsSize % TARGET_POINTER_SIZE) == 0); + return _shadowStackLocalsSize; +} + +unsigned Llvm::getCatchArgOffset() const +{ + return 0; +} diff --git a/src/coreclr/jit/llvmtypes.cpp b/src/coreclr/jit/llvmtypes.cpp index 0c0d0ba7912c..c98019af96e2 100644 --- a/src/coreclr/jit/llvmtypes.cpp +++ b/src/coreclr/jit/llvmtypes.cpp @@ -286,6 +286,15 @@ Type* Llvm::getLlvmTypeForParameterType(CORINFO_CLASS_HANDLE classHnd) return getLlvmTypeForCorInfoType(parameterCorInfoType, innerParameterHandle); } +Type* Llvm::getIntPtrLlvmType() +{ +#ifdef TARGET_64BIT + return Type::getInt64Ty(_llvmContext); +#else + return Type::getInt32Ty(_llvmContext); +#endif +} + unsigned Llvm::getElementSize(CORINFO_CLASS_HANDLE classHandle, CorInfoType corInfoType) { if (classHandle != NO_CLASS_HANDLE) diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs index d019c1022d2c..e94f01a87b38 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs @@ -177,6 +177,9 @@ public DependencyList Import() _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); // TODO-LLVM: make these into normal "ReadyToRunHelper" instead of hardcoding things here. + helperMethod = helperType.GetKnownMethod("HandleExceptionWasmMutuallyProtectingCatches", null); + _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); + helperMethod = helperType.GetKnownMethod("HandleExceptionWasmFilteredCatch", null); _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(helperMethod), "Wasm EH"); diff --git a/src/coreclr/tools/aot/ILCompiler.LLVM/Compiler/LLVMCodegenCompilation.cs b/src/coreclr/tools/aot/ILCompiler.LLVM/Compiler/LLVMCodegenCompilation.cs index 8d2e3b822509..55e8d4f95649 100644 --- a/src/coreclr/tools/aot/ILCompiler.LLVM/Compiler/LLVMCodegenCompilation.cs +++ b/src/coreclr/tools/aot/ILCompiler.LLVM/Compiler/LLVMCodegenCompilation.cs @@ -83,7 +83,6 @@ protected override void CompileInternal(string outputFile, ObjectDumper dumper) LLVMObjectWriter.EmitObject(outputFile, nodes, NodeFactory, this, dumper); - Console.WriteLine($"RyuJIT compilation results, total methods {totalMethodCount} RyuJit Methods {ryuJitMethodCount} {((decimal)ryuJitMethodCount * 100 / totalMethodCount):n4}%"); } @@ -152,29 +151,28 @@ private unsafe void CompileSingleMethod(CorInfoImpl corInfo, LLVMMethodCodeNode methodCodeNodeNeedingCode.CompilationCompleted = true; return; } - - if (methodIL.GetExceptionRegions().Length == 0 && !_disableRyuJit) + if (!_disableRyuJit) { - var mangledName = NodeFactory.NameMangler.GetMangledMethodName(method).ToString(); - var sig = method.Signature; - corInfo.RegisterLlvmCallbacks((IntPtr)Unsafe.AsPointer(ref corInfo), _outputFile, - Module.Target, - Module.DataLayout); corInfo.RegisterLlvmCallbacks((IntPtr)Unsafe.AsPointer(ref corInfo), _outputFile, Module.Target, Module.DataLayout); corInfo.CompileMethod(methodCodeNodeNeedingCode); methodCodeNodeNeedingCode.CompilationCompleted = true; + // TODO: delete this external function when old module is gone + var mangledName = NodeFactory.NameMangler.GetMangledMethodName(method).ToString(); LLVMValueRef externFunc = ILImporter.GetOrCreateLLVMFunction(Module, mangledName, - GetLLVMSignatureForMethod(sig, method.RequiresInstArg())); + GetLLVMSignatureForMethod(method.Signature, method.RequiresInstArg())); externFunc.Linkage = LLVMLinkage.LLVMExternalLinkage; ILImporter.GenerateRuntimeExportThunk(this, method, externFunc); ryuJitMethodCount++; } - else ILImporter.CompileMethod(this, methodCodeNodeNeedingCode); + else + { + ILImporter.CompileMethod(this, methodCodeNodeNeedingCode); + } } catch (CodeGenerationFailedException) { diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.Llvm.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.Llvm.cs index e9e7e04caf6b..a107b3cc5f7c 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.Llvm.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.Llvm.cs @@ -26,8 +26,13 @@ public unsafe sealed partial class CorInfoImpl public static void addCodeReloc(IntPtr thisHandle, void* handle) { var _this = GetThis(thisHandle); - var obj = _this.HandleToObject((IntPtr)handle); + + AddCodeRelocImpl(_this, obj); + } + + private static void AddCodeRelocImpl(CorInfoImpl _this, object obj) + { ISymbolNode node; if (obj is ISymbolNode symbolNode) { @@ -47,7 +52,6 @@ public static void addCodeReloc(IntPtr thisHandle, void* handle) MethodDesc virtualCallTarget = (MethodDesc)helperNode.Target; _this._codeRelocs.Add(new Relocation(RelocType.IMAGE_REL_BASED_REL32, 0, _this._compilation.NodeFactory.MethodEntrypoint(virtualCallTarget))); - return; } @@ -108,10 +112,15 @@ private static byte[] AppendNullByte(byte[] inputArray) public static byte* getMangledMethodName(IntPtr thisHandle, CORINFO_METHOD_STRUCT_* ftn) { var _this = GetThis(thisHandle); - MethodDesc method = _this.HandleToObject(ftn); - return (byte*)_this.GetPin(AppendNullByte(_this._compilation.NameMangler.GetMangledMethodName(method).UnderlyingArray)); + return GetMangledMethodNameImpl(_this, method); + } + + private static byte* GetMangledMethodNameImpl(CorInfoImpl _this, MethodDesc method) + { + Utf8String mangledName = _this._compilation.NameMangler.GetMangledMethodName(method); + return (byte*)_this.GetPin(AppendNullByte(mangledName.UnderlyingArray)); } [UnmanagedCallersOnly] @@ -131,6 +140,31 @@ private static byte[] AppendNullByte(byte[] inputArray) return (byte*)_this.GetPin(sb.UnderlyingArray); } + [UnmanagedCallersOnly] + public static byte* getEHDispatchFunctionName(IntPtr thisHandle, CORINFO_EH_CLAUSE_FLAGS handlerType) + { + var _this = GetThis(thisHandle); + string dispatchMethodName = handlerType switch + { + CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_SAMETRY => "HandleExceptionWasmMutuallyProtectingCatches", + CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_NONE => "HandleExceptionWasmCatch", + CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_FILTER => "HandleExceptionWasmFilteredCatch", + CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_FINALLY or CORINFO_EH_CLAUSE_FLAGS.CORINFO_EH_CLAUSE_FAULT => "HandleExceptionWasmFault", + _ => throw new NotSupportedException() + }; + + // TODO-LLVM: we are breaking the abstraction here. Compiler is not allowed to access methods from the + // managed runtime directly and assume they are compiled into CoreLib. The dispatch routine should be + // made into a RuntimeExport once we solve the issues around calling convention mismatch for them. + MetadataType ehType = _this._compilation.TypeSystemContext.SystemModule.GetKnownType("System.Runtime", "EH"); + MethodDesc dispatchMethod = ehType.GetKnownMethod(dispatchMethodName, null); + + // Codegen is asking for the dispatcher; assume it'll use it. + AddCodeRelocImpl(_this, dispatchMethod); + + return GetMangledMethodNameImpl(_this, dispatchMethod); + } + // IL backend does not use the mangled name. The unmangled name is easier to read. [UnmanagedCallersOnly] public static byte* getTypeName(IntPtr thisHandle, CORINFO_CLASS_STRUCT_* structHnd) @@ -363,6 +397,7 @@ enum EEApiId { GetMangledMethodName, GetSymbolMangledName, + GetEHDispatchFunctionName, GetTypeName, AddCodeReloc, IsRuntimeImport, @@ -386,6 +421,7 @@ public void RegisterLlvmCallbacks(IntPtr corInfoPtr, string outputFileName, stri void** callbacks = stackalloc void*[(int)EEApiId.Count + 1]; callbacks[(int)EEApiId.GetMangledMethodName] = (delegate* unmanaged)&getMangledMethodName; callbacks[(int)EEApiId.GetSymbolMangledName] = (delegate* unmanaged)&getSymbolMangledName; + callbacks[(int)EEApiId.GetEHDispatchFunctionName] = (delegate* unmanaged)&getEHDispatchFunctionName; callbacks[(int)EEApiId.GetTypeName] = (delegate* unmanaged)&getTypeName; callbacks[(int)EEApiId.AddCodeReloc] = (delegate* unmanaged)&addCodeReloc; callbacks[(int)EEApiId.IsRuntimeImport] = (delegate* unmanaged)&isRuntimeImport;