diff --git a/build.sh b/build.sh index ff6488efac7d..b993303e77e4 100755 --- a/build.sh +++ b/build.sh @@ -3,9 +3,41 @@ # the default target triple to x86_64-linux-gnu mkdir -p build cd build +export CFLAGS="-ffixed-x18" +export CXXFLAGS="-ffixed-x18" cmake -GNinja -DLLVM_ENABLE_PROJECTS="clang" \ -DLLVM_TARGETS_TO_BUILD="AArch64" \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCOMPILER_RT_BUILD_BUILTINS=ON \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt" \ -DLLVM_DEFAULT_TARGET_TRIPLE="aarch64-linux-gnu" \ ../llvm ninja clang +# free up disk space by stubbing out files that aren't needed for the next targets +rm -rf _CPack_Packages +for file in lib/libclangStaticAnalyzerCheckers.a lib/libclangCodeGen.a lib/libclangSema.a lib/libclangAST.a lib/libclangStaticAnalyzerCore.a lib/CodeGen/**/CMakeFiles/*.dir/*.o lib/Transforms/**/CMakeFiles/*.dir/*.o; do + mv $file $file.bak + touch $file --reference=$file.bak + rm $file.bak +done +ninja builtins +# free up disk space by deleting binaries that aren't needed once builtins are built +unused_progs=(clang-tblgen +llvm-ar +llvm-min-tblgen +llvm-nm +llvm-objcopy +llvm-objdump +llvm-ranlib +llvm-readelf +llvm-readobj +llvm-size +llvm-strip +llvm-symbolizer +llvm-tblgen +sancov) +for prog in $unused_progs; do + rm bin/$prog +done +# copy to expected target triple +cp -arv lib/clang/19/lib/aarch64-linux-gnu lib/clang/19/lib/aarch64-unknown-linux-gnu diff --git a/cross-build-rtlibs.sh b/cross-build-rtlibs.sh new file mode 100755 index 000000000000..f51d26028767 --- /dev/null +++ b/cross-build-rtlibs.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# build compiler-rt, libcxx, libcxxabi, and libunwind with our patched clang +# see https://llvm.org/docs/HowToCrossCompileBuiltinsOnArm.html +set -x + +# build stub crt object files to prevent wrong-arch ones from being selected +fake_crt_dir=$(pwd)/fake_crt +mkdir -p $fake_crt_dir +for crtobj in crtn.o crti.o Scrt1.o; do + aarch64-linux-gnu-gcc -c -x c /dev/null -o $fake_crt_dir/$crtobj +done + +# build runtime libs +mkdir -p build-rtlibs +cd build-rtlibs + +# find existing aarch64 crt for building test programs +crt_candidates="$(ls -d /usr/lib/gcc/aarch64-linux-gnu/*) +$(ls -d /usr/lib/gcc-cross/aarch64-linux-gnu/*)" +crt_dir=$(echo $crt_candidates | sort -V | tail -n1) + +cross_flags="-B$fake_crt_dir -B$crt_dir --gcc-toolchain=/usr --gcc-triple=aarch64-linux-gnu -isystem /usr/aarch64-linux-gnu/include -march=armv8+memtag -ffixed-x18" +#--sysroot=/usr/aarch64-linux-gnu/ --gcc-install-dir=/usr/lib/gcc/aarch64-linux-gnu/14.1.0 +export LDFLAGS="-L/usr/aarch64-linux-gnu/lib" +cmake -GNinja -DLLVM_TARGETS_TO_BUILD="AArch64" -DLLVM_DEFAULT_TARGET_TRIPLE="aarch64-linux-gnu" \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_C_FLAGS="$cross_flags --target=aarch64-linux-gnu" -DCMAKE_CXX_FLAGS="$cross_flags --target=aarch64-linux-gnu" \ + -DCMAKE_C_COMPILER="$(pwd)/../build/bin/clang" -DCMAKE_CXX_COMPILER="$(pwd)/../build/bin/clang" \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=true \ + -DCOMPILER_RT_BUILD_BUILTINS=ON \ + -DCOMPILER_RT_USE_BUILTINS_LIBRARY=ON \ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF -DCOMPILER_RT_BUILD_PROFILE=OFF -DCOMPILER_RT_BUILD_MEMPROF=OFF -DCOMPILER_RT_BUILD_ORC=OFF -DCOMPILER_RT_BUILD_XRAY=OFF -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ + -DLIBCXX_USE_COMPILER_RT=YES \ + -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi;compiler-rt;libunwind' \ + ../runtimes +ninja +find include +cd compiler-rt/lib/linux +# rename CRT files to expected filenames +cp -a clang_rt.crtend-aarch64.o crtendS.o +cp -a clang_rt.crtbegin-aarch64.o crtbeginS.o diff --git a/llvm/include/llvm/Transforms/Utils/AArch64LoadStoreTagging.h b/llvm/include/llvm/Transforms/Utils/AArch64LoadStoreTagging.h new file mode 100644 index 000000000000..2911382984b2 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/AArch64LoadStoreTagging.h @@ -0,0 +1,25 @@ +//===-- AArch64LoadStoreTagging.h - Tag addrs used in loads and stores ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_AARCH64LOADSTORETAGGING_H +#define LLVM_TRANSFORMS_AARCH64LOADSTORETAGGING_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class AArch64LoadStoreTaggingPass : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + static bool isRequired() { return true; } +}; + +} // namespace llvm + +#endif // LLVM_TRANSFORMS_AARCH64LOADSTORETAGGING_H diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 3bb2ce0ae346..a60c8ed25b74 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -123,6 +123,7 @@ #include "llvm/Transforms/Scalar/SpeculativeExecution.h" #include "llvm/Transforms/Scalar/TailRecursionElimination.h" #include "llvm/Transforms/Scalar/WarnMissedTransforms.h" +#include "llvm/Transforms/Utils/AArch64LoadStoreTagging.h" #include "llvm/Transforms/Utils/AddDiscriminators.h" #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" @@ -1174,6 +1175,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true)); + MPM.addPass(createModuleToFunctionPassAdaptor(AArch64LoadStoreTaggingPass())); + if (EnableModuleInliner) MPM.addPass(buildModuleInlinerPipeline(Level, Phase)); else @@ -2120,6 +2123,8 @@ ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass())); + MPM.addPass(createModuleToFunctionPassAdaptor(AArch64LoadStoreTaggingPass())); + return MPM; } diff --git a/llvm/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp b/llvm/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp index 7763bd67e2b1..3e04cbae8acf 100644 --- a/llvm/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp +++ b/llvm/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp @@ -116,204 +116,7 @@ static bool atomicReadDroppedOnZero(unsigned Opcode) { void AArch64DeadRegisterDefinitions::processMachineBasicBlock( MachineBasicBlock &MBB) { const MachineFunction &MF = *MBB.getParent(); - for (MachineBasicBlock::iterator II = MBB.begin(), E = MBB.end(); II != E; ++II) { - MachineInstr &MI = *II; - if (MI.mayLoadOrStore()) { - bool tag = true; - unsigned operand_number = 0; - /* - * TODO: check if we need to do anything for LDRDl, LDRQl, LDRSWl, LDRSl, - * LDRWl and LDRXl. They load literals and have no GPR64sp operands but - * are marked as may load - */ - switch (MI.getOpcode()) { - case AArch64::LDRBBroW: - case AArch64::LDRBBroX: - case AArch64::LDRBBui: - case AArch64::LDRBroW: - case AArch64::LDRBroX: - case AArch64::LDRBui: - case AArch64::LDRDroW: - case AArch64::LDRDroX: - case AArch64::LDRDui: - case AArch64::LDRHHroW: - case AArch64::LDRHHroX: - case AArch64::LDRHHui: - case AArch64::LDRHroW: - case AArch64::LDRHroX: - case AArch64::LDRHui: - case AArch64::LDRQroW: - case AArch64::LDRQroX: - case AArch64::LDRQui: - case AArch64::LDRSBWroW: - case AArch64::LDRSBWroX: - case AArch64::LDRSBWui: - case AArch64::LDRSBXroW: - case AArch64::LDRSBXroX: - case AArch64::LDRSBXui: - case AArch64::LDRSHWroW: - case AArch64::LDRSHWroX: - case AArch64::LDRSHWui: - case AArch64::LDRSHXroW: - case AArch64::LDRSHXroX: - case AArch64::LDRSHXui: - case AArch64::LDRSWroW: - case AArch64::LDRSWroX: - case AArch64::LDRSWui: - case AArch64::LDRSroW: - case AArch64::LDRSroX: - case AArch64::LDRSui: - case AArch64::LDRWroW: - case AArch64::LDRWroX: - case AArch64::LDRWui: - case AArch64::LDRXroW: - case AArch64::LDRXroX: - case AArch64::LDRXui: - case AArch64::LDR_PXI: - case AArch64::LDR_TX: - case AArch64::LDR_ZXI: - case AArch64::LDRAAindexed: - case AArch64::LDRABindexed: - case AArch64::STRBBroW: - case AArch64::STRBBroX: - case AArch64::STRBBui: - case AArch64::STRBroW: - case AArch64::STRBroX: - case AArch64::STRBui: - case AArch64::STRDroW: - case AArch64::STRDroX: - case AArch64::STRDui: - case AArch64::STRHHroW: - case AArch64::STRHHroX: - case AArch64::STRHHui: - case AArch64::STRHroW: - case AArch64::STRHroX: - case AArch64::STRHui: - case AArch64::STRQroW: - case AArch64::STRQroX: - case AArch64::STRQui: - case AArch64::STRSroW: - case AArch64::STRSroX: - case AArch64::STRSui: - case AArch64::STRWroW: - case AArch64::STRWroX: - case AArch64::STRWui: - case AArch64::STRXroW: - case AArch64::STRXroX: - case AArch64::STRXui: - case AArch64::STR_PXI: - case AArch64::STR_TX: - case AArch64::STR_ZXI: - operand_number = 1; - break; - case AArch64::LDPDi: - case AArch64::LDPQi: - case AArch64::LDPSWi: - case AArch64::LDPSi: - case AArch64::LDPWi: - case AArch64::LDPXi: - case AArch64::LDRAAwriteback: - case AArch64::LDRABwriteback: - case AArch64::LDRBBpost: - case AArch64::LDRBBpre: - case AArch64::LDRBpost: - case AArch64::LDRBpre: - case AArch64::LDRDpost: - case AArch64::LDRDpre: - case AArch64::LDRHHpost: - case AArch64::LDRHHpre: - case AArch64::LDRHpost: - case AArch64::LDRHpre: - case AArch64::LDRQpost: - case AArch64::LDRQpre: - case AArch64::LDRSBWpost: - case AArch64::LDRSBWpre: - case AArch64::LDRSBXpost: - case AArch64::LDRSBXpre: - case AArch64::LDRSHWpost: - case AArch64::LDRSHWpre: - case AArch64::LDRSHXpost: - case AArch64::LDRSHXpre: - case AArch64::LDRSWpost: - case AArch64::LDRSWpre: - case AArch64::LDRSpost: - case AArch64::LDRSpre: - case AArch64::LDRWpost: - case AArch64::LDRWpre: - case AArch64::LDRXpost: - case AArch64::LDRXpre: - case AArch64::STPDi: - case AArch64::STPQi: - case AArch64::STPSi: - case AArch64::STPWi: - case AArch64::STPXi: - case AArch64::STRBBpost: - case AArch64::STRBBpre: - case AArch64::STRBpost: - case AArch64::STRBpre: - case AArch64::STRDpost: - case AArch64::STRDpre: - case AArch64::STRHHpost: - case AArch64::STRHHpre: - case AArch64::STRHpost: - case AArch64::STRHpre: - case AArch64::STRQpost: - case AArch64::STRQpre: - case AArch64::STRSpost: - case AArch64::STRSpre: - case AArch64::STRWpost: - case AArch64::STRWpre: - case AArch64::STRXpost: - case AArch64::STRXpre: - operand_number = 2; - break; - case AArch64::LDPDpost: - case AArch64::LDPDpre: - case AArch64::LDPQpost: - case AArch64::LDPQpre: - case AArch64::LDPSWpost: - case AArch64::LDPSWpre: - case AArch64::LDPSpost: - case AArch64::LDPSpre: - case AArch64::LDPWpost: - case AArch64::LDPWpre: - case AArch64::LDPXpost: - case AArch64::LDPXpre: - case AArch64::LDR_ZA: - case AArch64::STPDpost: - case AArch64::STPDpre: - case AArch64::STPQpost: - case AArch64::STPQpre: - case AArch64::STPSpost: - case AArch64::STPSpre: - case AArch64::STPWpost: - case AArch64::STPWpre: - case AArch64::STPXpost: - case AArch64::STPXpre: - case AArch64::STR_ZA: - operand_number = 3; - break; - default: - tag = false; - break; - } - assert(tag); - const MCInstrDesc &EXTRII = TII->get(AArch64::EXTRWrri); - - auto op = MI.getOperand(operand_number); - LLVM_DEBUG(dbgs() << "opcode num is " << MI.getOpcode() << " checking operand " << operand_number << "\n"); - assert(op.isReg()); - auto reg = op.getReg(); - MachineInstrBuilder MIB2 = BuildMI(MBB, MI, MI.getDebugLoc(), EXTRII); - MIB2.addReg(reg).addReg(reg).addReg(AArch64::X18).addImm(56); - - MachineInstrBuilder MIB3 = BuildMI(MBB, MI, MI.getDebugLoc(), EXTRII); - MIB3.addReg(reg).addReg(reg).addReg(reg).addImm(4); - II = MIB3; - II++; - Changed = true; - continue; - } + for (MachineInstr &MI : MBB) { if (usesFrameIndex(MI)) { // We need to skip this instruction because while it appears to have a // dead def it uses a frame index which might expand into a multi diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp index a46f959617df..62cf6a2c47ac 100644 --- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp +++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp @@ -66,7 +66,6 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Compiler.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" #include @@ -75,8 +74,6 @@ #include #include -#define DEBUG_TYPE "aarch64-fastisel" - using namespace llvm; namespace { @@ -2141,18 +2138,6 @@ bool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr, BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II).addReg(SrcReg); addLoadStoreOperands(Addr, MIB, MachineMemOperand::MOStore, ScaleFactor, MMO); - //bool sp_relative = Addr.isFIBase(); - //bool reg_offset = !Addr.getOffsetReg(); - //if (!sp_relative) { - // const MCInstrDesc &II2 = TII.get(AArch64::EXTRXrri); - // MachineInstrBuilder MIB2 = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II2); - // MIB2.addReg(Addr.getReg()).addReg(Addr.getReg()).addReg(AArch64::X18).addImm(56); - - // const MCInstrDesc &II3 = TII.get(AArch64::EXTRXrri); - // MachineInstrBuilder MIB3 = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II3); - // MIB3.addReg(Addr.getReg()).addReg(Addr.getReg()).addReg(Addr.getReg()).addImm(8); - //} - return true; } diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 7947d73f9a4d..53bbf21d78c1 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -10463,7 +10463,10 @@ getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const unsigned DwarfRegNum = MRI->getDwarfRegNum(Reg, false); if (!Subtarget->isXRegisterReserved(DwarfRegNum) && !MRI->isReservedReg(MF, Reg)) - Reg = 0; + report_fatal_error( + Twine("Register \"" + StringRef(RegName) + + "\" is not reserved! Did you forget to pass -ffixed-" + + StringRef(RegName) + "?")); } if (Reg) return Reg; diff --git a/llvm/lib/Transforms/Utils/AArch64LoadStoreTagging.cpp b/llvm/lib/Transforms/Utils/AArch64LoadStoreTagging.cpp new file mode 100644 index 000000000000..6c5f363770b1 --- /dev/null +++ b/llvm/lib/Transforms/Utils/AArch64LoadStoreTagging.cpp @@ -0,0 +1,75 @@ +//===-- AArch64LoadStoreTagging.cpp - Tag addrs used in loads and stores --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/AArch64LoadStoreTagging.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" + +using namespace llvm; + +Value *readRegister(IRBuilder<> &IRB, StringRef Name) { + auto *C = &IRB.getContext(); + Module *M = IRB.GetInsertBlock()->getParent()->getParent(); + auto &DL = M->getDataLayout(); + Type *IntptrTy = IRB.getIntPtrTy(DL); + + Function *ReadRegister = + Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy); + MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)}); + Value *Args[] = {MetadataAsValue::get(*C, MD)}; + return IRB.CreateCall(ReadRegister, Args); +} + +PreservedAnalyses +AArch64LoadStoreTaggingPass::run(Function &F, FunctionAnalysisManager &AM) { + errs() << "instrumenting AArch64 loads/stores in " << F.getName() << "\n"; + + IRBuilder<> IRB(F.getContext()); + Module *M = F.getParent(); + auto &DL = M->getDataLayout(); + Type *IntptrTy = IRB.getIntPtrTy(DL); + Type *PtrTy = IRB.getPtrTy(); + + auto *TopEightBitsSet = ConstantInt::get(IntptrTy, 0xff00'0000'0000'0000); + + for (Function::iterator BB = F.begin(), BBE = F.end(); BB != BBE; ++BB) { + BasicBlock &B = *BB; + for (BasicBlock::iterator I = B.begin(), IE = B.end(); I != IE; ++I) { + StoreInst *SI = dyn_cast(&*I); + LoadInst *LI = dyn_cast(&*I); + if (LI || SI) { + Value *Pointer = SI ? SI->getPointerOperand() : LI->getPointerOperand(); + if (AllocaInst *AI = dyn_cast(Pointer)) { + continue; + } + + IRBuilder<> IRB(&*I); + Value *ReadX18 = readRegister(IRB, "x18"); + Value *PtrToInt = IRB.CreatePtrToInt(Pointer, IntptrTy, "makeint"); + + Value *And = IRB.CreateAnd(ReadX18, TopEightBitsSet, "andhighbitmask"); + Value *Or = IRB.CreateOr(PtrToInt, And, "ortag"); + + Value *IntToPtrInst = IRB.CreateIntToPtr(Or, PtrTy, "makeptr"); + if (SI) { + SI->setOperand(SI->getPointerOperandIndex(), IntToPtrInst); + } else { + assert(LI); + LI->setOperand(LI->getPointerOperandIndex(), IntToPtrInst); + } + } + } + } + // return true; + + return PreservedAnalyses::none(); +} diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt index 51e8821773c3..5989c0e5a4b4 100644 --- a/llvm/lib/Transforms/Utils/CMakeLists.txt +++ b/llvm/lib/Transforms/Utils/CMakeLists.txt @@ -1,4 +1,5 @@ add_llvm_component_library(LLVMTransformUtils + AArch64LoadStoreTagging.cpp AddDiscriminators.cpp AMDGPUEmitPrintf.cpp ASanStackFrameLayout.cpp