From 23f7268c6fdcc90771168a442793dafa8bde183d Mon Sep 17 00:00:00 2001 From: quic-areg Date: Mon, 5 Oct 2026 14:40:31 -0700 Subject: [PATCH] [Relocator] Restore parallelism in scanRelocations Restores parallel scanRelocations phase in a two-phase approach similar to lld: Phase 1 (scanRelocationParallel) may run concurrently over input files. It only performs the checks that are common to every backend and records which relocations need further processing. It creates no GOT/PLT slots and no dynamic relocations. Phase 2 (replayScanRelocations) is serial and visits the recorded relocations in input order. Slot allocation happens here and retains determinism. Relocation callbacks run before phase 1. Callbacks for different relocations still run in parallel, but all callbacks finish before backend scanning begins so callback mutations cannot race with scanner reads. This was possible in the old parallel scanRelocations implementation. Benchmarks: 16 threads, hyperfine 10 runs 2 warmups. Show below are scan time, wall clock time, for this PR, before this PR, and the old parallel scanRelocations implementation: Workload This commit origin/main (serial) Old parallel ld.eld 22.115 ms, 1.607 s 32.205 ms, 1.573 s 33.317 ms, 1.656 s Linux kernel 7315.164 ms, 16.850 s 7404.781 ms, 16.934 s 7790.618 ms, 17.229 s Future work: Move more relocation classification into the parallel phase. The serial phase should process only relocations that update shared linker state, such as GOT/PLT and dynamic-relocation handling, instead of walking every relocation. Fixes #1938 Signed-off-by: quic-areg --- include/eld/Config/LinkerConfig.h | 9 +- include/eld/Core/Module.h | 3 +- include/eld/Object/ObjectLinker.h | 9 +- include/eld/Script/Plugin.h | 2 + include/eld/Target/Relocator.h | 64 ++++++- lib/Object/ObjectLinker.cpp | 157 +++++++++++++---- lib/Script/Plugin.cpp | 9 +- lib/Target/AArch64/AArch64Relocator.cpp | 101 ++++------- lib/Target/AArch64/AArch64Relocator.h | 30 ++-- lib/Target/ARM/ARMRelocator.cpp | 69 ++------ lib/Target/ARM/ARMRelocator.h | 23 +-- lib/Target/Hexagon/HexagonRelocator.cpp | 68 ++------ lib/Target/Hexagon/HexagonRelocator.h | 23 ++- lib/Target/RISCV/RISCVRelocator.cpp | 161 +++++++----------- lib/Target/RISCV/RISCVRelocator.h | 17 +- lib/Target/Relocator.cpp | 113 ++++++++++++ lib/Target/Template/TemplateRelocator.cpp | 54 +----- lib/Target/Template/TemplateRelocator.h | 9 +- lib/Target/X86/X86_32Relocator.cpp | 4 +- lib/Target/X86/X86_32Relocator.h | 7 +- lib/Target/X86/x86_64Relocator.cpp | 70 ++------ lib/Target/X86/x86_64Relocator.h | 23 ++- .../GOTPLTDeterminism/GOTPLTDeterminism.test | 16 +- test/X86/i386/standalone/Backend/ELF32.test | 3 + .../X86/x86_64/linux/relocCOPY/relocCOPY.test | 9 +- .../ParallelScanCopyGOT.test | 32 ++++ .../ParallelScanRelocations.test | 25 +++ 27 files changed, 622 insertions(+), 488 deletions(-) create mode 100644 test/X86/x86_64/standalone/ParallelScanCopyGOT/ParallelScanCopyGOT.test create mode 100644 test/X86/x86_64/standalone/ParallelScanRelocations/ParallelScanRelocations.test diff --git a/include/eld/Config/LinkerConfig.h b/include/eld/Config/LinkerConfig.h index 3cf7820825..0ad72a8ecc 100644 --- a/include/eld/Config/LinkerConfig.h +++ b/include/eld/Config/LinkerConfig.h @@ -67,13 +67,14 @@ class LinkerConfig { enum EnableThreadsOpt { NoThreads = 0, AssignOutputSections = 0x1, + ScanRelocations = 0x2, SyncRelocations = 0x4, CheckCrossRefs = 0x8, CreateOutputSections = 0x10, ApplyRelocations = 0x20, LinkerRelaxation = 0x40, AssignVersionScriptNodes = 0x80, - AllThreads = 0x1 | 0x4 | 0x8 | 0x10 | 0x20 | 0x40 | 0x80, + AllThreads = 0x1 | 0x2 | 0x4 | 0x8 | 0x10 | 0x20 | 0x40 | 0x80, }; enum SymDefStyle { Default, Provide, UnknownSymDefStyle }; @@ -151,6 +152,10 @@ class LinkerConfig { return EnableThreads & LinkerConfig::AssignOutputSections; } + bool isScanRelocationsMultiThreaded() const { + return EnableThreads & LinkerConfig::ScanRelocations; + } + bool isSyncRelocationsMultiThreaded() const { return EnableThreads & LinkerConfig::SyncRelocations; } @@ -179,6 +184,8 @@ class LinkerConfig { EnableThreads = NoThreads; if (EnableThreadsOpt & AssignOutputSections) EnableThreads |= AssignOutputSections; + if (EnableThreadsOpt & ScanRelocations) + EnableThreads |= ScanRelocations; if (EnableThreadsOpt & SyncRelocations) EnableThreads |= SyncRelocations; if (EnableThreadsOpt & CheckCrossRefs) diff --git a/include/eld/Core/Module.h b/include/eld/Core/Module.h index 052c9f6efc..ce4a560f08 100644 --- a/include/eld/Core/Module.h +++ b/include/eld/Core/Module.h @@ -39,6 +39,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" #include +#include #include #include #include @@ -702,7 +703,7 @@ class Module { LDSymbol *DotSymbol = nullptr; Linker *L = nullptr; LayoutInfo *ThisLayoutInfo = nullptr; - bool Failure = false; + std::atomic Failure = false; bool UsesLto = false; LinkState State = LinkState::Unknown; ReplaceFragsVectorT ReplaceFrags; diff --git a/include/eld/Object/ObjectLinker.h b/include/eld/Object/ObjectLinker.h index b382a87217..88bb1c18e1 100644 --- a/include/eld/Object/ObjectLinker.h +++ b/include/eld/Object/ObjectLinker.h @@ -185,9 +185,10 @@ class ObjectLinker { /// scanRelocations - scan all relocation entries by output symbols. bool scanRelocations(bool IsPartialLink = false); - void scanRelocationsHelper(InputFile *Input, bool IsPartialLink, - LinkerScript::PluginVectorT PluginVect, - Relocator::CopyRelocs &); + void scanRelocationsHelper( + InputFile *Input, size_t InputIndex, bool IsPartialLink, bool IsParallel, + Relocator::CopyRelocs &CopyRelocSet, + const llvm::DenseMap &SkipRelocProcessing); bool finalizeScanRelocations(); @@ -336,7 +337,7 @@ class ObjectLinker { void applySubAlign(); // Get Plugin list for Relocation registration callback. - LinkerScript::PluginVectorT getLinkerPluginWithLinkerConfigs(); + LinkerScript::PluginVectorT getLinkerPluginsWithRelocCallbacks(); // ------------------ All sections ------------------------- const std::vector
&getAllInputSections() const { diff --git a/include/eld/Script/Plugin.h b/include/eld/Script/Plugin.h index e5b8a56ee0..8b393c7b02 100644 --- a/include/eld/Script/Plugin.h +++ b/include/eld/Script/Plugin.h @@ -141,6 +141,8 @@ class Plugin { bool isRelocTypeRegistered(uint32_t RelocType, Relocation *R); + bool hasRegisteredRelocations() const; + plugin::LinkerPluginConfig *getLinkerPluginConfig() const; // -----------------Check if timing is enabled --------------- diff --git a/include/eld/Target/Relocator.h b/include/eld/Target/Relocator.h index dbccd1d1b2..4f598ecacb 100644 --- a/include/eld/Target/Relocator.h +++ b/include/eld/Target/Relocator.h @@ -15,11 +15,13 @@ #include "eld/Core/Module.h" #include "eld/Readers/Relocation.h" +#include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include -#include namespace eld { @@ -69,7 +71,10 @@ class Relocator { ReservePLT = 4, }; - typedef std::unordered_set CopyRelocs; + /// Symbols needing a copy relocation, in the order phase 2 discovered them. + /// Insertion order is preserved so that the copy relocations, and the + /// .dynbss sections created for them, are deterministic. + using CopyRelocs = llvm::SetVector; public: Relocator(LinkerConfig &pConfig, Module &pModule) @@ -80,9 +85,32 @@ class Relocator { /// apply - general apply function virtual Result applyRelocation(Relocation &pRelocation) = 0; - virtual void scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder, - ELFSection &pSection, InputFile &pInput, - CopyRelocs &CopyRelocs) = 0; + virtual bool shouldScanRelocations() const { return true; } + + /// Parallel relocation scanning runs in two phases: + /// + /// Phase 1 (scanRelocationParallel) may run concurrently over input files. + /// It only performs the checks that are common to every backend and records + /// which relocations need further processing. It creates no GOT/PLT slots + /// and no dynamic relocations. + /// + /// Phase 2 (replayScanRelocations) is serial and visits the recorded + /// relocations in input order. Slot allocation happens here. + /// + /// Prepare for phase 1. Must be called before a parallel scan. + void initScanRelocations(); + + /// Serial single pass scan. + void scanRelocation(Relocation &Reloc, ELFSection &Section, InputFile &Input, + CopyRelocs &CopyRelocSet); + + void scanRelocationParallel(Relocation &Reloc, ELFSection &Section, + InputFile &Input, size_t InputIndex, + size_t SectIndex, size_t RelocIndex); + + /// Phase 2: Replays the relocations recorded by phase 1 in input + /// order, serially. + void replayScanRelocations(CopyRelocs &CopyRelocSet); // Issue an undefined reference error if the symbol is a magic section symbol. void issueUndefRefForMagicSymbol(const Relocation &pReloc); @@ -174,6 +202,23 @@ class Relocator { protected: bool reportNonPICRelocation(const Relocation &reloc) const; + virtual bool isRelocSupported(const Relocation &Reloc) const { return true; } + + virtual void diagnoseUnsupportedReloc(const Relocation &Reloc, + const ELFSection &Section, + const InputFile &Input) const; + + virtual bool canIssueUndefRef(ResolveInfo *Sym); + + /// Phase 1: Called for a relocation whose target section is not allocatable, + /// just before the relocation is dropped. + virtual void scanNonAllocReloc(Relocation &Reloc, ELFSection &Section) {} + + /// Phase 2: Handle one relocation recorded by phase 1, serially. + virtual void scanDeferredRelocation(InputFile &Input, Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) = 0; + virtual bool isPICRelocTypeSupported(const Relocation &reloc) const { return true; } @@ -185,6 +230,15 @@ class Relocator { Module &m_Module; std::mutex m_RelocMutex; std::unordered_map RelocNameMap; + +private: + bool scanRelocationPrologue(Relocation &Reloc, ELFSection &Section, + InputFile &Input); + + struct DeferredInput { + llvm::SmallVector RelocationsBySection; + }; + llvm::SmallVector DeferredRelocations; }; Relocator::Result checkSignedRange(Relocation &Rel, Relocator &R, int64_t Value, diff --git a/lib/Object/ObjectLinker.cpp b/lib/Object/ObjectLinker.cpp index 96ab91d80b..58175f5d11 100644 --- a/lib/Object/ObjectLinker.cpp +++ b/lib/Object/ObjectLinker.cpp @@ -2289,67 +2289,166 @@ void ObjectLinker::createCopyRelocation(ResolveInfo &Sym, addCopyReloc(getTargetBackend(), *CopySym.resolveInfo(), Type); } -void ObjectLinker::scanRelocationsHelper(InputFile *Input, bool IsPartialLink, - LinkerScript::PluginVectorT PVect, - Relocator::CopyRelocs &CopyRelocs) { +void ObjectLinker::scanRelocationsHelper( + InputFile *Input, size_t InputIndex, bool IsPartialLink, bool IsParallel, + Relocator::CopyRelocs &CopyRelocSet, + const llvm::DenseMap &SkipRelocProcessing) { ELFObjectFile *ObjFile = llvm::dyn_cast(Input); if (!ObjFile) return; - uint32_t NumPlugins = PVect.size(); + bool ShouldScanRelocations = + getTargetBackend().getRelocator()->shouldScanRelocations(); + size_t SectIndex = 0; for (auto &Rs : ObjFile->getRelocationSections()) { + const size_t ThisSectIndex = SectIndex++; if (Rs->isIgnore()) continue; if (Rs->isDiscard()) continue; + size_t RelocIndex = 0; for (auto &Relocation : Rs->getLink()->getRelocations()) { + const size_t ThisRelocIndex = RelocIndex++; // Skip unneeded relocations - if (getTargetBackend().maySkipRelocProcessing(Relocation)) + auto SkipIt = SkipRelocProcessing.find(Relocation); + bool ShouldSkip = + SkipIt != SkipRelocProcessing.end() + ? SkipIt->second + : getTargetBackend().maySkipRelocProcessing(Relocation); + if (ShouldSkip) continue; - if (NumPlugins) { - for (auto &P : PVect) - P->callReloc(Relocation->type(), Relocation); - } ELFSection *RelocSection = Rs; // scan relocation - if (!IsPartialLink) - getTargetBackend().getRelocator()->scanRelocation( - *Relocation, *ThisModule->getIRBuilder(), *RelocSection, *Input, - CopyRelocs); - else + if (IsPartialLink) { getTargetBackend().getRelocator()->partialScanRelocation(*Relocation, *RelocSection); + } else if (!ShouldScanRelocations) { + continue; + } else if (!IsParallel) { + getTargetBackend().getRelocator()->scanRelocation( + *Relocation, *RelocSection, *Input, CopyRelocSet); + } else { + getTargetBackend().getRelocator()->scanRelocationParallel( + *Relocation, *RelocSection, *Input, InputIndex, ThisSectIndex, + ThisRelocIndex); + } } // for all relocations } // for all relocation section } -LinkerScript::PluginVectorT ObjectLinker::getLinkerPluginWithLinkerConfigs() { +LinkerScript::PluginVectorT ObjectLinker::getLinkerPluginsWithRelocCallbacks() { const LinkerScript::PluginVectorT PluginVect = ThisModule->getScript().getPlugins(); - LinkerScript::PluginVectorT PluginVectHavingLinkerConfigs; + LinkerScript::PluginVectorT PluginVectHavingRelocCallbacks; for (auto &P : PluginVect) { - if (P->getLinkerPluginConfig()) - PluginVectHavingLinkerConfigs.push_back(P); + if (P->hasRegisteredRelocations()) + PluginVectHavingRelocCallbacks.push_back(P); } - return PluginVectHavingLinkerConfigs; + return PluginVectHavingRelocCallbacks; } bool ObjectLinker::scanRelocations(bool IsPartialLink) { - LinkerScript::PluginVectorT PluginVect = getLinkerPluginWithLinkerConfigs(); + LinkerScript::PluginVectorT PluginVect = getLinkerPluginsWithRelocCallbacks(); getTargetBackend().provideSymbols(); + Relocator *Reloc = getTargetBackend().getRelocator(); + + auto &Inputs = ThisModule->getObjectList(); + const bool RunParallel = Reloc->shouldScanRelocations() && !IsPartialLink && + ThisConfig.options().numThreads() > 1 && + ThisConfig.isScanRelocationsMultiThreaded(); + + // Snapshot the relocations that will receive callbacks. Relocations added by + // callbacks are scanned later but do not receive a callback in this pass. + llvm::SmallVector, 0> CallbackWork; + llvm::SmallVector, 0> + SkipRelocProcessingByInput(Inputs.size()); + if (!PluginVect.empty()) { + CallbackWork.resize(Inputs.size()); + for (size_t I = 0; I != Inputs.size(); ++I) { + auto *ObjFile = llvm::dyn_cast(Inputs[I]); + if (!ObjFile) + continue; + for (ELFSection *Rs : ObjFile->getRelocationSections()) { + if (Rs->isIgnore() || Rs->isDiscard()) + continue; + for (Relocation *R : Rs->getLink()->getRelocations()) + CallbackWork[I].push_back(R); + } + } + } + + auto RunForEachInput = [&](auto Fn) { + if (!RunParallel) { + for (size_t I = 0; I != Inputs.size(); ++I) + Fn(I); + return; + } + std::atomic NextInput{0}; + size_t NumWorkers = + std::min(ThisConfig.options().numThreads(), Inputs.size()); + llvm::parallelFor(0, NumWorkers, [&](size_t) { + for (size_t I; (I = NextInput.fetch_add(1, std::memory_order_relaxed)) < + Inputs.size();) + Fn(I); + }); + }; - // Slots are allocated as relocations are scanned in input order. - std::vector> AllCopyRelocs; - for (auto &Input : ThisModule->getObjectList()) { - auto CopyRelocs = std::make_unique(); - scanRelocationsHelper(Input, IsPartialLink, PluginVect, *CopyRelocs); - AllCopyRelocs.push_back(std::move(CopyRelocs)); + auto RecordSkipDecisionsForInput = [&](size_t I) { + // Preserve the skip result from before callbacks can change symbol or + // section state. + for (Relocation *R : CallbackWork[I]) { + SkipRelocProcessingByInput[I][R] = + getTargetBackend().maySkipRelocProcessing(R); + } + }; + + auto RunCallbacksForInput = [&](size_t I) { + for (Relocation *R : CallbackWork[I]) { + if (SkipRelocProcessingByInput[I].lookup(R)) + continue; + for (Plugin *P : PluginVect) + P->callReloc(R->type(), R); + } + }; + + // Callbacks may change state read by backend scanning, so finish all of them + // before scanning begins. + if (!PluginVect.empty()) { + RunForEachInput(RecordSkipDecisionsForInput); + RunForEachInput(RunCallbacksForInput); + } + + Relocator::CopyRelocs CopyRelocSet; + if (!RunParallel) { + if (ThisModule->getPrinter()->traceThreads()) + ThisConfig.raise(Diag::threads_disabled) << "ScanRelocations"; + for (size_t I = 0; I != Inputs.size(); ++I) + scanRelocationsHelper(Inputs[I], I, IsPartialLink, false, CopyRelocSet, + SkipRelocProcessingByInput[I]); + } else { + if (ThisModule->getPrinter()->traceThreads()) + ThisConfig.raise(Diag::threads_enabled) + << "ScanRelocations" << ThisConfig.options().numThreads(); + Reloc->initScanRelocations(); + std::atomic NextInput{0}; + size_t NumWorkers = + std::min(ThisConfig.options().numThreads(), Inputs.size()); + llvm::parallelFor(0, NumWorkers, [&](size_t) { + for (size_t I; (I = NextInput.fetch_add(1, std::memory_order_relaxed)) < + Inputs.size();) + scanRelocationsHelper(Inputs[I], I, IsPartialLink, true, CopyRelocSet, + SkipRelocProcessingByInput[I]); + }); + } + + if (RunParallel) { + Reloc->replayScanRelocations(CopyRelocSet); } + // assume there is only one copy relocation type per target Relocation::Type CopyRelocType = getTargetBackend().getCopyRelType(); - for (const auto &RelocVec : AllCopyRelocs) - for (auto &Reloc : *RelocVec) - createCopyRelocation(*Reloc, CopyRelocType); + for (ResolveInfo *Sym : CopyRelocSet) + createCopyRelocation(*Sym, CopyRelocType); // If there is a undefined symbol, fail the link. No point fixing the // relocations. This is overridden by --noinhibit-exec. diff --git a/lib/Script/Plugin.cpp b/lib/Script/Plugin.cpp index f463745c6f..99fdf1c66b 100644 --- a/lib/Script/Plugin.cpp +++ b/lib/Script/Plugin.cpp @@ -409,6 +409,11 @@ bool Plugin::isRelocTypeRegistered(uint32_t RelocType, Relocation *R) { return false; } +bool Plugin::hasRegisteredRelocations() const { + return RelocBitVector && + (RelocBitVector->any() || SlowPathRelocBitVector->any()); +} + plugin::LinkerPluginConfig *Plugin::getLinkerPluginConfig() const { return LinkerPluginConfigHandle; } @@ -634,6 +639,4 @@ void Plugin::callActBeforeWritingOutputHook() { P->ActBeforeWritingOutput(); } -void Plugin::clearResources() { - PluginCommandLineOptions.clear(); -} \ No newline at end of file +void Plugin::clearResources() { PluginCommandLineOptions.clear(); } diff --git a/lib/Target/AArch64/AArch64Relocator.cpp b/lib/Target/AArch64/AArch64Relocator.cpp index 1bb0badab7..c420bb28b7 100644 --- a/lib/Target/AArch64/AArch64Relocator.cpp +++ b/lib/Target/AArch64/AArch64Relocator.cpp @@ -149,18 +149,18 @@ AArch64Relocator::AArch64Relocator(AArch64LDBackend &pParent, LinkerConfig &pConfig, Module &pModule) : Relocator(pConfig, pModule), m_Target(pParent) {} -bool AArch64Relocator::isRelocSupported(Relocation &pReloc) const { - Relocation::Type type = pReloc.type(); +bool AArch64Relocator::isRelocSupported(const Relocation &Reloc) const { + Relocation::Type Type = Reloc.type(); // valid types are 0x0, 0x100-0x244 - if ((type < 0x100 || type > 0x244) && (type != 0x0) && - (type != R_AARCH64_COPY_INSN)) + if ((Type < 0x100 || Type > 0x244) && (Type != 0x0) && + (Type != R_AARCH64_COPY_INSN)) return false; - auto iter = ApplyFunctions.find(type); + auto Iter = ApplyFunctions.find(Type); - assert(iter != ApplyFunctions.end()); + assert(Iter != ApplyFunctions.end()); - return iter->second.func != unsupport; + return Iter->second.func != unsupport; } bool AArch64Relocator::isPICRelocTypeSupported(const Relocation &reloc) const { @@ -600,71 +600,34 @@ void AArch64Relocator::partialScanRelocation(Relocation &pReloc, } } -void AArch64Relocator::scanRelocation(Relocation &pReloc, - eld::IRBuilder &pBuilder, - ELFSection &pSection, - InputFile &pInputFile, - CopyRelocs &CopyRelocs) { - if (LinkerConfig::Object == config().codeGenType()) - return; - - if (!isRelocSupported(pReloc)) { - config().raise(Diag::unsupported_reloc) - << pReloc.type() << pSection.getDecoratedName(config().options()) - << pInputFile.getInput()->decoratedPath(); - m_Target.getModule().setFailure(true); - return; - } - - if (!checkPICRelocSupported(pReloc)) - return; - - // rsym - The relocation target symbol - ResolveInfo *rsym = pReloc.symInfo(); - assert(nullptr != rsym && - "ResolveInfo of relocation not set while scanRelocation"); - - // Check if we are tracing relocations. - if (m_Module.getPrinter()->traceReloc()) { - std::lock_guard relocGuard(m_RelocMutex); - std::string relocName = getName(pReloc.type()); - if (config().options().traceReloc(relocName)) - config().raise(Diag::reloc_trace) - << relocName << pReloc.symInfo()->name() - << pInputFile.getInput()->decoratedPath(); - } - - // check if we should issue undefined reference for the relocation target - // symbol - { - if (rsym->isUndef() || rsym->isBitCode()) { - std::lock_guard relocGuard(m_RelocMutex); - if (m_Target.canIssueUndef(rsym)) { - if (rsym->visibility() != ResolveInfo::Default) - issueInvisibleRef(pReloc, pInputFile); - issueUndefRef(pReloc, pInputFile, &pSection); - } - } - } - - ELFSection *section = pSection.getLink(); +void AArch64Relocator::diagnoseUnsupportedReloc(const Relocation &Reloc, + const ELFSection &Section, + const InputFile &Input) const { + Relocator::diagnoseUnsupportedReloc(Reloc, Section, Input); + m_Target.getModule().setFailure(true); +} - if (!section->isAlloc()) { - // Cannot have authenticated relocations in non-alloc sections (like .debug) - if (isAuthReloc(pReloc)) { - std::lock_guard relocGuard(m_RelocMutex); - config().raise(Diag::reloc_nonalloc_section) - << getName(pReloc.type()) << pReloc.symInfo()->name() - << pSection.getDecoratedName(config().options()); - m_Target.getModule().setFailure(true); - } +void AArch64Relocator::scanNonAllocReloc(Relocation &Reloc, + ELFSection &Section) { + // Cannot have authenticated relocations in non-alloc sections (like .debug) + if (!isAuthReloc(Reloc)) return; - } + std::lock_guard relocGuard(m_RelocMutex); + config().raise(Diag::reloc_nonalloc_section) + << getName(Reloc.type()) << Reloc.symInfo()->name() + << Section.getDecoratedName(config().options()); + m_Target.getModule().setFailure(true); +} - if (rsym->isLocal()) // rsym is local - scanLocalReloc(pInputFile, pReloc, *section); - else // rsym is external - scanGlobalReloc(pInputFile, pReloc, pBuilder, *section, CopyRelocs); +void AArch64Relocator::scanDeferredRelocation(InputFile &Input, + Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) { + if (Reloc.symInfo()->isLocal()) + scanLocalReloc(Input, Reloc, Section); + else + scanGlobalReloc(Input, Reloc, *module().getIRBuilder(), Section, + CopyRelocSet); } //===----------------------------------------------------------------------===// diff --git a/lib/Target/AArch64/AArch64Relocator.h b/lib/Target/AArch64/AArch64Relocator.h index 4eaac2de34..c7261a428e 100644 --- a/lib/Target/AArch64/AArch64Relocator.h +++ b/lib/Target/AArch64/AArch64Relocator.h @@ -62,23 +62,31 @@ class AArch64Relocator : public Relocator { bool isTLSDescStatic(Relocation &pReloc) const; - /// scanRelocation - determine the empty entries are needed or not and create - /// the empty entries if needed. - /// For AArch64, following entries are check to create: - /// - GOT entry (for .got section) - /// - PLT entry (for .plt section) - /// - dynamic relocation entries (for .rel.plt and .rel.dyn sections) - void scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder, - ELFSection &pSection, InputFile &pInput, - CopyRelocs &) override; - // Handle partial linking void partialScanRelocation(Relocation &pReloc, const ELFSection &pSection) override; +protected: + bool isRelocSupported(const Relocation &Reloc) const override; + + void diagnoseUnsupportedReloc(const Relocation &Reloc, + const ELFSection &Section, + const InputFile &Input) const override; + + /// An authenticated relocation is not valid in a non-alloc section. + void scanNonAllocReloc(Relocation &Reloc, ELFSection &Section) override; + + /// Determine whether GOT, PLT, or dynamic relocation entries are needed and + /// create them. For AArch64: + /// - GOT entry (for .got section) + /// - PLT entry (for .plt section) + /// - dynamic relocation entries (for .rel.plt and .rel.dyn sections) + void scanDeferredRelocation(InputFile &Input, Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) override; + private: bool isPICRelocTypeSupported(const Relocation &reloc) const override; - bool isRelocSupported(Relocation &pReloc) const; bool relocNeedsDynRel(Relocation &pReloc) const; void scanLocalReloc(InputFile &pInput, Relocation &pReloc, diff --git a/lib/Target/ARM/ARMRelocator.cpp b/lib/Target/ARM/ARMRelocator.cpp index d88e85fb0f..74a7504da0 100644 --- a/lib/Target/ARM/ARMRelocator.cpp +++ b/lib/Target/ARM/ARMRelocator.cpp @@ -896,53 +896,15 @@ void ARMRelocator::handleScanForNonPreemptibleIFunc(Relocation &R, RI->setReserved(RI->reserved() | ReservePLT); } -void ARMRelocator::scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder, - ELFSection &pSection, InputFile &pInputFile, - CopyRelocs &CopyRelocs) { - if (LinkerConfig::Object == config().codeGenType()) - return; - - if (!checkPICRelocSupported(pReloc)) - return; +void ARMRelocator::scanDeferredRelocation(InputFile &Input, Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) { + ResolveInfo *Sym = Reloc.symInfo(); - // rsym - The relocation target symbol - ResolveInfo *rsym = pReloc.symInfo(); - assert(nullptr != rsym && - "ResolveInfo of relocation not set while scanRelocation"); + if (Reloc.type() == llvm::ELF::R_ARM_TARGET1) + Reloc.setType(llvm::ELF::R_ARM_ABS32); - // Check if we are tracing relocations. - if (m_Module.getPrinter()->traceReloc()) { - std::lock_guard relocGuard(m_RelocMutex); - std::string relocName = getName(pReloc.type()); - if (config().options().traceReloc(relocName)) - config().raise(Diag::reloc_trace) - << relocName << pReloc.symInfo()->name() - << pInputFile.getInput()->decoratedPath(); - } - - // check if we should issue undefined reference for the relocation target - // symbol - { - if (rsym->isUndef() || rsym->isBitCode()) { - std::lock_guard relocGuard(m_RelocMutex); - if (m_Target.canIssueUndef(rsym)) { - if (rsym->visibility() != ResolveInfo::Default) - issueInvisibleRef(pReloc, pInputFile); - issueUndefRef(pReloc, pInputFile, &pSection); - } - } - } - - ELFSection *section = pSection.getLink(); - - if (!section->isAlloc()) - return; - - if (pReloc.type() == llvm::ELF::R_ARM_TARGET1) { - pReloc.setType(llvm::ELF::R_ARM_ABS32); - } - - Relocation::Type Type = pReloc.type(); + Relocation::Type Type = Reloc.type(); // Set R_ARM_TARGET2 to R_ARM_ABS32/R_ARM_REL32/R_ARM_GOT_PREL. if (Type == llvm::ELF::R_ARM_TARGET2) { @@ -959,18 +921,19 @@ void ARMRelocator::scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder, } } - ELFObjectFile *Obj = llvm::dyn_cast(&pInputFile); + ELFObjectFile *Obj = llvm::dyn_cast(&Input); // A reference to a non-preemptible IFunc in a static link is resolved // through an IRELATIVE PLT entry regardless of whether the symbol is local // or global. - if (rsym->isIFunc() && config().isCodeStatic()) - return handleScanForNonPreemptibleIFunc(pReloc, Obj); - - if (rsym->isLocal()) // rsym is local - scanLocalReloc(pInputFile, Type, pReloc, *section); - else // rsym is external - scanGlobalReloc(pInputFile, Type, pReloc, pBuilder, *section, CopyRelocs); + if (Sym->isIFunc() && config().isCodeStatic()) + return handleScanForNonPreemptibleIFunc(Reloc, Obj); + + if (Sym->isLocal()) + scanLocalReloc(Input, Type, Reloc, Section); + else + scanGlobalReloc(Input, Type, Reloc, *module().getIRBuilder(), Section, + CopyRelocSet); } //=========================================// diff --git a/lib/Target/ARM/ARMRelocator.h b/lib/Target/ARM/ARMRelocator.h index d232fd2592..5999edc1a5 100644 --- a/lib/Target/ARM/ARMRelocator.h +++ b/lib/Target/ARM/ARMRelocator.h @@ -49,16 +49,6 @@ class ARMRelocator : public Relocator { R->setTargetData(0); } - /// scanRelocation - determine the empty entries are needed or not and create - /// the empty entries if needed. - /// For ARM, following entries are check to create: - /// - GOT entry (for .got section) - /// - PLT entry (for .plt section) - /// - dynamin relocation entries (for .rel.plt and .rel.dyn sections) - void scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder, - ELFSection &pSection, InputFile &pInputFile, - CopyRelocs &) override; - ELFSegment *getSBRELSegment() const { return m_Target.getSBRELSegment(); } void setSBRELSegment(ELFSegment *S) { m_Target.setSBRELSegment(S); } @@ -71,6 +61,19 @@ class ARMRelocator : public Relocator { return StaticTLSBlockVarOffset; } +protected: + /// Determine whether GOT, PLT, or dynamic relocation entries are needed and + /// create them. For ARM: + /// - GOT entry (for .got section) + /// - PLT entry (for .plt section) + /// - dynamic relocation entries (for .rel.plt and .rel.dyn sections) + /// + /// R_ARM_TARGET1 and R_ARM_TARGET2 are normalized to a concrete type here, + /// before the local/global dispatch. + void scanDeferredRelocation(InputFile &Input, Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) override; + private: bool isPICRelocTypeSupported(const Relocation &reloc) const override; void scanLocalReloc(InputFile &pInput, Relocation::Type, Relocation &pReloc, diff --git a/lib/Target/Hexagon/HexagonRelocator.cpp b/lib/Target/Hexagon/HexagonRelocator.cpp index 162bab07a9..9b510711a5 100644 --- a/lib/Target/Hexagon/HexagonRelocator.cpp +++ b/lib/Target/Hexagon/HexagonRelocator.cpp @@ -268,65 +268,19 @@ bool HexagonRelocator::isPICRelocTypeSupported(const Relocation &reloc) const { } } -bool HexagonRelocator::isRelocSupported(Relocation &pReloc) const { - return pReloc.type() < HEXAGON_MAXRELOCS; +bool HexagonRelocator::isRelocSupported(const Relocation &Reloc) const { + return Reloc.type() < HEXAGON_MAXRELOCS; } -void HexagonRelocator::scanRelocation(Relocation &pReloc, - eld::IRBuilder &pLinker, - ELFSection &pSection, - InputFile &pInputFile, - CopyRelocs &CopyRelocs) { - if (LinkerConfig::Object == config().codeGenType()) - return; - - if (!isRelocSupported(pReloc)) { - config().raise(Diag::unsupported_reloc) - << pReloc.type() << pSection.getDecoratedName(config().options()) - << pInputFile.getInput()->decoratedPath(); - return; - } - - // If we are generating a shared library check for invalid relocations - if (!checkPICRelocSupported(pReloc)) - return; - - // rsym - The relocation target symbol - ResolveInfo *rsym = pReloc.symInfo(); - assert(nullptr != rsym && - "ResolveInfo of relocation not set while scanRelocation"); - - // Check if we are tracing relocations. - if (m_Module.getPrinter()->traceReloc()) { - std::lock_guard relocGuard(m_RelocMutex); - std::string relocName = getName(pReloc.type()); - if (config().options().traceReloc(relocName)) - config().raise(Diag::reloc_trace) - << relocName << pReloc.symInfo()->name() - << pInputFile.getInput()->decoratedPath(); - } - - // check if we should issue undefined reference for the relocation target - // symbol - { - if (rsym->isUndef() || rsym->isBitCode()) { - std::lock_guard relocGuard(m_RelocMutex); - if (m_Target.canIssueUndef(rsym)) { - if (rsym->visibility() != ResolveInfo::Default) - issueInvisibleRef(pReloc, pInputFile); - issueUndefRef(pReloc, pInputFile, &pSection); - } - } - } - ELFSection *section = pSection.getLink(); - - if (!section->isAlloc()) - return; - - if (rsym->isLocal()) // rsym is local - scanLocalReloc(pInputFile, pReloc, pLinker, *section); - else // rsym is external - scanGlobalReloc(pInputFile, pReloc, pLinker, *section, CopyRelocs); +void HexagonRelocator::scanDeferredRelocation(InputFile &Input, + Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) { + eld::IRBuilder &Linker = *module().getIRBuilder(); + if (Reloc.symInfo()->isLocal()) + scanLocalReloc(Input, Reloc, Linker, Section); + else + scanGlobalReloc(Input, Reloc, Linker, Section, CopyRelocSet); } void HexagonRelocator::scanLocalReloc(InputFile &InputFile, Relocation &pReloc, diff --git a/lib/Target/Hexagon/HexagonRelocator.h b/lib/Target/Hexagon/HexagonRelocator.h index a6760db09a..0fd3f66830 100644 --- a/lib/Target/Hexagon/HexagonRelocator.h +++ b/lib/Target/Hexagon/HexagonRelocator.h @@ -32,16 +32,6 @@ class HexagonRelocator : public Relocator { Result applyRelocation(Relocation &pRelocation) override; - /// scanRelocation - determine the empty entries are needed or not and create - /// the empty entries if needed. - /// For Hexagon, following entries are check to create: - /// - GOT entry (for .got and .got.plt sections) - /// - PLT entry (for .plt section) - /// - dynamin relocation entries (for .rel.plt and .rel.dyn sections) - void scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder, - ELFSection &pSection, InputFile &pInput, - CopyRelocs &) override; - uint32_t getAddend(const Relocation *R) const override { return R->addend() - m_Target.getPacketOffset(*R); } @@ -73,6 +63,17 @@ class HexagonRelocator : public Relocator { void defineSymbolforGuard(eld::IRBuilder &pLinker, ResolveInfo *pSym, HexagonLDBackend &pTarget); + bool isRelocSupported(const Relocation &Reloc) const override; + + /// Determine whether GOT, PLT, or dynamic relocation entries are needed and + /// create them. For Hexagon: + /// - GOT entry (for .got and .got.plt sections) + /// - PLT entry (for .plt section) + /// - dynamic relocation entries (for .rel.plt and .rel.dyn sections) + void scanDeferredRelocation(InputFile &Input, Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) override; + private: bool isPICRelocTypeSupported(const Relocation &reloc) const override; virtual void scanLocalReloc(InputFile &pInput, Relocation &pReloc, @@ -90,8 +91,6 @@ class HexagonRelocator : public Relocator { HexagonTLSStub::StubType StaticStub, HexagonTLSStub::StubType DynStub); - bool isRelocSupported(Relocation &pReloc) const; - HexagonLDBackend &m_Target; LDSymbol *m_Guard; }; diff --git a/lib/Target/RISCV/RISCVRelocator.cpp b/lib/Target/RISCV/RISCVRelocator.cpp index 80a646102e..127bc062dc 100644 --- a/lib/Target/RISCV/RISCVRelocator.cpp +++ b/lib/Target/RISCV/RISCVRelocator.cpp @@ -379,8 +379,8 @@ RISCVGOT *RISCVRelocator::getTLSModuleID(ResolveInfo *R, bool isStatic) { return G; } -bool RISCVRelocator::isRelocSupported(Relocation &pReloc) const { - return RelocDescs.count(pReloc.type()) != 0; +bool RISCVRelocator::isRelocSupported(const Relocation &Reloc) const { + return RelocDescs.count(Reloc.type()) != 0; } // Check if relocation type is legal in code-independent links. @@ -402,111 +402,72 @@ bool RISCVRelocator::isPICRelocTypeSupported(const Relocation &reloc) const { } } -void RISCVRelocator::scanRelocation(Relocation &pReloc, eld::IRBuilder &pLinker, - ELFSection &pSection, InputFile &pInputFile, - CopyRelocs &CopyRelocs) { - if (LinkerConfig::Object == config().codeGenType()) - return; - - if (!isRelocSupported(pReloc)) { - config().raise(Diag::unsupported_reloc) - << pReloc.type() << pSection.getDecoratedName(config().options()) - << pInputFile.getInput()->decoratedPath(); - m_Target.getModule().setFailure(true); - return; - } - - if (!checkPICRelocSupported(pReloc)) - return; +void RISCVRelocator::diagnoseUnsupportedReloc(const Relocation &Reloc, + const ELFSection &Section, + const InputFile &Input) const { + Relocator::diagnoseUnsupportedReloc(Reloc, Section, Input); + m_Target.getModule().setFailure(true); +} - auto ProcessOneReloc = [&](Relocation &pReloc) -> void { - // rsym - The relocation target symbol - ResolveInfo *rsym = pReloc.symInfo(); - assert(nullptr != rsym && - "ResolveInfo of relocation not set while scanRelocation"); +void RISCVRelocator::scanDeferredRelocation(InputFile &Input, Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) { + ResolveInfo *Sym = Reloc.symInfo(); + eld::IRBuilder &Linker = *module().getIRBuilder(); + ELFObjectFile *Obj = llvm::dyn_cast(&Input); + + // Common relocation processing for both local and global symbols. + switch (Reloc.type()) { + case llvm::ELF::R_RISCV_TLSDESC_HI20: + case llvm::ELF::R_RISCV_TLSDESC_LOAD_LO12: + case llvm::ELF::R_RISCV_TLSDESC_ADD_LO12: + case llvm::ELF::R_RISCV_TLSDESC_CALL: { + if (config().isBuildingExecutable()) { + // Non-preemptible symbols in executables will be optimized or relaxed, + // no GOT needed. + if (!m_Target.isSymbolPreemptible(*Sym)) + return; - // Check if we are tracing relocations. - if (m_Module.getPrinter()->traceReloc()) { - std::lock_guard relocGuard(m_RelocMutex); - std::string relocName = getName(pReloc.type()); - if (config().options().traceReloc(relocName)) - config().raise(Diag::reloc_trace) - << relocName << pReloc.symInfo()->name() - << pInputFile.getInput()->decoratedPath(); - } + // RISC-V seems to dictate how each instruction in the sequence is + // transformed during IE/LE optimization. In particular, the instruction + // with R_RISCV_TLSDESC_ADD_LO12 is transformed to AUIPC, although it + // would be easier to keep the original AUIPC and remove the one with + // R_RISCV_TLSDESC_ADD_LO12. Therefore, the new load instruction will + // need a new relocation to indicate its base address. Reuse the + // existing R_RISCV_TLSDESC_ADD_LO12 as this is where the new auipc will + // be created. + if (Reloc.type() == llvm::ELF::R_RISCV_TLSDESC_ADD_LO12) + m_Target.setNewBaseForTLSDESCRelaxation(Reloc); + + if (Sym->reserved() & ReserveGOT) + return; - // check if we should issue undefined reference for the relocation target - // symbol - { - if (rsym->isUndef() || rsym->isBitCode()) { - std::lock_guard relocGuard(m_RelocMutex); - if (m_Target.canIssueUndef(rsym)) { - if (rsym->visibility() != ResolveInfo::Default) - issueInvisibleRef(pReloc, pInputFile); - issueUndefRef(pReloc, pInputFile, &pSection); - } - } + RISCVGOT *G = m_Target.createGOT(GOT::TLS_IE, Sym); + G->setValueType(GOT::TLSStaticSymbolValue); + helper_DynRel_init(Obj, &Reloc, Sym, G, 0x0, + is32bit() ? llvm::ELF::R_RISCV_TLS_TPREL32 + : llvm::ELF::R_RISCV_TLS_TPREL64, + m_Target); + } else { + if (Sym->reserved() & ReserveGOT) + return; + RISCVGOT *G = m_Target.createGOT(GOT::TLS_DESC, Sym); + helper_DynRel_init(Obj, &Reloc, Sym, G->getFirst(), 0x0, + llvm::ELF::R_RISCV_TLSDESC, m_Target); } - ELFSection *section = pSection.getLink(); - - if (!section->isAlloc()) - return; - - ELFObjectFile *Obj = llvm::dyn_cast(&pInputFile); - - // Common relocation processing for both local and global symbols. - switch (pReloc.type()) { - case llvm::ELF::R_RISCV_TLSDESC_HI20: - case llvm::ELF::R_RISCV_TLSDESC_LOAD_LO12: - case llvm::ELF::R_RISCV_TLSDESC_ADD_LO12: - case llvm::ELF::R_RISCV_TLSDESC_CALL: { - std::lock_guard relocGuard(m_RelocMutex); - if (config().isBuildingExecutable()) { - // Non-preemptible symbols in executables will be optimized or relaxed, - // no GOT needed. - if (!m_Target.isSymbolPreemptible(*rsym)) - return; - - // RISC-V seems to dictate how each instruction in the sequence is - // transformed during IE/LE optimization. In particular, the instruction - // with R_RISCV_TLSDESC_ADD_LO12 is transformed to AUIPC, although it - // would be easier to keep the original AUIPC and remove the one with - // R_RISCV_TLSDESC_ADD_LO12. Therefore, the new load instruction will - // need a new relocation to indicate its base address. Reuse the - // existing R_RISCV_TLSDESC_ADD_LO12 as this is where the new auipc will - // be created. - if (pReloc.type() == llvm::ELF::R_RISCV_TLSDESC_ADD_LO12) - m_Target.setNewBaseForTLSDESCRelaxation(pReloc); - - if (rsym->reserved() & ReserveGOT) - return; - - RISCVGOT *G = m_Target.createGOT(GOT::TLS_IE, rsym); - G->setValueType(GOT::TLSStaticSymbolValue); - helper_DynRel_init(Obj, &pReloc, rsym, G, 0x0, - is32bit() ? llvm::ELF::R_RISCV_TLS_TPREL32 - : llvm::ELF::R_RISCV_TLS_TPREL64, - m_Target); - } else { - if (rsym->reserved() & ReserveGOT) - return; - RISCVGOT *G = m_Target.createGOT(GOT::TLS_DESC, rsym); - helper_DynRel_init(Obj, &pReloc, rsym, G->getFirst(), 0x0, - llvm::ELF::R_RISCV_TLSDESC, m_Target); - } + Sym->setReserved(Sym->reserved() | ReserveGOT); + return; + } - rsym->setReserved(rsym->reserved() | ReserveGOT); - return; - } - } + default: + break; + } - if (rsym->isLocal()) // rsym is local - scanLocalReloc(pInputFile, pReloc, pLinker, *section); - else // rsym is external - scanGlobalReloc(pInputFile, pReloc, pLinker, *section, CopyRelocs); - }; - ProcessOneReloc(pReloc); + if (Sym->isLocal()) + scanLocalReloc(Input, Reloc, Linker, Section); + else + scanGlobalReloc(Input, Reloc, Linker, Section, CopyRelocSet); } uint32_t RISCVRelocator::getNumRelocs() const { diff --git a/lib/Target/RISCV/RISCVRelocator.h b/lib/Target/RISCV/RISCVRelocator.h index 621c46aeb2..0cd3707302 100644 --- a/lib/Target/RISCV/RISCVRelocator.h +++ b/lib/Target/RISCV/RISCVRelocator.h @@ -28,10 +28,6 @@ class RISCVRelocator : public Relocator { Result applyRelocation(Relocation &pRelocation) override; - void scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder, - ELFSection &pSection, InputFile &pInput, - CopyRelocs &) override; - // Handle partial linking void partialScanRelocation(Relocation &pReloc, const ELFSection &pSection) override; @@ -48,6 +44,17 @@ class RISCVRelocator : public Relocator { bool is32bit() const { return config().targets().is32Bits(); } +protected: + bool isRelocSupported(const Relocation &Reloc) const override; + + void diagnoseUnsupportedReloc(const Relocation &Reloc, + const ELFSection &Section, + const InputFile &Input) const override; + + void scanDeferredRelocation(InputFile &Input, Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) override; + private: bool isPICRelocTypeSupported(const Relocation &reloc) const override; @@ -60,8 +67,6 @@ class RISCVRelocator : public Relocator { RISCVGOT *getTLSModuleID(ResolveInfo *R, bool isStatic); - bool isRelocSupported(Relocation &pReloc) const; - RISCVGOT *getTLSModuleID(ResolveInfo *rsym); void handleScanForNonPreemptibleIFunc(Relocation &R, ELFObjectFile *Obj); diff --git a/lib/Target/Relocator.cpp b/lib/Target/Relocator.cpp index a16bb7a09b..e979d8cd4d 100644 --- a/lib/Target/Relocator.cpp +++ b/lib/Target/Relocator.cpp @@ -31,6 +31,119 @@ using namespace eld; +void Relocator::initScanRelocations() { + const auto &Inputs = m_Module.getObjectList(); + DeferredRelocations.clear(); + DeferredRelocations.resize(Inputs.size()); + for (size_t I = 0; I != Inputs.size(); ++I) { + auto *ObjFile = llvm::dyn_cast(Inputs[I]); + if (!ObjFile) + continue; + auto &RelocationsBySection = DeferredRelocations[I].RelocationsBySection; + for (ELFSection *Rs : ObjFile->getRelocationSections()) + RelocationsBySection.emplace_back(Rs->getLink()->getRelocationCount()); + } +} + +void Relocator::diagnoseUnsupportedReloc(const Relocation &Reloc, + const ELFSection &Section, + const InputFile &Input) const { + config().raise(Diag::unsupported_reloc) + << Reloc.type() << Section.getDecoratedName(config().options()) + << Input.getInput()->decoratedPath(); +} + +bool Relocator::canIssueUndefRef(ResolveInfo *Sym) { + return getTarget().canIssueUndef(Sym); +} + +bool Relocator::scanRelocationPrologue(Relocation &Reloc, ELFSection &Section, + InputFile &Input) { + if (LinkerConfig::Object == config().codeGenType()) + return false; + + // TODO: phase 1 runs concurrently, so the order in which these diagnostics + // interleave between input files is not deterministic. + if (!isRelocSupported(Reloc)) { + diagnoseUnsupportedReloc(Reloc, Section, Input); + return false; + } + + if (!checkPICRelocSupported(Reloc)) + return false; + + ResolveInfo *Sym = Reloc.symInfo(); + assert(nullptr != Sym && + "ResolveInfo of relocation not set while scanRelocation"); + + if (m_Module.getPrinter()->traceReloc()) { + std::string RelocName = getName(Reloc.type()); + if (config().options().traceReloc(RelocName)) + config().raise(Diag::reloc_trace) + << RelocName << Sym->name() << Input.getInput()->decoratedPath(); + } + + // Report an undefined reference for the relocation target symbol. + if ((Sym->isUndef() || Sym->isBitCode()) && canIssueUndefRef(Sym)) { + std::lock_guard relocGuard(m_RelocMutex); + if (Sym->visibility() != ResolveInfo::Default) + issueInvisibleRef(Reloc, Input); + issueUndefRef(Reloc, Input, &Section); + } + + ELFSection *LinkedSection = Section.getLink(); + if (!LinkedSection->isAlloc()) { + scanNonAllocReloc(Reloc, Section); + return false; + } + + return true; +} + +void Relocator::scanRelocation(Relocation &Reloc, ELFSection &Section, + InputFile &Input, CopyRelocs &CopyRelocSet) { + if (!scanRelocationPrologue(Reloc, Section, Input)) + return; + scanDeferredRelocation(Input, Reloc, *Section.getLink(), CopyRelocSet); +} + +void Relocator::scanRelocationParallel(Relocation &Reloc, ELFSection &Section, + InputFile &Input, size_t InputIndex, + size_t SectIndex, size_t RelocIndex) { + if (!scanRelocationPrologue(Reloc, Section, Input)) + return; + + // Defer everything that reaches here to the serial phase, so that no slot is + // allocated and nothing is appended to a shared section off the main thread. + assert(InputIndex < DeferredRelocations.size() && + "input missing from scan bookkeeping"); + DeferredRelocations[InputIndex].RelocationsBySection[SectIndex].set( + RelocIndex); +} + +void Relocator::replayScanRelocations(CopyRelocs &CopyRelocSet) { + const auto &Inputs = m_Module.getObjectList(); + for (size_t I = 0; I != Inputs.size(); ++I) { + auto *ObjFile = llvm::dyn_cast(Inputs[I]); + if (!ObjFile) + continue; + auto &RelocationsBySection = DeferredRelocations[I].RelocationsBySection; + size_t SectIndex = 0; + for (ELFSection *Rs : ObjFile->getRelocationSections()) { + const llvm::BitVector &Deferred = RelocationsBySection[SectIndex++]; + if (Deferred.none()) + continue; + ELFSection *Section = Rs->getLink(); + size_t RelocIndex = 0; + for (Relocation *Reloc : Section->getRelocations()) { + if (Deferred.test(RelocIndex++)) + scanDeferredRelocation(*Inputs[I], *Reloc, *Section, CopyRelocSet); + } + } + } + DeferredRelocations.clear(); +} + //===----------------------------------------------------------------------===// // Relocator //===----------------------------------------------------------------------===// diff --git a/lib/Target/Template/TemplateRelocator.cpp b/lib/Target/Template/TemplateRelocator.cpp index 4f6cca0f64..041b9f1b36 100644 --- a/lib/Target/Template/TemplateRelocator.cpp +++ b/lib/Target/Template/TemplateRelocator.cpp @@ -61,51 +61,15 @@ const char *TemplateRelocator::getName(Relocation::Type pType) const { return ""; } -void TemplateRelocator::scanRelocation(Relocation &pReloc, - eld::IRBuilder &pLinker, - ELFSection &pSection, - InputFile &pInputFile, - CopyRelocs &CopyRelocs) { - if (LinkerConfig::Object == config().codeGenType()) - return; - - // rsym - The relocation target symbol - ResolveInfo *rsym = pReloc.symInfo(); - assert(nullptr != rsym && - "ResolveInfo of relocation not set while scanRelocation"); - - // Check if we are tracing relocations. - if (m_Module.getPrinter()->traceReloc()) { - std::lock_guard relocGuard(m_RelocMutex); - std::string relocName = getName(pReloc.type()); - if (config().options().traceReloc(relocName)) - config().raise(Diag::reloc_trace) - << relocName << pReloc.symInfo()->name() - << pInputFile.getInput()->decoratedPath(); - } - - // check if we should issue undefined reference for the relocation target - // symbol - { - if (rsym->isUndef() || rsym->isBitCode()) { - std::lock_guard relocGuard(m_RelocMutex); - if (m_Target.canIssueUndef(rsym)) { - if (rsym->visibility() != ResolveInfo::Default) - issueInvisibleRef(pReloc, pInputFile); - issueUndefRef(pReloc, pInputFile, &pSection); - } - } - } - - ELFSection *section = pSection.getLink(); - - if (!section->isAlloc()) - return; - - if (rsym->isLocal()) // rsym is local - scanLocalReloc(pInputFile, pReloc, pLinker, *section); - else // rsym is external - scanGlobalReloc(pInputFile, pReloc, pLinker, *section); +void TemplateRelocator::scanDeferredRelocation(InputFile &Input, + Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) { + eld::IRBuilder &Linker = *module().getIRBuilder(); + if (Reloc.symInfo()->isLocal()) + scanLocalReloc(Input, Reloc, Linker, Section); + else + scanGlobalReloc(Input, Reloc, Linker, Section); } Relocation::Size TemplateRelocator::getSize(Relocation::Type pType) const { diff --git a/lib/Target/Template/TemplateRelocator.h b/lib/Target/Template/TemplateRelocator.h index 5945cd4f46..f2763a6de6 100644 --- a/lib/Target/Template/TemplateRelocator.h +++ b/lib/Target/Template/TemplateRelocator.h @@ -28,10 +28,6 @@ class TemplateRelocator : public Relocator { Result applyRelocation(Relocation &pRelocation) override; - void scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder, - ELFSection &pSection, InputFile &pInput, - CopyRelocs &) override; - // Handle partial linking void partialScanRelocation(Relocation &pReloc, const ELFSection &pSection) override; @@ -46,6 +42,11 @@ class TemplateRelocator : public Relocator { uint32_t getNumRelocs() const override; +protected: + void scanDeferredRelocation(InputFile &Input, Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) override; + private: virtual void scanLocalReloc(InputFile &pInput, Relocation &pReloc, eld::IRBuilder &pBuilder, ELFSection &pSection); diff --git a/lib/Target/X86/X86_32Relocator.cpp b/lib/Target/X86/X86_32Relocator.cpp index 569d872bbb..a0a189b717 100644 --- a/lib/Target/X86/X86_32Relocator.cpp +++ b/lib/Target/X86/X86_32Relocator.cpp @@ -16,8 +16,8 @@ Relocator::Result X86_32Relocator::applyRelocation(Relocation &) { return Relocator::Unsupport; } -void X86_32Relocator::scanRelocation(Relocation &, eld::IRBuilder &, - ELFSection &, InputFile &, CopyRelocs &) {} +void X86_32Relocator::scanDeferredRelocation(InputFile &, Relocation &, + ELFSection &, CopyRelocs &) {} const char *X86_32Relocator::getName(Relocation::Type) const { return ""; diff --git a/lib/Target/X86/X86_32Relocator.h b/lib/Target/X86/X86_32Relocator.h index 11950e77e3..4d2dd4ac86 100644 --- a/lib/Target/X86/X86_32Relocator.h +++ b/lib/Target/X86/X86_32Relocator.h @@ -21,8 +21,7 @@ class X86_32Relocator : public Relocator { Result applyRelocation(Relocation &) override; - void scanRelocation(Relocation &, eld::IRBuilder &, ELFSection &, InputFile &, - CopyRelocs &) override; + bool shouldScanRelocations() const override { return false; } X86_32LDBackend &getTarget() override { return m_Target; } const X86_32LDBackend &getTarget() const override { return m_Target; } @@ -32,6 +31,10 @@ class X86_32Relocator : public Relocator { Size getSize(Relocation::Type) const override; uint32_t getNumRelocs() const override; +protected: + void scanDeferredRelocation(InputFile &, Relocation &, ELFSection &, + CopyRelocs &) override; + private: X86_32LDBackend &m_Target; }; diff --git a/lib/Target/X86/x86_64Relocator.cpp b/lib/Target/X86/x86_64Relocator.cpp index b669b3cf6b..d98dbead86 100644 --- a/lib/Target/X86/x86_64Relocator.cpp +++ b/lib/Target/X86/x86_64Relocator.cpp @@ -70,9 +70,9 @@ Relocator::Size x86_64Relocator::getSize(Relocation::Type pType) const { return x86_64Relocs[pType].Size; } -bool x86_64Relocator::isRelocSupported(const Relocation &pReloc) const { +bool x86_64Relocator::isRelocSupported(const Relocation &Reloc) const { - switch (pReloc.type()) { + switch (Reloc.type()) { case llvm::ELF::R_X86_64_NONE: case llvm::ELF::R_X86_64_64: case llvm::ELF::R_X86_64_PC32: @@ -101,60 +101,15 @@ bool x86_64Relocator::isRelocSupported(const Relocation &pReloc) const { } } -void x86_64Relocator::scanRelocation(Relocation &pReloc, - eld::IRBuilder &pLinker, - ELFSection &pSection, - InputFile &pInputFile, - CopyRelocs &CopyRelocs) { - if (LinkerConfig::Object == config().codeGenType()) - return; - - if (!isRelocSupported(pReloc)) { - config().raise(Diag::unsupported_reloc) - << pReloc.type() << pSection.getDecoratedName(config().options()) - << pInputFile.getInput()->decoratedPath(); - return; - } - - if (!checkPICRelocSupported(pReloc)) - return; - - // rsym - The relocation target symbol - ResolveInfo *rsym = pReloc.symInfo(); - assert(nullptr != rsym && - "ResolveInfo of relocation not set while scanRelocation"); - - // Check if we are tracing relocations. - if (m_Module.getPrinter()->traceReloc()) { - std::lock_guard relocGuard(m_RelocMutex); - std::string relocName = getName(pReloc.type()); - if (config().options().traceReloc(relocName)) - config().raise(Diag::reloc_trace) - << relocName << pReloc.symInfo()->name() - << pInputFile.getInput()->decoratedPath(); - } - - // check if we should issue undefined reference for the relocation target - // symbol - if (rsym->isUndef() || rsym->isBitCode()) { - std::lock_guard relocGuard(m_RelocMutex); - if (!m_Target.canProvideSymbol(rsym)) { - if (m_Target.canIssueUndef(rsym)) { - if (rsym->visibility() != ResolveInfo::Default) - issueInvisibleRef(pReloc, pInputFile); - issueUndefRef(pReloc, pInputFile, &pSection); - } - } - } - ELFSection *section = pSection.getLink(); - - if (!section->isAlloc()) - return; - - if (rsym->isLocal()) // rsym is local - scanLocalReloc(pInputFile, pReloc, pLinker, *section); - else // rsym is external - scanGlobalReloc(pInputFile, pReloc, pLinker, *section, CopyRelocs); +void x86_64Relocator::scanDeferredRelocation(InputFile &Input, + Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) { + if (Reloc.symInfo()->isLocal()) + scanLocalReloc(Input, Reloc, *module().getIRBuilder(), Section); + else + scanGlobalReloc(Input, Reloc, *module().getIRBuilder(), Section, + CopyRelocSet); } namespace { @@ -346,9 +301,8 @@ void x86_64Relocator::scanGlobalReloc(InputFile &pInputFile, Relocation &pReloc, << rsym->resolvedOrigin()->getInput()->decoratedPath(); return; } + // Copy relocations are created after the whole scan completes. copyRelocs.insert(rsym); - // Do not emit a dynamic relocation here; copy reloc will be created - // later return; } // No copy reloc needed: emit a dynamic relocation as before diff --git a/lib/Target/X86/x86_64Relocator.h b/lib/Target/X86/x86_64Relocator.h index 97b882cb71..5098d68652 100644 --- a/lib/Target/X86/x86_64Relocator.h +++ b/lib/Target/X86/x86_64Relocator.h @@ -32,16 +32,6 @@ class x86_64Relocator : public Relocator { Result applyRelocation(Relocation &pRelocation) override; - /// scanRelocation - determine the empty entries are needed or not and create - /// the empty entries if needed. - /// For x86_64, following entries are check to create: - /// - GOT entry (for .got and .got.plt sections) - /// - PLT entry (for .plt section) - /// - dynamin relocation entries (for .rel.plt and .rel.dyn sections) - void scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder, - ELFSection &pSection, InputFile &pInput, - CopyRelocs &) override; - // Handle partial linking void partialScanRelocation(Relocation &pReloc, const ELFSection &pSection) override; @@ -62,6 +52,17 @@ class x86_64Relocator : public Relocator { void defineSymbolforGuard(eld::IRBuilder &pLinker, ResolveInfo *pSym, x86_64LDBackend &pTarget); + bool isRelocSupported(const Relocation &Reloc) const override; + + /// Determine whether GOT, PLT, or dynamic relocation entries are needed and + /// create them. For x86_64: + /// - GOT entry (for .got and .got.plt sections) + /// - PLT entry (for .plt section) + /// - dynamic relocation entries (for .rel.plt and .rel.dyn sections) + void scanDeferredRelocation(InputFile &Input, Relocation &Reloc, + ELFSection &Section, + CopyRelocs &CopyRelocSet) override; + private: virtual void scanLocalReloc(InputFile &pInput, Relocation &pReloc, eld::IRBuilder &pBuilder, ELFSection &pSection); @@ -70,8 +71,6 @@ class x86_64Relocator : public Relocator { eld::IRBuilder &pBuilder, ELFSection &pSection, CopyRelocs &); - bool isRelocSupported(const Relocation &pReloc) const; - x86_64GOT *getTLSModuleID(ResolveInfo *R, bool isStatic = false); x86_64LDBackend &m_Target; diff --git a/test/Common/standalone/GOTPLTDeterminism/GOTPLTDeterminism.test b/test/Common/standalone/GOTPLTDeterminism/GOTPLTDeterminism.test index 3b15068a8f..3b95e848ff 100644 --- a/test/Common/standalone/GOTPLTDeterminism/GOTPLTDeterminism.test +++ b/test/Common/standalone/GOTPLTDeterminism/GOTPLTDeterminism.test @@ -8,15 +8,25 @@ # RUN: %clang %clangopts -c -fPIC %t/c.c -o %t/c.o # RUN: %clang %clangopts -c -fPIC %t/main.c -o %t/main.o -# RUN: %link %linkopts -shared --enable-threads=all -o %t/out.1 %t/main.o %t/a.o %t/b.o %t/c.o -# RUN: %link %linkopts -shared --enable-threads=all -o %t/out.2 %t/main.o %t/a.o %t/b.o %t/c.o -# RUN: %link %linkopts -shared --enable-threads=all -o %t/out.3 %t/main.o %t/a.o %t/b.o %t/c.o +# RUN: %link %linkopts -shared --enable-threads=all --thread-count=8 -o %t/out.1 %t/main.o %t/a.o %t/b.o %t/c.o +# RUN: %link %linkopts -shared --enable-threads=all --thread-count=8 -o %t/out.2 %t/main.o %t/a.o %t/b.o %t/c.o +# RUN: %link %linkopts -shared --enable-threads=all --thread-count=8 -o %t/out.3 %t/main.o %t/a.o %t/b.o %t/c.o # RUN: cmp %t/out.1 %t/out.2 # RUN: cmp %t/out.1 %t/out.3 # RUN: %link %linkopts -shared --no-threads -o %t/out.serial %t/main.o %t/a.o %t/b.o %t/c.o # RUN: cmp %t/out.1 %t/out.serial +# RUN: %readelf -r -s %t/out.1 > %t/tables.parallel +# RUN: %readelf -r -s %t/out.serial > %t/tables.serial +# RUN: %diff %t/tables.parallel %t/tables.serial + +## Changing the input order is allowed to change slot order, but must still be +## deterministic and must still agree between a serial and a parallel link. +# RUN: %link %linkopts -shared --enable-threads=all --thread-count=8 -o %t/rev.parallel %t/main.o %t/c.o %t/b.o %t/a.o +# RUN: %link %linkopts -shared --no-threads -o %t/rev.serial %t/main.o %t/c.o %t/b.o %t/a.o +# RUN: cmp %t/rev.parallel %t/rev.serial + #--- a.c extern int g0, g1, g2, g3, g4, g5, g6, g7; int fa(void) { return g0 + g1 + g2 + g3 + g4 + g5 + g6 + g7; } diff --git a/test/X86/i386/standalone/Backend/ELF32.test b/test/X86/i386/standalone/Backend/ELF32.test index 89857d4110..3bd71b11ec 100644 --- a/test/X86/i386/standalone/Backend/ELF32.test +++ b/test/X86/i386/standalone/Backend/ELF32.test @@ -9,8 +9,11 @@ # RUN: %readelf -h %t.explicit | %filecheck %s --check-prefix=EXPLICIT # RUN: %link %t.o -o %t.inferred # RUN: %readelf -h %t.inferred | %filecheck %s --check-prefix=INFERRED +# RUN: %link -m elf_i386 --enable-threads=all --thread-count=4 --trace=threads \ +# RUN: %t.o -o %t.threaded 2>&1 | %filecheck %s --check-prefix=THREADS # # EXPLICIT: Class: ELF32 # EXPLICIT: Machine: Intel 80386 # INFERRED: Class: ELF32 # INFERRED: Machine: Intel 80386 +# THREADS: Threads Disabled : ScanRelocations diff --git a/test/X86/x86_64/linux/relocCOPY/relocCOPY.test b/test/X86/x86_64/linux/relocCOPY/relocCOPY.test index eb22245561..7053899da5 100644 --- a/test/X86/x86_64/linux/relocCOPY/relocCOPY.test +++ b/test/X86/x86_64/linux/relocCOPY/relocCOPY.test @@ -17,10 +17,17 @@ RUN: %readelf -r %t.out | %filecheck %s --check-prefix=CHECK-COPY-RELOC RUN: %readelf -s %t.out | %filecheck %s --check-prefix=CHECK-BSS-SYMBOLS RUN: %run_and_print_exit %t.out | %filecheck %s --check-prefix=EXITCODE +## Copy relocations are created after the relocation scan, so a threaded scan +## must produce the same output as a serial one. +RUN: %link %linkopts --enable-threads=all --thread-count=8 -o %t.threaded %t.1.o %t.libu.so +RUN: %link %linkopts --no-threads -o %t.serial %t.1.o %t.libu.so +RUN: cmp %t.threaded %t.serial +RUN: %readelf -r %t.threaded | %filecheck %s --check-prefix=CHECK-COPY-RELOC + CHECK-ABS-TYPE-DAG: {{.*}}R_X86_64_64{{.*}}v{{.*}} CHECK-ABS-TYPE-DAG: {{.*}}R_X86_64_32S{{.*}}u{{.*}} CHECK-COPY-RELOC-DAG: {{.*}}R_X86_64_COPY{{.*}}u{{.*}} CHECK-COPY-RELOC-DAG: {{.*}}R_X86_64_COPY{{.*}}v{{.*}} CHECK-BSS-SYMBOLS-DAG: {{.*}}OBJECT{{.*}}u{{.*}} CHECK-BSS-SYMBOLS-DAG: {{.*}}OBJECT{{.*}}v{{.*}} -EXITCODE: 16 \ No newline at end of file +EXITCODE: 16 diff --git a/test/X86/x86_64/standalone/ParallelScanCopyGOT/ParallelScanCopyGOT.test b/test/X86/x86_64/standalone/ParallelScanCopyGOT/ParallelScanCopyGOT.test new file mode 100644 index 0000000000..1e3da9a13d --- /dev/null +++ b/test/X86/x86_64/standalone/ParallelScanCopyGOT/ParallelScanCopyGOT.test @@ -0,0 +1,32 @@ +## A copy relocation must not change the scan classification of a deferred +## PIC GOT reference to the same symbol. + +# RUN: split-file %s %t +# RUN: %llvm-mc -triple=x86_64 -filetype=obj %t/copy.s -o %t/copy.o +# RUN: %clang %clangopts -fPIC -c %t/pic.c -o %t/pic.o +# RUN: %clang %clangopts -fPIC -c %t/lib.c -o %t/lib.o +# RUN: %link %linkopts -shared %t/lib.o -o %t/libdata.so + +# RUN: %link %linkopts --enable-threads=all --thread-count=8 -Bdynamic \ +# RUN: -e _start %t/copy.o %t/pic.o %t/libdata.so -o %t/out +# RUN: %link %linkopts --no-threads -Bdynamic -e _start %t/copy.o %t/pic.o \ +# RUN: %t/libdata.so -o %t/serial +# RUN: %readelf -r -s %t/out | %filecheck %s +# RUN: %readelf -r -s %t/serial | %filecheck %s + +# CHECK-DAG: R_X86_64_COPY{{.*}}data +# CHECK-DAG: R_X86_64_GLOB_DAT{{.*}}data + +#--- copy.s + .text + .quad data + .globl _start +_start: + retq + +#--- pic.c +extern int data; +int read_data(void) { return data; } + +#--- lib.c +int data = 42; diff --git a/test/X86/x86_64/standalone/ParallelScanRelocations/ParallelScanRelocations.test b/test/X86/x86_64/standalone/ParallelScanRelocations/ParallelScanRelocations.test new file mode 100644 index 0000000000..d7e670b533 --- /dev/null +++ b/test/X86/x86_64/standalone/ParallelScanRelocations/ParallelScanRelocations.test @@ -0,0 +1,25 @@ +## The x86 relocation scan records allocation work per input in parallel and +## replays it in input order. + +# RUN: split-file %s %t +# RUN: %llvm-mc -triple=x86_64 -filetype=obj %t/a.s -o %t/a.o +# RUN: %llvm-mc -triple=x86_64 -filetype=obj %t/b.s -o %t/b.o + +# RUN: %link %linkopts -shared --enable-threads=all --thread-count=4 \ +# RUN: --trace=threads %t/a.o %t/b.o -o %t/out.so 2>&1 | %filecheck %s + +# CHECK: Note: Threads Enabled ScanRelocations, Number of threads = 4 + +#--- a.s + .text + .globl call_foo +call_foo: + callq foo@PLT + movq foo@GOTPCREL(%rip), %rax + retq + +#--- b.s + .text + .globl foo +foo: + retq