[Relocator] Restore parallelism in scanRelocations

include/eld/Config/LinkerConfig.h

@@ -67,13 +67,14 @@ class LinkerConfig {
   enum EnableThreadsOpt {
     NoThreads = 0,
     AssignOutputSections = 0x1,
+    ScanRelocations = 0x2,
     SyncRelocations = 0x4,
     CheckCrossRefs = 0x8,
     CreateOutputSections = 0x10,
     ApplyRelocations = 0x20,
     LinkerRelaxation = 0x40,
     AssignVersionScriptNodes = 0x80,
-    AllThreads = 0x1 | 0x4 | 0x8 | 0x10 | 0x20 | 0x40 | 0x80,
+    AllThreads = 0x1 | 0x2 | 0x4 | 0x8 | 0x10 | 0x20 | 0x40 | 0x80,
   };
 
   enum SymDefStyle { Default, Provide, UnknownSymDefStyle };
@@ -151,6 +152,10 @@ class LinkerConfig {
     return EnableThreads & LinkerConfig::AssignOutputSections;
   }
 
+  bool isScanRelocationsMultiThreaded() const {
+    return EnableThreads & LinkerConfig::ScanRelocations;
+  }
+
   bool isSyncRelocationsMultiThreaded() const {
     return EnableThreads & LinkerConfig::SyncRelocations;
   }
@@ -179,6 +184,8 @@ class LinkerConfig {
     EnableThreads = NoThreads;
     if (EnableThreadsOpt & AssignOutputSections)
       EnableThreads |= AssignOutputSections;
+    if (EnableThreadsOpt & ScanRelocations)
+      EnableThreads |= ScanRelocations;
     if (EnableThreadsOpt & SyncRelocations)
       EnableThreads |= SyncRelocations;
     if (EnableThreadsOpt & CheckCrossRefs)

include/eld/Core/Module.h

@@ -39,6 +39,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include <array>
+#include <atomic>
 #include <climits>
 #include <mutex>
 #include <optional>
@@ -702,7 +703,7 @@ class Module {
   LDSymbol *DotSymbol = nullptr;
   Linker *L = nullptr;
   LayoutInfo *ThisLayoutInfo = nullptr;
-  bool Failure = false;
+  std::atomic<bool> Failure = false;
   bool UsesLto = false;
   LinkState State = LinkState::Unknown;
   ReplaceFragsVectorT ReplaceFrags;

include/eld/Object/ObjectLinker.h

@@ -185,9 +185,10 @@ class ObjectLinker {
   /// scanRelocations - scan all relocation entries by output symbols.
   bool scanRelocations(bool IsPartialLink = false);
 
-  void scanRelocationsHelper(InputFile *Input, bool IsPartialLink,
-                             LinkerScript::PluginVectorT PluginVect,
-                             Relocator::CopyRelocs &);
+  void scanRelocationsHelper(
+      InputFile *Input, size_t InputIndex, bool IsPartialLink, bool IsParallel,
+      Relocator::CopyRelocs &CopyRelocSet,
+      const llvm::DenseMap<Relocation *, bool> &SkipRelocProcessing);
 
   bool finalizeScanRelocations();
 
@@ -336,7 +337,7 @@ class ObjectLinker {
   void applySubAlign();
 
   // Get Plugin list for Relocation registration callback.
-  LinkerScript::PluginVectorT getLinkerPluginWithLinkerConfigs();
+  LinkerScript::PluginVectorT getLinkerPluginsWithRelocCallbacks();
 
   // ------------------ All sections -------------------------
   const std::vector<Section *> &getAllInputSections() const {

include/eld/Script/Plugin.h

@@ -141,6 +141,8 @@ class Plugin {
 
   bool isRelocTypeRegistered(uint32_t RelocType, Relocation *R);
 
+  bool hasRegisteredRelocations() const;
+
   plugin::LinkerPluginConfig *getLinkerPluginConfig() const;
 
   // -----------------Check if timing is enabled ---------------

include/eld/Target/Relocator.h

@@ -15,11 +15,13 @@
 
 #include "eld/Core/Module.h"
 #include "eld/Readers/Relocation.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include <mutex>
-#include <unordered_set>
 
 namespace eld {
 
@@ -69,7 +71,10 @@ class Relocator {
     ReservePLT = 4,
   };
 
-  typedef std::unordered_set<ResolveInfo *> CopyRelocs;
+  /// Symbols needing a copy relocation, in the order phase 2 discovered them.
+  /// Insertion order is preserved so that the copy relocations, and the
+  /// .dynbss sections created for them, are deterministic.
+  using CopyRelocs = llvm::SetVector<ResolveInfo *>;
 
 public:
   Relocator(LinkerConfig &pConfig, Module &pModule)
@@ -80,9 +85,32 @@ class Relocator {
   /// apply - general apply function
   virtual Result applyRelocation(Relocation &pRelocation) = 0;
 
-  virtual void scanRelocation(Relocation &pReloc, eld::IRBuilder &pBuilder,
-                              ELFSection &pSection, InputFile &pInput,
-                              CopyRelocs &CopyRelocs) = 0;
+  virtual bool shouldScanRelocations() const { return true; }
+
+  /// Parallel relocation scanning runs in two phases:
+  ///
+  /// Phase 1 (scanRelocationParallel) may run concurrently over input files.
+  /// It only performs the checks that are common to every backend and records
+  /// which relocations need further processing. It creates no GOT/PLT slots
+  /// and no dynamic relocations.
+  ///
+  /// Phase 2 (replayScanRelocations) is serial and visits the recorded
+  /// relocations in input order. Slot allocation happens here.
+  ///
+  /// Prepare for phase 1. Must be called before a parallel scan.
+  void initScanRelocations();
+
+  /// Serial single pass scan.
+  void scanRelocation(Relocation &Reloc, ELFSection &Section, InputFile &Input,
+                      CopyRelocs &CopyRelocSet);
+
+  void scanRelocationParallel(Relocation &Reloc, ELFSection &Section,
+                              InputFile &Input, size_t InputIndex,
+                              size_t SectIndex, size_t RelocIndex);
+
+  /// Phase 2: Replays the relocations recorded by phase 1 in input
+  /// order, serially.
+  void replayScanRelocations(CopyRelocs &CopyRelocSet);
 
   // Issue an undefined reference error if the symbol is a magic section symbol.
   void issueUndefRefForMagicSymbol(const Relocation &pReloc);
@@ -174,6 +202,23 @@ class Relocator {
 protected:
   bool reportNonPICRelocation(const Relocation &reloc) const;
 
+  virtual bool isRelocSupported(const Relocation &Reloc) const { return true; }
+
+  virtual void diagnoseUnsupportedReloc(const Relocation &Reloc,
+                                        const ELFSection &Section,
+                                        const InputFile &Input) const;
+
+  virtual bool canIssueUndefRef(ResolveInfo *Sym);
+
+  /// Phase 1:  Called for a relocation whose target section is not allocatable,
+  /// just before the relocation is dropped.
+  virtual void scanNonAllocReloc(Relocation &Reloc, ELFSection &Section) {}
+
+  /// Phase 2: Handle one relocation recorded by phase 1, serially.
+  virtual void scanDeferredRelocation(InputFile &Input, Relocation &Reloc,
+                                      ELFSection &Section,
+                                      CopyRelocs &CopyRelocSet) = 0;
+
   virtual bool isPICRelocTypeSupported(const Relocation &reloc) const {
     return true;
   }
@@ -185,6 +230,15 @@ class Relocator {
   Module &m_Module;
   std::mutex m_RelocMutex;
   std::unordered_map<std::string, uint32_t> RelocNameMap;
+
+private:
+  bool scanRelocationPrologue(Relocation &Reloc, ELFSection &Section,
+                              InputFile &Input);
+
+  struct DeferredInput {
+    llvm::SmallVector<llvm::BitVector, 0> RelocationsBySection;
+  };
+  llvm::SmallVector<DeferredInput, 0> DeferredRelocations;
 };
 
 Relocator::Result checkSignedRange(Relocation &Rel, Relocator &R, int64_t Value,

lib/Object/ObjectLinker.cpp

@@ -2289,67 +2289,166 @@ void ObjectLinker::createCopyRelocation(ResolveInfo &Sym,
     addCopyReloc(getTargetBackend(), *CopySym.resolveInfo(), Type);
 }
 
-void ObjectLinker::scanRelocationsHelper(InputFile *Input, bool IsPartialLink,
-                                         LinkerScript::PluginVectorT PVect,
-                                         Relocator::CopyRelocs &CopyRelocs) {
+void ObjectLinker::scanRelocationsHelper(
+    InputFile *Input, size_t InputIndex, bool IsPartialLink, bool IsParallel,
+    Relocator::CopyRelocs &CopyRelocSet,
+    const llvm::DenseMap<Relocation *, bool> &SkipRelocProcessing) {
   ELFObjectFile *ObjFile = llvm::dyn_cast<ELFObjectFile>(Input);
   if (!ObjFile)
     return;
-  uint32_t NumPlugins = PVect.size();
+  bool ShouldScanRelocations =
+      getTargetBackend().getRelocator()->shouldScanRelocations();
+  size_t SectIndex = 0;
   for (auto &Rs : ObjFile->getRelocationSections()) {
+    const size_t ThisSectIndex = SectIndex++;
     if (Rs->isIgnore())
       continue;
     if (Rs->isDiscard())
       continue;
+    size_t RelocIndex = 0;
     for (auto &Relocation : Rs->getLink()->getRelocations()) {
+      const size_t ThisRelocIndex = RelocIndex++;
       // Skip unneeded relocations
-      if (getTargetBackend().maySkipRelocProcessing(Relocation))
+      auto SkipIt = SkipRelocProcessing.find(Relocation);
+      bool ShouldSkip =
+          SkipIt != SkipRelocProcessing.end()
+              ? SkipIt->second
+              : getTargetBackend().maySkipRelocProcessing(Relocation);
+      if (ShouldSkip)
         continue;
-      if (NumPlugins) {
-        for (auto &P : PVect)
-          P->callReloc(Relocation->type(), Relocation);
-      }
       ELFSection *RelocSection = Rs;
       // scan relocation
-      if (!IsPartialLink)
-        getTargetBackend().getRelocator()->scanRelocation(
-            *Relocation, *ThisModule->getIRBuilder(), *RelocSection, *Input,
-            CopyRelocs);
-      else
+      if (IsPartialLink) {
         getTargetBackend().getRelocator()->partialScanRelocation(*Relocation,
                                                                  *RelocSection);
+      } else if (!ShouldScanRelocations) {
+        continue;
+      } else if (!IsParallel) {
+        getTargetBackend().getRelocator()->scanRelocation(
+            *Relocation, *RelocSection, *Input, CopyRelocSet);
+      } else {
+        getTargetBackend().getRelocator()->scanRelocationParallel(
+            *Relocation, *RelocSection, *Input, InputIndex, ThisSectIndex,
+            ThisRelocIndex);
+      }
     } // for all relocations
   } // for all relocation section
 }
 
-LinkerScript::PluginVectorT ObjectLinker::getLinkerPluginWithLinkerConfigs() {
+LinkerScript::PluginVectorT ObjectLinker::getLinkerPluginsWithRelocCallbacks() {
   const LinkerScript::PluginVectorT PluginVect =
       ThisModule->getScript().getPlugins();
-  LinkerScript::PluginVectorT PluginVectHavingLinkerConfigs;
+  LinkerScript::PluginVectorT PluginVectHavingRelocCallbacks;
   for (auto &P : PluginVect) {
-    if (P->getLinkerPluginConfig())
-      PluginVectHavingLinkerConfigs.push_back(P);
+    if (P->hasRegisteredRelocations())
+      PluginVectHavingRelocCallbacks.push_back(P);
   }
-  return PluginVectHavingLinkerConfigs;
+  return PluginVectHavingRelocCallbacks;
 }
 
 bool ObjectLinker::scanRelocations(bool IsPartialLink) {
-  LinkerScript::PluginVectorT PluginVect = getLinkerPluginWithLinkerConfigs();
+  LinkerScript::PluginVectorT PluginVect = getLinkerPluginsWithRelocCallbacks();
 
   getTargetBackend().provideSymbols();
+  Relocator *Reloc = getTargetBackend().getRelocator();
+
+  auto &Inputs = ThisModule->getObjectList();
+  const bool RunParallel = Reloc->shouldScanRelocations() && !IsPartialLink &&
+                           ThisConfig.options().numThreads() > 1 &&
+                           ThisConfig.isScanRelocationsMultiThreaded();
+
+  // Snapshot the relocations that will receive callbacks. Relocations added by
+  // callbacks are scanned later but do not receive a callback in this pass.
+  llvm::SmallVector<llvm::SmallVector<Relocation *, 0>, 0> CallbackWork;
+  llvm::SmallVector<llvm::DenseMap<Relocation *, bool>, 0>
+      SkipRelocProcessingByInput(Inputs.size());
+  if (!PluginVect.empty()) {
+    CallbackWork.resize(Inputs.size());
+    for (size_t I = 0; I != Inputs.size(); ++I) {
+      auto *ObjFile = llvm::dyn_cast<ELFObjectFile>(Inputs[I]);
+      if (!ObjFile)
+        continue;
+      for (ELFSection *Rs : ObjFile->getRelocationSections()) {
+        if (Rs->isIgnore() || Rs->isDiscard())
+          continue;
+        for (Relocation *R : Rs->getLink()->getRelocations())
+          CallbackWork[I].push_back(R);
+      }
+    }
+  }
+
+  auto RunForEachInput = [&](auto Fn) {
+    if (!RunParallel) {
+      for (size_t I = 0; I != Inputs.size(); ++I)
+        Fn(I);
+      return;
+    }
+    std::atomic<size_t> NextInput{0};
+    size_t NumWorkers =
+        std::min<size_t>(ThisConfig.options().numThreads(), Inputs.size());
+    llvm::parallelFor(0, NumWorkers, [&](size_t) {
+      for (size_t I; (I = NextInput.fetch_add(1, std::memory_order_relaxed)) <
+                     Inputs.size();)
+        Fn(I);
+    });
+  };
 
-  // Slots are allocated as relocations are scanned in input order.
-  std::vector<std::unique_ptr<Relocator::CopyRelocs>> AllCopyRelocs;
-  for (auto &Input : ThisModule->getObjectList()) {
-    auto CopyRelocs = std::make_unique<Relocator::CopyRelocs>();
-    scanRelocationsHelper(Input, IsPartialLink, PluginVect, *CopyRelocs);
-    AllCopyRelocs.push_back(std::move(CopyRelocs));
+  auto RecordSkipDecisionsForInput = [&](size_t I) {
+    // Preserve the skip result from before callbacks can change symbol or
+    // section state.
+    for (Relocation *R : CallbackWork[I]) {
+      SkipRelocProcessingByInput[I][R] =
+          getTargetBackend().maySkipRelocProcessing(R);
+    }
+  };
+
+  auto RunCallbacksForInput = [&](size_t I) {
+    for (Relocation *R : CallbackWork[I]) {
+      if (SkipRelocProcessingByInput[I].lookup(R))
+        continue;
+      for (Plugin *P : PluginVect)
+        P->callReloc(R->type(), R);
+    }
+  };
+
+  // Callbacks may change state read by backend scanning, so finish all of them
+  // before scanning begins.
+  if (!PluginVect.empty()) {
+    RunForEachInput(RecordSkipDecisionsForInput);
+    RunForEachInput(RunCallbacksForInput);
+  }
+
+  Relocator::CopyRelocs CopyRelocSet;
+  if (!RunParallel) {
+    if (ThisModule->getPrinter()->traceThreads())
+      ThisConfig.raise(Diag::threads_disabled) << "ScanRelocations";
+    for (size_t I = 0; I != Inputs.size(); ++I)
+      scanRelocationsHelper(Inputs[I], I, IsPartialLink, false, CopyRelocSet,
+                            SkipRelocProcessingByInput[I]);
+  } else {
+    if (ThisModule->getPrinter()->traceThreads())
+      ThisConfig.raise(Diag::threads_enabled)
+          << "ScanRelocations" << ThisConfig.options().numThreads();
+    Reloc->initScanRelocations();
+    std::atomic<size_t> NextInput{0};
+    size_t NumWorkers =
+        std::min<size_t>(ThisConfig.options().numThreads(), Inputs.size());
+    llvm::parallelFor(0, NumWorkers, [&](size_t) {
+      for (size_t I; (I = NextInput.fetch_add(1, std::memory_order_relaxed)) <
+                     Inputs.size();)
+        scanRelocationsHelper(Inputs[I], I, IsPartialLink, true, CopyRelocSet,
+                              SkipRelocProcessingByInput[I]);
+    });
+  }
+
+  if (RunParallel) {
+    Reloc->replayScanRelocations(CopyRelocSet);
   }
+
   // assume there is only one copy relocation type per target
   Relocation::Type CopyRelocType = getTargetBackend().getCopyRelType();
-  for (const auto &RelocVec : AllCopyRelocs)
-    for (auto &Reloc : *RelocVec)
-      createCopyRelocation(*Reloc, CopyRelocType);
+  for (ResolveInfo *Sym : CopyRelocSet)
+    createCopyRelocation(*Sym, CopyRelocType);
 
   // If there is a undefined symbol, fail the link. No point fixing the
   // relocations. This is overridden by --noinhibit-exec.