LLVM and SPIRV-LLVM-Translator pulldown (WW08 2026)

.gitignore

@@ -20,8 +20,8 @@
 # vim swap files
 .*.sw?
 .sw?
-#OS X specific files.
-.DS_store
+#macOS specific
+.DS_Store
 
 # Ignore the user specified CMake presets in subproject directories.
 /*/CMakeUserPresets.json
@@ -51,6 +51,7 @@ cscope.out
 autoconf/aclocal.m4
 autoconf/autom4te.cache
 /compile_commands.json
+/tablegen_compile_commands.yml
 # Visual Studio built-in CMake configuration
 /CMakeSettings.json
 # CLion project configuration

.mailmap

@@ -32,7 +32,7 @@
 Min Hsu <min.hsu@sifive.com> <min@myhsu.dev>
 Min Hsu <min.hsu@sifive.com> <minyihh@uci.edu>
 <qcf@ecnelises.com> <qiucofan@cn.ibm.com> <qiucf@cn.ibm.com>
-<rnk@google.com> <reid@kleckner.net>
+<rnk@llvm.org> <rnk@nvidia.com> <rnk@google.com> <reid@kleckner.net>
 <thakis@chromium.org> <nicolasweber@gmx.de>
 Jianjian GUAN <jacquesguan@me.com>
 Jianjian GUAN <jacquesguan@me.com> <Jianjian.Guan@streamcomputing.com>

bolt/docs/CommandLineArgumentReference.md

@@ -724,7 +724,7 @@
 
 - `--plt=<value>`
 
-  Optimize PLT calls (requires linking with -znow)
+  Optimize PLT calls (requires linking with -znow on non-x86 architectures)
   - `none`: do not optimize PLT calls
   - `hot`: optimize executed (hot) PLT calls
   - `all`: optimize all PLT calls

bolt/include/bolt/Core/BinaryContext.h

@@ -234,9 +234,6 @@ class BinaryContext {
   /// Functions to be considered for the output in a sorted order.
   BinaryFunctionListType OutputFunctions;
 
-  /// A mutex that is used to control parallel accesses to BinaryFunctions.
-  mutable llvm::sys::RWMutex BinaryFunctionsMutex;
-
   /// Functions injected by BOLT.
   BinaryFunctionListType InjectedBinaryFunctions;
 

bolt/include/bolt/Core/BinaryFunction.h

@@ -771,6 +771,9 @@ class BinaryFunction {
   /// disassembled state was later invalidated.
   void clearDisasmState();
 
+  /// Reset the function state into Empty state, i.e. pre-disassembly form.
+  void resetState();
+
   /// Release memory allocated for CFG and instructions.
   /// We still keep basic blocks for address translation/mapping purposes.
   void releaseCFG() {

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -1783,6 +1783,10 @@ class MCPlusBuilder {
     return 0;
   }
 
+  virtual void patchPLTEntryForBTI(BinaryFunction &PLTFunction, MCInst &Call) {
+    llvm_unreachable("not implemented");
+  }
+
   virtual bool analyzeVirtualMethodCall(InstructionIterator Begin,
                                         InstructionIterator End,
                                         std::vector<MCInst *> &MethodFetchInsns,

bolt/include/bolt/Profile/DataAggregator.h

@@ -19,6 +19,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Program.h"
+#include <limits>
 #include <unordered_map>
 
 namespace llvm {
@@ -108,10 +109,14 @@ class DataAggregator : public DataReader {
   ///   (FT_ONLY, FT_EXTERNAL_ORIGIN, or FT_EXTERNAL_RETURN).
   struct Trace {
     static constexpr const uint64_t EXTERNAL = 0ULL;
-    static constexpr const uint64_t BR_ONLY = -1ULL;
-    static constexpr const uint64_t FT_ONLY = -1ULL;
-    static constexpr const uint64_t FT_EXTERNAL_ORIGIN = -2ULL;
-    static constexpr const uint64_t FT_EXTERNAL_RETURN = -3ULL;
+    static constexpr const uint64_t BR_ONLY =
+        std::numeric_limits<uint64_t>::max();
+    static constexpr const uint64_t FT_ONLY =
+        std::numeric_limits<uint64_t>::max();
+    static constexpr const uint64_t FT_EXTERNAL_ORIGIN =
+        std::numeric_limits<uint64_t>::max() - 1;
+    static constexpr const uint64_t FT_EXTERNAL_RETURN =
+        std::numeric_limits<uint64_t>::max() - 2;
 
     uint64_t Branch;
     uint64_t From;

bolt/include/bolt/Rewrite/RewriteInstance.h

@@ -230,6 +230,10 @@ class RewriteInstance {
   /// disassembleFunctions(), also preserve the original version.
   void rewriteFile();
 
+  /// Rewrite functions in place by overwriting their original locations.
+  /// Used by non-relocation mode and for patched functions.
+  void rewriteFunctionsInPlace(raw_fd_ostream &OS);
+
   /// Return address of a function in the new binary corresponding to
   /// \p OldAddress address in the original binary.
   uint64_t getNewFunctionAddress(uint64_t OldAddress);

bolt/lib/Core/BinaryContext.cpp

@@ -1556,38 +1556,25 @@ void BinaryContext::foldFunction(BinaryFunction &ChildBF,
   ChildBF.Aliases.clear();
 
   if (HasRelocations) {
-    // Merge execution counts of ChildBF into those of ParentBF.
-    // Without relocations, we cannot reliably merge profiles as both functions
-    // continue to exist and either one can be executed.
+    // Merge execution counts of ChildBF into those of ParentBF. We require
+    // relocations as without relocations we cannot reliably merge profiles as
+    // both functions continue to exist and either one can be executed.
     ChildBF.mergeProfileDataInto(ParentBF);
 
-    std::shared_lock<llvm::sys::RWMutex> ReadBfsLock(BinaryFunctionsMutex,
-                                                     std::defer_lock);
-    std::unique_lock<llvm::sys::RWMutex> WriteBfsLock(BinaryFunctionsMutex,
-                                                      std::defer_lock);
-    // Remove ChildBF from the global set of functions in relocs mode.
-    ReadBfsLock.lock();
-    auto FI = BinaryFunctions.find(ChildBF.getAddress());
-    ReadBfsLock.unlock();
-
-    assert(FI != BinaryFunctions.end() && "function not found");
-    assert(&ChildBF == &FI->second && "function mismatch");
-
-    WriteBfsLock.lock();
-    ChildBF.clearDisasmState();
-    FI = BinaryFunctions.erase(FI);
-    WriteBfsLock.unlock();
-
-  } else {
-    // In non-relocation mode we keep the function, but rename it.
-    std::string NewName = "__ICF_" + ChildName.str();
+    // Clear CFG state to free memory, but keep function in map.
+    // The function is marked as folded and will not be emitted.
+    ChildBF.resetState();
+  }
 
-    WriteCtxLock.lock();
-    ChildBF.getSymbols().push_back(Ctx->getOrCreateSymbol(NewName));
-    WriteCtxLock.unlock();
+  // Add a new symbol to the function. In relocation mode, this is a
+  // placeholder so that getSymbol() doesn't crash. In non-relocation mode,
+  // this effectively renames the function.
+  WriteCtxLock.lock();
+  ChildBF.getSymbols().push_back(
+      Ctx->getOrCreateSymbol("__ICF_" + ChildName.str()));
+  WriteCtxLock.unlock();
 
-    ChildBF.setFolded(&ParentBF);
-  }
+  ChildBF.setFolded(&ParentBF);
 
   ParentBF.setHasFunctionsFoldedInto();
 }
@@ -1967,6 +1954,12 @@ bool BinaryContext::shouldEmit(const BinaryFunction &Function) const {
   if (Function.isPseudo())
     return false;
 
+  // In relocation mode, folded functions should not be emitted - their code
+  // is part of the parent. In non-relocation mode, folded functions are still
+  // emitted at their original location.
+  if (HasRelocations && Function.isFolded())
+    return false;
+
   if (opts::processAllFunctions())
     return true;
 

bolt/lib/Core/BinaryFunction.cpp

@@ -147,6 +147,11 @@ static cl::opt<bool> TrapOnAVX512(
     cl::init(false), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltCategory));
 
 bool shouldPrint(const BinaryFunction &Function) {
+  // PLT stubs are disassembled for BTI binaries, therefore they should be
+  // printed.
+  if (Function.getBinaryContext().usesBTI() && Function.isPLTFunction())
+    return true;
+
   if (Function.isIgnored())
     return false;
 
@@ -3251,6 +3256,30 @@ void BinaryFunction::clearDisasmState() {
   clearList(TakenBranches);
 }
 
+void BinaryFunction::resetState() {
+  clearDisasmState();
+
+  // Clear CFG state too.
+  if (hasCFG()) {
+    releaseCFG();
+
+    for (BinaryBasicBlock *BB : BasicBlocks)
+      delete BB;
+    clearList(BasicBlocks);
+
+    for (BinaryBasicBlock *BB : DeletedBasicBlocks)
+      delete BB;
+    clearList(DeletedBasicBlocks);
+
+    Layout.clear();
+  }
+
+  IsSimple = false;
+  IsIgnored = true;
+
+  CurrentState = State::Empty;
+}
+
 void BinaryFunction::setTrapOnEntry() {
   clearDisasmState();
 
@@ -3285,24 +3314,7 @@ void BinaryFunction::setIgnored() {
   if (CurrentState == State::Empty)
     return;
 
-  clearDisasmState();
-
-  // Clear CFG state too.
-  if (hasCFG()) {
-    releaseCFG();
-
-    for (BinaryBasicBlock *BB : BasicBlocks)
-      delete BB;
-    clearList(BasicBlocks);
-
-    for (BinaryBasicBlock *BB : DeletedBasicBlocks)
-      delete BB;
-    clearList(DeletedBasicBlocks);
-
-    Layout.clear();
-  }
-
-  CurrentState = State::Empty;
+  resetState();
 
   // Fix external references in the original function body.
   if (BC.HasRelocations) {

bolt/lib/Core/DebugNames.cpp

@@ -677,11 +677,11 @@ static constexpr uint32_t getDebugNamesHeaderSize() {
   constexpr uint32_t BucketCountLength = sizeof(uint32_t);
   constexpr uint32_t NameCountLength = sizeof(uint32_t);
   constexpr uint32_t AbbrevTableSizeLength = sizeof(uint32_t);
-  constexpr uint32_t AugmentationStringSizeLenght = sizeof(uint32_t);
+  constexpr uint32_t AugmentationStringSizeLength = sizeof(uint32_t);
   return VersionLength + PaddingLength + CompUnitCountLength +
          LocalTypeUnitCountLength + ForeignTypeUnitCountLength +
          BucketCountLength + NameCountLength + AbbrevTableSizeLength +
-         AugmentationStringSizeLenght;
+         AugmentationStringSizeLength;
 }
 
 void DWARF5AcceleratorTable::emitHeader() const {

bolt/lib/Passes/IdenticalCodeFolding.cpp

@@ -33,6 +33,8 @@ namespace opts {
 
 extern cl::OptionCategory BoltOptCategory;
 
+extern bool isHotTextMover(const BinaryFunction &Function);
+
 static cl::opt<bool>
     ICFUseDFS("icf-dfs", cl::desc("use DFS ordering when using -icf option"),
               cl::ReallyHidden, cl::cat(BoltOptCategory));
@@ -186,6 +188,11 @@ static bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
                             bool CongruentSymbols) {
   assert(A.hasCFG() && B.hasCFG() && "both functions should have CFG");
 
+  // Hot text mover functions should not be folded. They need to stay in their
+  // original section to avoid being placed on hot/huge pages.
+  if (opts::isHotTextMover(A) || opts::isHotTextMover(B))
+    return false;
+
   // Compare the two functions, one basic block at a time.
   // Currently we require two identical basic blocks to have identical
   // instruction sequences and the same index in their corresponding

bolt/lib/Passes/LongJmp.cpp

@@ -499,6 +499,11 @@ Error LongJmpPass::relaxStub(BinaryBasicBlock &StubBB, bool &Modified) {
                 << RealTargetSym->getName() << "\n";
       exit(1);
     }
+    if (TargetFunction && TargetFunction->isPLTFunction()) {
+      BC.MIB->patchPLTEntryForBTI(*TargetFunction,
+                                  *StubBB.getLastNonPseudoInstr());
+      return;
+    }
     if (TargetFunction && TargetFunction->isIgnored()) {
       // Includes PLT functions.
       BC.errs() << "BOLT-ERROR: Cannot add BTI landing pad to ignored function "

bolt/lib/Passes/PLTCall.cpp

@@ -23,7 +23,9 @@ namespace opts {
 extern cl::OptionCategory BoltOptCategory;
 
 static cl::opt<bolt::PLTCall::OptType>
-    PLT("plt", cl::desc("optimize PLT calls (requires linking with -znow)"),
+    PLT("plt",
+        cl::desc("optimize PLT calls (requires linking with -znow on "
+                 "non-x86 architectures)"),
         cl::init(bolt::PLTCall::OT_NONE),
         cl::values(clEnumValN(bolt::PLTCall::OT_NONE, "none",
                               "do not optimize PLT calls"),
@@ -76,7 +78,12 @@ Error PLTCall::runOnFunctions(BinaryContext &BC) {
   }
 
   if (NumCallsOptimized) {
-    BC.RequiresZNow = true;
+    // On X86-64, PLT optimization does not require -znow because the indirect
+    // call through GOT works correctly with lazy binding. At runtime, the
+    // resolver will populate the GOT entry on first call just like with a
+    // regular PLT call.
+    if (!BC.isX86())
+      BC.RequiresZNow = true;
     BC.outs() << "BOLT-INFO: " << NumCallsOptimized
               << " PLT calls in the binary were optimized.\n";
   }

bolt/lib/Passes/PatchEntries.cpp

@@ -36,14 +36,20 @@ Error PatchEntries::runOnFunctions(BinaryContext &BC) {
   if (!opts::ForcePatch) {
     // Mark the binary for patching if we did not create external references
     // for original code in any of functions we are not going to emit.
-    bool NeedsPatching = llvm::any_of(
-        llvm::make_second_range(BC.getBinaryFunctions()),
-        [&](BinaryFunction &BF) {
-          return (!BC.shouldEmit(BF) && !BF.hasExternalRefRelocations()) ||
-                 BF.needsPatch();
-        });
-
-    if (!NeedsPatching)
+    auto needsPatching = [&](const BinaryFunction &BF) {
+      // FIXME: keep compatibility for NFC testing.
+      if (BF.isFolded())
+        return false;
+
+      // Patching is always needed if explicitly requested.
+      if (BF.needsPatch())
+        return true;
+
+      return !BC.shouldEmit(BF) && !BF.hasExternalRefRelocations();
+    };
+
+    if (!llvm::any_of(llvm::make_second_range(BC.getBinaryFunctions()),
+                      needsPatching))
       return Error::success();
   }
 

bolt/lib/Profile/DataAggregator.cpp

@@ -1729,10 +1729,10 @@ std::error_code DataAggregator::parseMemEvents() {
     if (std::error_code EC = Sample.getError())
       return EC;
 
-    if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Sample->PC))
+    if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Sample->PC)) {
       BF->setHasProfileAvailable();
-
-    MemSamples.emplace_back(std::move(Sample.get()));
+      MemSamples.emplace_back(std::move(Sample.get()));
+    }
   }
 
   return std::error_code();