[LLD][ThinLTO] Add --thinlto-single-module to allow compiling partial modules.

This change introduces an LLD switch --thinlto-single-module to allow compiling only a part of the input modules. This is specifically enables:

  1. Fast investigating/debugging modules of interest without spending time on compiling unrelated modules.
  2. Compiler debug dump with -mllvm -debug-only= for specific modules.

It will be useful for large applications which has 1K+ input modules for thinLTO.

The switch can be combined with `--lto-obj-path=` or `--lto-emit-asm` to obtain intermediate object files or assembly files. So far the module name matching is implemented as a fuzzy name lookup where the modules with name containing the switch value are compiled.

E.g,
Command:
     ld.lld main.o thin.a --thinlto-single-module=thin.a --lto-obj-path=single.o
log:
     [ThinLTO] Selecting thin.a(thin1.o at 168) to compile
     [ThinLTO] Selecting thin.a(thin2.o at 228) to compile
Command:
     ld.lld main.o thin.a --thinlto-single-module=thin1.o --lto-obj-path=single.o
log:
     [ThinLTO] Selecting thin.a(thin1.o at 168) to compile

Differential Revision: https://reviews.llvm.org/D80406

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index cc4547b..cfdd1eb 100644 (file)
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -125,6 +125,7 @@ struct Configuration {
   std::vector<llvm::StringRef> filterList;
   std::vector<llvm::StringRef> searchPaths;
   std::vector<llvm::StringRef> symbolOrderingFile;
+  std::vector<llvm::StringRef> thinLTOModulesToCompile;
   std::vector<llvm::StringRef> undefined;
   std::vector<SymbolVersion> dynamicList;
   std::vector<uint8_t> buildIdVector;

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index bcef615..78a6014 100644 (file)
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1002,6 +1002,8 @@ static void readConfigs(opt::InputArgList &args) {
       getOldNewOptions(args, OPT_thinlto_object_suffix_replace_eq);
   config->thinLTOPrefixReplace =
       getOldNewOptions(args, OPT_thinlto_prefix_replace_eq);
+  config->thinLTOModulesToCompile =
+      args::getStrings(args, OPT_thinlto_single_module_eq);
   config->timeTraceEnabled = args.hasArg(OPT_time_trace);
   config->timeTraceGranularity =
       args::getInteger(args, OPT_time_trace_granularity, 500);
@@ -1973,7 +1975,10 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // Likewise, --plugin-opt=emit-llvm and --plugin-opt=emit-asm are the
   // options to create output files in bitcode or assembly code
   // repsectively. No object files are generated.
-  if (config->thinLTOIndexOnly || config->emitLLVM || config->ltoEmitAsm)
+  // Also bail out here when only certain thinLTO modules are specified for
+  // compilation. The intermediate object file are the expected output.
+  if (config->thinLTOIndexOnly || config->emitLLVM || config->ltoEmitAsm ||
+      !config->thinLTOModulesToCompile.empty())
     return;
 
   // Apply symbol renames for -wrap.

diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 1f1c217..b8041af 100644 (file)
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -140,6 +140,9 @@ static lto::Config createConfig() {
   c.HasWholeProgramVisibility = config->ltoWholeProgramVisibility;
   c.AlwaysEmitRegularLTOObj = !config->ltoObjPath.empty();
 
+  for (const llvm::StringRef &name : config->thinLTOModulesToCompile)
+    c.ThinLTOModulesToCompile.emplace_back(name);
+
   c.TimeTraceEnabled = config->timeTraceEnabled;
   c.TimeTraceGranularity = config->timeTraceGranularity;
 
@@ -296,12 +299,14 @@ std::vector<InputFile *> BitcodeCompiler::compile() {
         },
         cache));
 
-  // Emit empty index files for non-indexed files
-  for (StringRef s : thinIndices) {
-    std::string path = getThinLTOOutputFile(s);
-    openFile(path + ".thinlto.bc");
-    if (config->thinLTOEmitImportsFiles)
-      openFile(path + ".imports");
+  // Emit empty index files for non-indexed files but not in single-module mode.
+  if (config->thinLTOModulesToCompile.empty()) {
+    for (StringRef s : thinIndices) {
+      std::string path = getThinLTOOutputFile(s);
+      openFile(path + ".thinlto.bc");
+      if (config->thinLTOEmitImportsFiles)
+        openFile(path + ".imports");
+    }
   }
 
   if (config->thinLTOIndexOnly) {

diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 939af8f..0a16faa 100644 (file)
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -562,6 +562,8 @@ def thinlto_jobs: JJ<"thinlto-jobs=">,
   HelpText<"Number of ThinLTO jobs. Default to --threads=">;
 def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
 def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
+def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
+  HelpText<"Specific a single module to compile in ThinLTO mode, for debugging only">;
 
 def: J<"plugin-opt=O">, Alias<lto_O>, HelpText<"Alias for --lto-O">;
 def: F<"plugin-opt=debug-pass-manager">,

diff --git a/lld/test/ELF/lto/thinlto-single-module.ll b/lld/test/ELF/lto/thinlto-single-module.ll
new file mode 100644 (file)
index 0000000..0530213
--- /dev/null
+++ b/lld/test/ELF/lto/thinlto-single-module.ll
@@ -0,0 +1,69 @@
+; REQUIRES: x86
+; RUN: rm -fr %t && mkdir %t && cd %t
+; RUN: opt -thinlto-bc -o main.o %s
+; RUN: opt -thinlto-bc -o thin1.o %S/Inputs/thin1.ll
+; RUN: opt -thinlto-bc -o thin2.o %S/Inputs/thin2.ll
+; RUN: llvm-ar qcT thin.a thin1.o thin2.o
+
+;; --thinlto-single-module=main.o should result in only main.o compiled, of which
+;; the object code is saved in single1.o1. Note that single1.o is always the dummy
+;; output, aka ld-temp.o. There should be no more object files generated.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --lto-obj-path=single1.o
+; RUN: llvm-readelf -S -s single1.o | FileCheck %s --check-prefix=DEFAULT
+; RUN: llvm-readelf -S -s single1.o1 | FileCheck %s --check-prefix=MAIN
+; RUN: not ls single1.o2
+; RUN: not ls a.out
+
+; DEFAULT:       Value        Size Type Bind   Vis     Ndx Name
+; DEFAULT:   0000000000000000    0 FILE LOCAL  DEFAULT ABS ld-temp.o
+; MAIN:          Value        Size Type Bind   Vis     Ndx Name
+; MAIN:      0000000000000000    0 FILE LOCAL  DEFAULT ABS thinlto-single-module.ll
+; MAIN-NEXT: 0000000000000000    3 FUNC GLOBAL DEFAULT   3 _start
+
+;; --thinlto-single-module=thin.a should result in only thin1.o and thin2.o compiled.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=thin.a --lto-obj-path=single2.o
+; RUN: llvm-readelf -S -s single2.o | FileCheck %s --check-prefix=DEFAULT
+; RUN: llvm-readelf -S -s single2.o1 | FileCheck %s --check-prefix=FOO
+; RUN: llvm-readelf -S -s single2.o2 | FileCheck %s --check-prefix=BLAH
+; RUN: not ls single1.o3
+
+;; Multiple --thinlto-single-module uses should result in a combination of inputs compiled.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --thinlto-single-module=thin2.o --lto-obj-path=single4.o
+; RUN: llvm-readelf -S -s single4.o | FileCheck %s --check-prefix=DEFAULT
+; RUN: llvm-readelf -S -s single4.o1 | FileCheck %s --check-prefix=MAIN
+; RUN: llvm-readelf -S -s single4.o2 | FileCheck %s --check-prefix=BLAH
+; RUN: not ls single4.o3
+
+; FOO:           Value        Size Type Bind   Vis     Ndx Name
+; FOO:       0000000000000000    0 FILE LOCAL  DEFAULT ABS thin1.ll
+; FOO-NEXT:  0000000000000000    6 FUNC GLOBAL DEFAULT   3 foo
+; BLAH:          Value        Size Type Bind   Vis     Ndx Name
+; BLAH:      0000000000000000    0 FILE LOCAL  DEFAULT ABS thin2.ll
+; BLAH-NEXT: 0000000000000000    4 FUNC GLOBAL DEFAULT   3 blah
+
+;; Check only main.o is in the result thin index file.
+;; Also check a *.thinlto.bc file generated for main.o only.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --thinlto-index-only=single5.idx
+; RUN: ls main.o.thinlto.bc
+; RUN: ls | FileCheck --implicit-check-not='thin.{{.*}}.thinlto.bc' /dev/null
+; RUN: FileCheck %s --check-prefix=IDX < single5.idx
+; RUN: count 1 < single5.idx
+
+; IDX: main.o
+
+;; Check temporary output generated for main.o only.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --save-temps
+; RUN: ls main.o.0.preopt.bc
+; RUN: not ls thin.*.0.preopt.bc
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-scei-ps4"
+
+declare i32 @blah(i32 %meh)
+declare i32 @foo(i32 %goo)
+
+define i32 @_start() {
+  call i32 @foo(i32 0)
+  call i32 @blah(i32 0)
+  ret i32 0
+}

diff --git a/llvm/include/llvm/LTO/Config.h b/llvm/include/llvm/LTO/Config.h
index 20fed34..0a3e523 100644 (file)
--- a/llvm/include/llvm/LTO/Config.h
+++ b/llvm/include/llvm/LTO/Config.h
@@ -130,6 +130,9 @@ struct Config {
   /// Statistics output file path.
   std::string StatsFile;
 
+  /// Specific thinLTO modules to compile.
+  std::vector<std::string> ThinLTOModulesToCompile;
+
   /// Time trace enabled.
   bool TimeTraceEnabled = false;
 

diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index c25aa07..93456c0 100644 (file)
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -17,6 +17,7 @@
 
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/LTO/Config.h"
 #include "llvm/Object/IRSymtab.h"
@@ -26,7 +27,6 @@
 
 namespace llvm {
 
-class BitcodeModule;
 class Error;
 class IRMover;
 class LLVMContext;
@@ -330,12 +330,17 @@ private:
     bool EmptyCombinedModule = true;
   } RegularLTO;
 
+  using ModuleMapType = MapVector<StringRef, BitcodeModule>;
+
   struct ThinLTOState {
     ThinLTOState(ThinBackend Backend);
 
     ThinBackend Backend;
     ModuleSummaryIndex CombinedIndex;
-    MapVector<StringRef, BitcodeModule> ModuleMap;
+    // The full set of bitcode modules in input order.
+    ModuleMapType ModuleMap;
+    // The bitcode modules to compile, if specified by the LTO Config.
+    Optional<ModuleMapType> ModulesToCompile;
     DenseMap<GlobalValue::GUID, StringRef> PrevailingModuleForGUID;
   } ThinLTO;
 

diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index a6c61c4..f4cf0d2 100644 (file)
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -870,12 +870,28 @@ Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
         "Expected at most one ThinLTO module per bitcode file",
         inconvertibleErrorCode());
 
+  if (!Conf.ThinLTOModulesToCompile.empty()) {
+    if (!ThinLTO.ModulesToCompile)
+      ThinLTO.ModulesToCompile = ModuleMapType();
+    // This is a fuzzy name matching where only modules with name containing the
+    // specified switch values are going to be compiled.
+    for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
+      if (BM.getModuleIdentifier().contains(Name)) {
+        ThinLTO.ModulesToCompile->insert({BM.getModuleIdentifier(), BM});
+        llvm::errs() << "[ThinLTO] Selecting " << BM.getModuleIdentifier()
+                     << " to compile\n";
+      }
+    }
+  }
+
   return Error::success();
 }
 
 unsigned LTO::getMaxTasks() const {
   CalledGetMaxTasks = true;
-  return RegularLTO.ParallelCodeGenParallelismLevel + ThinLTO.ModuleMap.size();
+  auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
+                                              : ThinLTO.ModuleMap.size();
+  return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
 }
 
 // If only some of the modules were split, we cannot correctly handle
@@ -1312,6 +1328,11 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
   if (ThinLTO.ModuleMap.empty())
     return Error::success();
 
+  if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
+    llvm::errs() << "warning: [ThinLTO] No module compiled\n";
+    return Error::success();
+  }
+
   if (Conf.CombinedIndexHook &&
       !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
     return Error::success();
@@ -1416,10 +1437,13 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
       ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
                       AddStream, Cache);
 
+  auto &ModuleMap =
+      ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
+
   // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for combined
   // module and parallel code generation partitions.
   unsigned Task = RegularLTO.ParallelCodeGenParallelismLevel;
-  for (auto &Mod : ThinLTO.ModuleMap) {
+  for (auto &Mod : ModuleMap) {
     if (Error E = BackendProc->start(Task, Mod.second, ImportLists[Mod.first],
                                      ExportLists[Mod.first],
                                      ResolvedODR[Mod.first], ThinLTO.ModuleMap))