[clang][deps] Make clang-scan-deps write modules in raw format

We have no use for debug info for the scanner modules, and writing raw
ast files speeds up scanning ~15% in some cases. Note that the compile
commands produced by the scanner will still build the obj format (if
requested), and the scanner can *read* obj format pcms, e.g. from a PCH.

rdar://108807592

Differential Revision: https://reviews.llvm.org/D149693

diff --git a/clang-tools-extra/clangd/Compiler.cpp b/clang-tools-extra/clangd/Compiler.cpp
index f242db44b43178c62e669ca99233afbf233125b8..7b93353cab1f3cdf98db64198579fff7b7ed4802 100644 (file)
--- a/clang-tools-extra/clangd/Compiler.cpp
+++ b/clang-tools-extra/clangd/Compiler.cpp
@@ -73,7 +73,7 @@ void disableUnsupportedOptions(CompilerInvocation &CI) {
   // Always default to raw container format as clangd doesn't registry any other
   // and clang dies when faced with unknown formats.
   CI.getHeaderSearchOpts().ModuleFormat =
-      PCHContainerOperations().getRawReader().getFormat().str();
+      PCHContainerOperations().getRawReader().getFormats().front().str();
 
   CI.getFrontendOpts().Plugins.clear();
   CI.getFrontendOpts().AddPluginActions.clear();

diff --git a/clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h b/clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h
index c13e052149d95875da1b549b400bacc2ad38e6e0..7a02d8725885a48680be16f4c9d0cb04a925b8ae 100644 (file)
--- a/clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h
+++ b/clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h
@@ -32,7 +32,7 @@ class ObjectFilePCHContainerWriter : public PCHContainerWriter {
 /// A PCHContainerReader implementation that uses LLVM to
 /// wraps Clang modules inside a COFF, ELF, or Mach-O container.
 class ObjectFilePCHContainerReader : public PCHContainerReader {
-  StringRef getFormat() const override { return "obj"; }
+  ArrayRef<StringRef> getFormats() const override;
 
   /// Returns the serialized AST inside the PCH container Buffer.
   StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const override;

diff --git a/clang/include/clang/Serialization/PCHContainerOperations.h b/clang/include/clang/Serialization/PCHContainerOperations.h
index 9f9700a418a9c9165270f1846dd5fc3e104d4119..be10feb5e351c3938e2d183135e3e7fdc30b84d5 100644 (file)
--- a/clang/include/clang/Serialization/PCHContainerOperations.h
+++ b/clang/include/clang/Serialization/PCHContainerOperations.h
@@ -56,7 +56,7 @@ class PCHContainerReader {
 public:
   virtual ~PCHContainerReader() = 0;
   /// Equivalent to the format passed to -fmodule-format=
-  virtual llvm::StringRef getFormat() const = 0;
+  virtual llvm::ArrayRef<llvm::StringRef> getFormats() const = 0;
 
   /// Returns the serialized AST inside the PCH container Buffer.
   virtual llvm::StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const = 0;
@@ -78,8 +78,7 @@ class RawPCHContainerWriter : public PCHContainerWriter {
 
 /// Implements read operations for a raw pass-through PCH container.
 class RawPCHContainerReader : public PCHContainerReader {
-  llvm::StringRef getFormat() const override { return "raw"; }
-
+  llvm::ArrayRef<llvm::StringRef> getFormats() const override;
   /// Simply returns the buffer contained in Buffer.
   llvm::StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const override;
 };
@@ -87,7 +86,9 @@ class RawPCHContainerReader : public PCHContainerReader {
 /// A registry of PCHContainerWriter and -Reader objects for different formats.
 class PCHContainerOperations {
   llvm::StringMap<std::unique_ptr<PCHContainerWriter>> Writers;
-  llvm::StringMap<std::unique_ptr<PCHContainerReader>> Readers;
+  llvm::StringMap<PCHContainerReader *> Readers;
+  llvm::SmallVector<std::unique_ptr<PCHContainerReader>> OwnedReaders;
+
 public:
   /// Automatically registers a RawPCHContainerWriter and
   /// RawPCHContainerReader.
@@ -96,13 +97,17 @@ public:
     Writers[Writer->getFormat()] = std::move(Writer);
   }
   void registerReader(std::unique_ptr<PCHContainerReader> Reader) {
-    Readers[Reader->getFormat()] = std::move(Reader);
+    assert(!Reader->getFormats().empty() &&
+           "PCHContainerReader must handle >=1 format");
+    for (llvm::StringRef Fmt : Reader->getFormats())
+      Readers[Fmt] = Reader.get();
+    OwnedReaders.push_back(std::move(Reader));
   }
   const PCHContainerWriter *getWriterOrNull(llvm::StringRef Format) {
     return Writers[Format].get();
   }
   const PCHContainerReader *getReaderOrNull(llvm::StringRef Format) {
-    return Readers[Format].get();
+    return Readers[Format];
   }
   const PCHContainerReader &getRawReader() {
     return *getReaderOrNull("raw");

diff --git a/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp b/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
index 07db4a63e2c10d808c8ccfc0721bed62ef205757..114a9c1e2eac1213a564327f22f7779ffeaf30ec 100644 (file)
--- a/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
+++ b/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
@@ -349,6 +349,11 @@ ObjectFilePCHContainerWriter::CreatePCHContainerGenerator(
       CI, MainFileName, OutputFileName, std::move(OS), Buffer);
 }
 
+ArrayRef<StringRef> ObjectFilePCHContainerReader::getFormats() const {
+  static StringRef Formats[] = {"obj", "raw"};
+  return Formats;
+}
+
 StringRef
 ObjectFilePCHContainerReader::ExtractPCH(llvm::MemoryBufferRef Buffer) const {
   StringRef PCH;

diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index 3269c9d0f479f2cf316cb8d860f1f7dd8f8b152c..bda49fb68e4609f34b2d33a40292a05a0186291c 100644 (file)
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -811,7 +811,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
                                      UserFilesAreVolatile);
   AST->ModuleCache = new InMemoryModuleCache;
   AST->HSOpts = std::make_shared<HeaderSearchOptions>();
-  AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormat());
+  AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormats().front());
   AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts,
                                          AST->getSourceManager(),
                                          AST->getDiagnostics(),

diff --git a/clang/lib/Serialization/PCHContainerOperations.cpp b/clang/lib/Serialization/PCHContainerOperations.cpp
index d4990fce2d99d9f5308951cbe426741d6b6351b9..56ca3394385b4f6178508857e9ceff129f1cfede 100644 (file)
--- a/clang/lib/Serialization/PCHContainerOperations.cpp
+++ b/clang/lib/Serialization/PCHContainerOperations.cpp
@@ -57,6 +57,11 @@ std::unique_ptr<ASTConsumer> RawPCHContainerWriter::CreatePCHContainerGenerator(
   return std::make_unique<RawPCHContainerGenerator>(std::move(OS), Buffer);
 }
 
+ArrayRef<llvm::StringRef> RawPCHContainerReader::getFormats() const {
+  static StringRef Raw("raw");
+  return ArrayRef(Raw);
+}
+
 StringRef
 RawPCHContainerReader::ExtractPCH(llvm::MemoryBufferRef Buffer) const {
   return Buffer.getBuffer();

diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 937f39ba3bffa836bb8da1ea857dee89182ad949..baf76f01cfb54cbf0ef4596677369e7ff79d5dce 100644 (file)
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -181,6 +181,7 @@ public:
     ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
     ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
     ScanInstance.getFrontendOpts().ModulesShareFileManager = false;
+    ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw";
 
     ScanInstance.setFileManager(FileMgr);
     // Support for virtual file system overlays.
@@ -309,12 +310,11 @@ DependencyScanningWorker::DependencyScanningWorker(
     : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()),
       EagerLoadModules(Service.shouldEagerLoadModules()) {
   PCHContainerOps = std::make_shared<PCHContainerOperations>();
+  // We need to read object files from PCH built outside the scanner.
   PCHContainerOps->registerReader(
       std::make_unique<ObjectFilePCHContainerReader>());
-  // We don't need to write object files, but the current PCH implementation
-  // requires the writer to be registered as well.
-  PCHContainerOps->registerWriter(
-      std::make_unique<ObjectFilePCHContainerWriter>());
+  // The scanner itself writes only raw ast files.
+  PCHContainerOps->registerWriter(std::make_unique<RawPCHContainerWriter>());
 
   switch (Service.getMode()) {
   case ScanningMode::DependencyDirectivesScan:

diff --git a/clang/test/ClangScanDeps/module-format.c b/clang/test/ClangScanDeps/module-format.c
new file mode 100644 (file)
index 0000000..001a011
--- /dev/null
+++ b/clang/test/ClangScanDeps/module-format.c
@@ -0,0 +1,64 @@
+// Check that the scanner produces raw ast files, even when builds produce the
+// obj format, and that the scanner can read obj format from PCH and modules
+// imported by PCH.
+
+// Unsupported on AIX because we don't support the requisite "__clangast"
+// section in XCOFF yet.
+// UNSUPPORTED: target={{.*}}-aix{{.*}}
+
+// REQUIRES: shell
+
+// RUN: rm -rf %t && mkdir %t
+// RUN: cp %S/Inputs/modules-pch/* %t
+
+// Scan dependencies of the PCH:
+//
+// RUN: rm -f %t/cdb_pch.json
+// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb_pch.json
+// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json -format experimental-full \
+// RUN:   -module-files-dir %t/build > %t/result_pch.json
+
+// Explicitly build the PCH:
+//
+// RUN: %deps-to-rsp %t/result_pch.json --module-name=ModCommon1 > %t/mod_common_1.cc1.rsp
+// RUN: %deps-to-rsp %t/result_pch.json --module-name=ModCommon2 > %t/mod_common_2.cc1.rsp
+// RUN: %deps-to-rsp %t/result_pch.json --module-name=ModPCH > %t/mod_pch.cc1.rsp
+// RUN: %deps-to-rsp %t/result_pch.json --tu-index=0 > %t/pch.rsp
+//
+// RUN: %clang @%t/mod_common_1.cc1.rsp
+// RUN: %clang @%t/mod_common_2.cc1.rsp
+// RUN: %clang @%t/mod_pch.cc1.rsp
+// RUN: %clang @%t/pch.rsp
+
+// Scan dependencies of the TU:
+//
+// RUN: rm -f %t/cdb_tu.json
+// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb_tu.json
+// RUN: clang-scan-deps -compilation-database %t/cdb_tu.json -format experimental-full \
+// RUN:   -module-files-dir %t/build > %t/result_tu.json
+
+// Explicitly build the TU:
+//
+// RUN: %deps-to-rsp %t/result_tu.json --module-name=ModTU > %t/mod_tu.cc1.rsp
+// RUN: %deps-to-rsp %t/result_tu.json --tu-index=0 > %t/tu.rsp
+//
+// RUN: %clang @%t/mod_tu.cc1.rsp
+// RUN: %clang @%t/tu.rsp
+
+// Check the module format for scanner modules:
+//
+// RUN: find %t/cache -name "*.pcm" -exec %clang_cc1 -module-file-info "{}" ";" | FileCheck %s -check-prefix=SCAN
+// SCAN: Module format: raw
+// SCAN: Module format: raw
+// SCAN: Module format: raw
+// SCAN: Module format: raw
+
+// Check the module format for built modules:
+//
+// RUN: find %t/build -name "*.pcm" -exec %clang_cc1 -module-file-info "{}" ";" | FileCheck %s -check-prefix=BUILD
+// BUILD: Module format: obj
+// BUILD: Module format: obj
+// BUILD: Module format: obj
+// BUILD: Module format: obj
+
+// FIXME: check pch format as well; -module-file-info does not work with a PCH

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index bb536112e20945bad447f407a3fdad057df9097d..448681bfe533a4882cc6f92ef89b03c020df2d8c 100644 (file)
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -3964,7 +3964,7 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
       TUKind, CacheCodeCompletionResults, IncludeBriefCommentsInCodeCompletion,
       /*AllowPCHWithCompilerErrors=*/true, SkipFunctionBodies, SingleFileParse,
       /*UserFilesAreVolatile=*/true, ForSerialization, RetainExcludedCB,
-      CXXIdx->getPCHContainerOperations()->getRawReader().getFormat(),
+      CXXIdx->getPCHContainerOperations()->getRawReader().getFormats().front(),
       &ErrUnit));
 
   // Early failures in LoadFromCommandLine may return with ErrUnit unset.

diff --git a/clang/tools/libclang/Indexing.cpp b/clang/tools/libclang/Indexing.cpp
index 24ca98d4de416f9237a720fd461b3bada8ab2931..b6dfb1f47b5986847690c6a6e8e2bb522e87687b 100644 (file)
--- a/clang/tools/libclang/Indexing.cpp
+++ b/clang/tools/libclang/Indexing.cpp
@@ -552,7 +552,7 @@ static CXErrorCode clang_indexSourceFile_Impl(
 
   // Make sure to use the raw module format.
   CInvok->getHeaderSearchOpts().ModuleFormat = std::string(
-      CXXIdx->getPCHContainerOperations()->getRawReader().getFormat());
+      CXXIdx->getPCHContainerOperations()->getRawReader().getFormats().front());
 
   auto Unit = ASTUnit::create(CInvok, Diags, CaptureDiagnostics,
                               /*UserFilesAreVolatile=*/true);