[clang][deps] Avoid minimizing PCH input files

This patch avoid minimizing input files that contributed to a PCH or its modules. This prevents the implicit modular build to fail on unexpected file size. Depends on D106146.

Reviewed By: dexonsmith

Differential Revision: https://reviews.llvm.org/D104536

diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 5458104..d651ff2 100644 (file)
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -46,25 +46,73 @@ private:
   DependencyConsumer &C;
 };
 
-/// A listener that collects the names and paths to imported modules.
-class ImportCollectingListener : public ASTReaderListener {
-  using PrebuiltModuleFilesT =
-      decltype(HeaderSearchOptions::PrebuiltModuleFiles);
-
+/// A listener that collects the imported modules and optionally the input
+/// files.
+class PrebuiltModuleListener : public ASTReaderListener {
 public:
-  ImportCollectingListener(PrebuiltModuleFilesT &PrebuiltModuleFiles)
-      : PrebuiltModuleFiles(PrebuiltModuleFiles) {}
+  PrebuiltModuleListener(llvm::StringMap<std::string> &PrebuiltModuleFiles,
+                         llvm::StringSet<> &InputFiles, bool VisitInputFiles)
+      : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
+        VisitInputFiles(VisitInputFiles) {}
 
   bool needsImportVisitation() const override { return true; }
+  bool needsInputFileVisitation() override { return VisitInputFiles; }
+  bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
 
   void visitImport(StringRef ModuleName, StringRef Filename) override {
-    PrebuiltModuleFiles[std::string(ModuleName)] = std::string(Filename);
+    PrebuiltModuleFiles.insert({ModuleName, Filename.str()});
+  }
+
+  bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
+                      bool isExplicitModule) override {
+    InputFiles.insert(Filename);
+    return true;
   }
 
 private:
-  PrebuiltModuleFilesT &PrebuiltModuleFiles;
+  llvm::StringMap<std::string> &PrebuiltModuleFiles;
+  llvm::StringSet<> &InputFiles;
+  bool VisitInputFiles;
 };
 
+using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
+
+/// Visit the given prebuilt module and collect all of the modules it
+/// transitively imports and contributing input files.
+static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
+                                CompilerInstance &CI,
+                                PrebuiltModuleFilesT &ModuleFiles,
+                                llvm::StringSet<> &InputFiles,
+                                bool VisitInputFiles) {
+  // Maps the names of modules that weren't yet visited to their PCM path.
+  llvm::StringMap<std::string> ModuleFilesWorklist;
+  // Contains PCM paths of all visited modules.
+  llvm::StringSet<> VisitedModuleFiles;
+
+  PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles,
+                                  VisitInputFiles);
+
+  auto GatherModuleFileInfo = [&](StringRef ASTFile) {
+    ASTReader::readASTFileControlBlock(
+        ASTFile, CI.getFileManager(), CI.getPCHContainerReader(),
+        /*FindModuleFileExtensions=*/false, Listener,
+        /*ValidateDiagnosticOptions=*/false);
+  };
+
+  GatherModuleFileInfo(PrebuiltModuleFilename);
+  while (!ModuleFilesWorklist.empty()) {
+    auto WorklistItemIt = ModuleFilesWorklist.begin();
+
+    if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) {
+      VisitedModuleFiles.insert(WorklistItemIt->getValue());
+      GatherModuleFileInfo(WorklistItemIt->getValue());
+      ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue();
+    }
+
+    ModuleFilesWorklist.erase(WorklistItemIt);
+  }
+}
+
 /// Transform arbitrary file name into an object-like file name.
 static std::string makeObjFileName(StringRef FileName) {
   SmallString<128> ObjFileName(FileName);
@@ -122,13 +170,33 @@ public:
 
     Compiler.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = true;
 
-    // Use the dependency scanning optimized file system if we can.
+    FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
+    Compiler.setFileManager(FileMgr);
+    Compiler.createSourceManager(*FileMgr);
+
+    llvm::StringSet<> PrebuiltModulesInputFiles;
+    // Store the list of prebuilt module files into header search options. This
+    // will prevent the implicit build to create duplicate modules and will
+    // force reuse of the existing prebuilt module files instead.
+    if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty())
+      visitPrebuiltModule(
+          Compiler.getPreprocessorOpts().ImplicitPCHInclude, Compiler,
+          Compiler.getHeaderSearchOpts().PrebuiltModuleFiles,
+          PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
+
+    // Use the dependency scanning optimized file system if requested to do so.
     if (DepFS) {
       const CompilerInvocation &CI = Compiler.getInvocation();
+      DepFS->clearIgnoredFiles();
+      // Ignore any files that contributed to prebuilt modules. The implicit
+      // build validates the modules by comparing the reported sizes of their
+      // inputs to the current state of the filesystem. Minimization would throw
+      // this mechanism off.
+      for (const auto &File : PrebuiltModulesInputFiles)
+        DepFS->ignoreFile(File.getKey());
       // Add any filenames that were explicity passed in the build settings and
       // that might be opened, as we want to ensure we don't run source
       // minimization on them.
-      DepFS->clearIgnoredFiles();
       for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries)
         DepFS->ignoreFile(Entry.Path);
       for (const auto &Entry : CI.getHeaderSearchOpts().VFSOverlayFiles)
@@ -146,24 +214,6 @@ public:
             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
     }
 
-    FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
-    Compiler.setFileManager(FileMgr);
-    Compiler.createSourceManager(*FileMgr);
-
-    if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
-      // Collect the modules that were prebuilt as part of the PCH and pass them
-      // to the compiler. This will prevent the implicit build to create
-      // duplicate modules and force reuse of existing prebuilt module files
-      // instead.
-      ImportCollectingListener Listener(
-          Compiler.getHeaderSearchOpts().PrebuiltModuleFiles);
-      ASTReader::readASTFileControlBlock(
-          Compiler.getPreprocessorOpts().ImplicitPCHInclude,
-          Compiler.getFileManager(), Compiler.getPCHContainerReader(),
-          /*FindModuleFileExtensions=*/false, Listener,
-          /*ValidateDiagnosticOptions=*/false);
-    }
-
     // Create the dependency collector that will collect the produced
     // dependencies.
     //

diff --git a/clang/test/ClangScanDeps/modules-pch.c b/clang/test/ClangScanDeps/modules-pch.c
index 8b8b753..b9b7ed7 100644 (file)
--- a/clang/test/ClangScanDeps/modules-pch.c
+++ b/clang/test/ClangScanDeps/modules-pch.c
@@ -6,7 +6,7 @@
 // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb.json
 // RUN: echo -%t > %t/result_pch.json
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN:   -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_pch.json
+// RUN:   -generate-modules-path-args -module-files-dir %t/build >> %t/result_pch.json
 // RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-PCH
 //
 // Check we didn't build the PCH during dependency scanning.
@@ -127,9 +127,8 @@
 //
 // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb.json
 // RUN: echo -%t > %t/result_tu.json
-// FIXME: Make this work with '-mode preprocess-minimized-sources'.
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN:   -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_tu.json
+// RUN:   -generate-modules-path-args -module-files-dir %t/build >> %t/result_tu.json
 // RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU
 //
 // CHECK-TU:      -[[PREFIX:.*]]
@@ -193,7 +192,7 @@
 // RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu_with_common.json > %t/cdb.json
 // RUN: echo -%t > %t/result_tu_with_common.json
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
-// RUN:   -generate-modules-path-args -module-files-dir %t/build -mode preprocess >> %t/result_tu_with_common.json
+// RUN:   -generate-modules-path-args -module-files-dir %t/build >> %t/result_tu_with_common.json
 // RUN: cat %t/result_tu_with_common.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU-WITH-COMMON
 //
 // CHECK-TU-WITH-COMMON:      -[[PREFIX:.*]]