Recommit r329442: Generate Libclang invocation reproducers using a new
authorAlex Lorenz <arphaman@gmail.com>
Sat, 7 Apr 2018 00:03:27 +0000 (00:03 +0000)
committerAlex Lorenz <arphaman@gmail.com>
Sat, 7 Apr 2018 00:03:27 +0000 (00:03 +0000)
-cc1gen-reproducer driver option

The recommit fixes:
- An MSAN failure (CCPrintOptions wasn't initialized in the Driver)
- Ensures that the strings in the libclang invocation files are escaped

Original message:

This commit is a follow up to the previous work that recorded Libclang invocations
into temporary files: r319702.

It adds a new -cc1 mode to clang: -cc1gen-reproducer. The goal of this mode is to generate
Clang reproducer files for Libclang tool invocation. The JSON format in the invocation
files is not really intended to be stable, so Libclang and Clang should be of the same version
when generating reproducers.
The new mode emits the information about the temporary files and Libclang-specific information
to stdout using JSON.

rdar://35322614

Differential Revision: https://reviews.llvm.org/D40983

llvm-svn: 329465

clang/include/clang/Driver/Driver.h
clang/lib/Driver/Driver.cpp
clang/test/Index/create-libclang-completion-reproducer.c [new file with mode: 0644]
clang/test/Index/create-libclang-parsing-reproducer.c [new file with mode: 0644]
clang/tools/driver/CMakeLists.txt
clang/tools/driver/cc1gen_reproducer_main.cpp [new file with mode: 0644]
clang/tools/driver/driver.cpp
clang/tools/libclang/CIndexer.cpp

index ca2ca30..39fce9c 100644 (file)
@@ -405,11 +405,19 @@ public:
   int ExecuteCompilation(Compilation &C,
      SmallVectorImpl< std::pair<int, const Command *> > &FailingCommands);
 
-  /// generateCompilationDiagnostics - Generate diagnostics information 
+  /// Contains the files in the compilation diagnostic report generated by
+  /// generateCompilationDiagnostics.
+  struct CompilationDiagnosticReport {
+    llvm::SmallVector<std::string, 4> TemporaryFiles;
+  };
+
+  /// generateCompilationDiagnostics - Generate diagnostics information
   /// including preprocessed source file(s).
-  /// 
-  void generateCompilationDiagnostics(Compilation &C,
-                                      const Command &FailingCommand);
+  ///
+  void generateCompilationDiagnostics(
+      Compilation &C, const Command &FailingCommand,
+      StringRef AdditionalInformation = "",
+      CompilationDiagnosticReport *GeneratedReport = nullptr);
 
   /// @}
   /// @name Helper Methods
index 1a62192..e2c2411 100644 (file)
@@ -89,14 +89,14 @@ Driver::Driver(StringRef ClangExecutable, StringRef DefaultTargetTriple,
     : Opts(createDriverOptTable()), Diags(Diags), VFS(std::move(VFS)),
       Mode(GCCMode), SaveTemps(SaveTempsNone), BitcodeEmbed(EmbedNone),
       LTOMode(LTOK_None), ClangExecutable(ClangExecutable),
-      SysRoot(DEFAULT_SYSROOT), 
-      DriverTitle("clang LLVM compiler"), CCPrintOptionsFilename(nullptr),
-      CCPrintHeadersFilename(nullptr), CCLogDiagnosticsFilename(nullptr),
-      CCCPrintBindings(false), CCPrintHeaders(false), CCLogDiagnostics(false),
+      SysRoot(DEFAULT_SYSROOT), DriverTitle("clang LLVM compiler"),
+      CCPrintOptionsFilename(nullptr), CCPrintHeadersFilename(nullptr),
+      CCLogDiagnosticsFilename(nullptr), CCCPrintBindings(false),
+      CCPrintOptions(false), CCPrintHeaders(false), CCLogDiagnostics(false),
       CCGenDiagnostics(false), DefaultTargetTriple(DefaultTargetTriple),
-      CCCGenericGCCName(""), Saver(Alloc),
-      CheckInputsExist(true), CCCUsePCH(true),
-      GenReproducer(false), SuppressMissingInputWarning(false) {
+      CCCGenericGCCName(""), Saver(Alloc), CheckInputsExist(true),
+      CCCUsePCH(true), GenReproducer(false),
+      SuppressMissingInputWarning(false) {
 
   // Provide a sane fallback if no VFS is specified.
   if (!this->VFS)
@@ -1111,8 +1111,9 @@ bool Driver::getCrashDiagnosticFile(StringRef ReproCrashFilename,
 // When clang crashes, produce diagnostic information including the fully
 // preprocessed source file(s).  Request that the developer attach the
 // diagnostic information to a bug report.
-void Driver::generateCompilationDiagnostics(Compilation &C,
-                                            const Command &FailingCommand) {
+void Driver::generateCompilationDiagnostics(
+    Compilation &C, const Command &FailingCommand,
+    StringRef AdditionalInformation, CompilationDiagnosticReport *Report) {
   if (C.getArgs().hasArg(options::OPT_fno_crash_diagnostics))
     return;
 
@@ -1238,6 +1239,8 @@ void Driver::generateCompilationDiagnostics(Compilation &C,
   SmallString<128> ReproCrashFilename;
   for (const char *TempFile : TempFiles) {
     Diag(clang::diag::note_drv_command_failed_diag_msg) << TempFile;
+    if (Report)
+      Report->TemporaryFiles.push_back(TempFile);
     if (ReproCrashFilename.empty()) {
       ReproCrashFilename = TempFile;
       llvm::sys::path::replace_extension(ReproCrashFilename, ".crash");
@@ -1266,6 +1269,11 @@ void Driver::generateCompilationDiagnostics(Compilation &C,
     ScriptOS << "# Original command: ";
     Cmd.Print(ScriptOS, "\n", /*Quote=*/true);
     Cmd.Print(ScriptOS, "\n", /*Quote=*/true, &CrashInfo);
+    if (!AdditionalInformation.empty())
+      ScriptOS << "\n# Additional information: " << AdditionalInformation
+               << "\n";
+    if (Report)
+      Report->TemporaryFiles.push_back(Script);
     Diag(clang::diag::note_drv_command_failed_diag_msg) << Script;
   }
 
diff --git a/clang/test/Index/create-libclang-completion-reproducer.c b/clang/test/Index/create-libclang-completion-reproducer.c
new file mode 100644 (file)
index 0000000..5683e10
--- /dev/null
@@ -0,0 +1,14 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: env CINDEXTEST_INVOCATION_EMISSION_PATH=%t not c-index-test -code-completion-at=%s:10:1 "-remap-file=%s,%S/Inputs/record-parsing-invocation-remap.c" %s
+// RUN: %clang -cc1gen-reproducer %t/libclang-* -v | FileCheck %s
+
+// Invocation file must be removed by clang:
+// RUN: ls %t | count 0
+
+// CHECK: REPRODUCER METAINFO: {"libclang.operation": "complete", "libclang.opts": "1", "invocation-args": ["-code-completion-at={{.*}}create-libclang-completion-reproducer.c:10:1"]}
+
+// CHECK: REPRODUCER:
+// CHECK-NEXT: {
+// CHECK-NEXT: "files":["{{.*}}.c","{{.*}}.sh"]
+// CHECK-NEXT: }
diff --git a/clang/test/Index/create-libclang-parsing-reproducer.c b/clang/test/Index/create-libclang-parsing-reproducer.c
new file mode 100644 (file)
index 0000000..0cfbfb1
--- /dev/null
@@ -0,0 +1,21 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: env CINDEXTEST_INVOCATION_EMISSION_PATH=%t not c-index-test -test-load-source all %s
+// RUN: %clang -cc1gen-reproducer %t/libclang-* -v | FileCheck %s
+
+// Invocation file must be removed by clang:
+// RUN: ls %t | count 0
+
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: env CINDEXTEST_INVOCATION_EMISSION_PATH=%t not c-index-test -test-load-source all "-remap-file=%s,%S/Inputs/record-parsing-invocation-remap.c" %s
+// RUN: %clang -cc1gen-reproducer %t/libclang-* -v | FileCheck %s
+
+#pragma clang __debug parser_crash
+
+// CHECK: REPRODUCER METAINFO: {"libclang.operation": "parse", "libclang.opts": "1"}
+
+// CHECK: REPRODUCER:
+// CHECK-NEXT: {
+// CHECK-NEXT: "files":["{{.*}}.c","{{.*}}.sh"]
+// CHECK-NEXT: }
index 22a4984..27566f7 100644 (file)
@@ -32,6 +32,7 @@ add_clang_tool(clang
   driver.cpp
   cc1_main.cpp
   cc1as_main.cpp
+  cc1gen_reproducer_main.cpp
 
   DEPENDS
   ${tablegen_deps}
diff --git a/clang/tools/driver/cc1gen_reproducer_main.cpp b/clang/tools/driver/cc1gen_reproducer_main.cpp
new file mode 100644 (file)
index 0000000..a4c034d
--- /dev/null
@@ -0,0 +1,196 @@
+//===-- cc1gen_reproducer_main.cpp - Clang reproducer generator  ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the entry point to the clang -cc1gen-reproducer functionality, which
+// generates reproducers for invocations for clang-based tools.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/VirtualFileSystem.h"
+#include "clang/Driver/Compilation.h"
+#include "clang/Driver/Driver.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace clang;
+
+namespace {
+
+struct UnsavedFileHash {
+  std::string Name;
+  std::string MD5;
+};
+
+struct ClangInvocationInfo {
+  std::string Toolchain;
+  std::string LibclangOperation;
+  std::string LibclangOptions;
+  std::vector<std::string> Arguments;
+  std::vector<std::string> InvocationArguments;
+  std::vector<UnsavedFileHash> UnsavedFileHashes;
+  bool Dump = false;
+};
+
+} // end anonymous namespace
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(UnsavedFileHash)
+
+namespace llvm {
+namespace yaml {
+
+template <> struct MappingTraits<UnsavedFileHash> {
+  static void mapping(IO &IO, UnsavedFileHash &Info) {
+    IO.mapRequired("name", Info.Name);
+    IO.mapRequired("md5", Info.MD5);
+  }
+};
+
+template <> struct MappingTraits<ClangInvocationInfo> {
+  static void mapping(IO &IO, ClangInvocationInfo &Info) {
+    IO.mapRequired("toolchain", Info.Toolchain);
+    IO.mapOptional("libclang.operation", Info.LibclangOperation);
+    IO.mapOptional("libclang.opts", Info.LibclangOptions);
+    IO.mapRequired("args", Info.Arguments);
+    IO.mapOptional("invocation-args", Info.InvocationArguments);
+    IO.mapOptional("unsaved_file_hashes", Info.UnsavedFileHashes);
+  }
+};
+
+} // end namespace yaml
+} // end namespace llvm
+
+static std::string generateReproducerMetaInfo(const ClangInvocationInfo &Info) {
+  std::string Result;
+  llvm::raw_string_ostream OS(Result);
+  OS << '{';
+  bool NeedComma = false;
+  auto EmitKey = [&](StringRef Key) {
+    if (NeedComma)
+      OS << ", ";
+    NeedComma = true;
+    OS << '"' << Key << "\": ";
+  };
+  auto EmitStringKey = [&](StringRef Key, StringRef Value) {
+    if (Value.empty())
+      return;
+    EmitKey(Key);
+    OS << '"' << Value << '"';
+  };
+  EmitStringKey("libclang.operation", Info.LibclangOperation);
+  EmitStringKey("libclang.opts", Info.LibclangOptions);
+  if (!Info.InvocationArguments.empty()) {
+    EmitKey("invocation-args");
+    OS << '[';
+    for (const auto &Arg : llvm::enumerate(Info.InvocationArguments)) {
+      if (Arg.index())
+        OS << ',';
+      OS << '"' << Arg.value() << '"';
+    }
+    OS << ']';
+  }
+  OS << '}';
+  // FIXME: Compare unsaved file hashes and report mismatch in the reproducer.
+  if (Info.Dump)
+    llvm::outs() << "REPRODUCER METAINFO: " << OS.str() << "\n";
+  return std::move(OS.str());
+}
+
+/// Generates a reproducer for a set of arguments from a specific invocation.
+static llvm::Optional<driver::Driver::CompilationDiagnosticReport>
+generateReproducerForInvocationArguments(ArrayRef<const char *> Argv,
+                                         const ClangInvocationInfo &Info) {
+  using namespace driver;
+  auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(Argv[0]);
+
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions;
+
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+  DiagnosticsEngine Diags(DiagID, &*DiagOpts, new IgnoringDiagConsumer());
+  ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
+  Driver TheDriver(Argv[0], llvm::sys::getDefaultTargetTriple(), Diags);
+  TheDriver.setTargetAndMode(TargetAndMode);
+
+  std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(Argv));
+  if (C && !C->containsError()) {
+    for (const auto &J : C->getJobs()) {
+      if (const Command *Cmd = dyn_cast<Command>(&J)) {
+        Driver::CompilationDiagnosticReport Report;
+        TheDriver.generateCompilationDiagnostics(
+            *C, *Cmd, generateReproducerMetaInfo(Info), &Report);
+        return Report;
+      }
+    }
+  }
+
+  return None;
+}
+
+std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes);
+
+static void printReproducerInformation(
+    llvm::raw_ostream &OS, const ClangInvocationInfo &Info,
+    const driver::Driver::CompilationDiagnosticReport &Report) {
+  OS << "REPRODUCER:\n";
+  OS << "{\n";
+  OS << R"("files":[)";
+  for (const auto &File : llvm::enumerate(Report.TemporaryFiles)) {
+    if (File.index())
+      OS << ',';
+    OS << '"' << File.value() << '"';
+  }
+  OS << "]\n}\n";
+}
+
+int cc1gen_reproducer_main(ArrayRef<const char *> Argv, const char *Argv0,
+                           void *MainAddr) {
+  if (Argv.size() < 1) {
+    llvm::errs() << "error: missing invocation file\n";
+    return 1;
+  }
+  // Parse the invocation descriptor.
+  StringRef Input = Argv[0];
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
+      llvm::MemoryBuffer::getFile(Input);
+  if (!Buffer) {
+    llvm::errs() << "error: failed to read " << Input << ": "
+                 << Buffer.getError().message() << "\n";
+    return 1;
+  }
+  llvm::yaml::Input YAML(Buffer.get()->getBuffer());
+  ClangInvocationInfo InvocationInfo;
+  YAML >> InvocationInfo;
+  if (Argv.size() > 1 && Argv[1] == StringRef("-v"))
+    InvocationInfo.Dump = true;
+
+  // Create an invocation that will produce the reproducer.
+  std::vector<const char *> DriverArgs;
+  for (const auto &Arg : InvocationInfo.Arguments)
+    DriverArgs.push_back(Arg.c_str());
+  std::string Path = GetExecutablePath(Argv0, /*CanonicalPrefixes=*/true);
+  DriverArgs[0] = Path.c_str();
+  llvm::Optional<driver::Driver::CompilationDiagnosticReport> Report =
+      generateReproducerForInvocationArguments(DriverArgs, InvocationInfo);
+
+  // Emit the information about the reproduce files to stdout.
+  int Result = 1;
+  if (Report) {
+    printReproducerInformation(llvm::outs(), InvocationInfo, *Report);
+    Result = 0;
+  }
+
+  // Remove the input file.
+  llvm::sys::fs::remove(Input);
+  return Result;
+}
index 72d56a7..32cb1b4 100644 (file)
@@ -205,6 +205,8 @@ extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
                     void *MainAddr);
 extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
                       void *MainAddr);
+extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
+                                  const char *Argv0, void *MainAddr);
 
 static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
                                     SmallVectorImpl<const char *> &ArgVector,
@@ -310,6 +312,8 @@ static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
     return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
   if (Tool == "as")
     return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
+  if (Tool == "gen-reproducer")
+    return cc1gen_reproducer_main(argv.slice(2), argv[0], GetExecutablePathVP);
 
   // Reject unknown tools.
   llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
index 62ea881..d902470 100644 (file)
@@ -22,6 +22,7 @@
 #include "llvm/Support/MutexGuard.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/YAMLParser.h"
 #include <cstdio>
 
 #ifdef __CYGWIN__
@@ -112,7 +113,7 @@ LibclangInvocationReporter::LibclangInvocationReporter(
   // Write out the information about the invocation to it.
   auto WriteStringKey = [&OS](StringRef Key, StringRef Value) {
     OS << R"(")" << Key << R"(":")";
-    OS << Value << '"';
+    OS << llvm::yaml::escape(Value) << '"';
   };
   OS << '{';
   WriteStringKey("toolchain", Idx.getClangToolchainPath());