Resubmit "[Tooling] Parse compilation database command lines on Windows."

This patch introduced the ability to decide at runtime whether to parse
JSON compilation database command lines using Gnu syntax or Windows
syntax.  However, there were many existing unit tests written that
hardcoded Gnu-specific paths.  These tests were now failing because
the auto-detection logic was choosing to parse them using Windows
rules.

This resubmission of the patch fixes this by introducing an enum
which defines the syntax mode, which defaults to auto-detect, but
for which the unit tests force Gnu style parsing.

Reviewed By: alexfh
Differential Revision: https://reviews.llvm.org/D23628

llvm-svn: 279120

diff --git a/clang/include/clang/Tooling/JSONCompilationDatabase.h b/clang/include/clang/Tooling/JSONCompilationDatabase.h
index 2a13fc1..ee64166 100644 (file)
--- a/clang/include/clang/Tooling/JSONCompilationDatabase.h
+++ b/clang/include/clang/Tooling/JSONCompilationDatabase.h
@@ -55,6 +55,7 @@ namespace tooling {
 ///
 /// JSON compilation databases can for example be generated in CMake projects
 /// by setting the flag -DCMAKE_EXPORT_COMPILE_COMMANDS.
+enum class JSONCommandLineSyntax { Windows, Gnu, AutoDetect };
 class JSONCompilationDatabase : public CompilationDatabase {
 public:
   /// \brief Loads a JSON compilation database from the specified file.
@@ -62,13 +63,15 @@ public:
   /// Returns NULL and sets ErrorMessage if the database could not be
   /// loaded from the given file.
   static std::unique_ptr<JSONCompilationDatabase>
-  loadFromFile(StringRef FilePath, std::string &ErrorMessage);
+  loadFromFile(StringRef FilePath, std::string &ErrorMessage,
+               JSONCommandLineSyntax Syntax);
 
   /// \brief Loads a JSON compilation database from a data buffer.
   ///
   /// Returns NULL and sets ErrorMessage if the database could not be loaded.
   static std::unique_ptr<JSONCompilationDatabase>
-  loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage);
+  loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage,
+                 JSONCommandLineSyntax Syntax);
 
   /// \brief Returns all compile comamnds in which the specified file was
   /// compiled.
@@ -89,8 +92,9 @@ public:
 
 private:
   /// \brief Constructs a JSON compilation database on a memory buffer.
-  JSONCompilationDatabase(std::unique_ptr<llvm::MemoryBuffer> Database)
-      : Database(std::move(Database)),
+  JSONCompilationDatabase(std::unique_ptr<llvm::MemoryBuffer> Database,
+                          JSONCommandLineSyntax Syntax)
+      : Database(std::move(Database)), Syntax(Syntax),
         YAMLStream(this->Database->getBuffer(), SM) {}
 
   /// \brief Parses the database file and creates the index.
@@ -123,6 +127,7 @@ private:
   FileMatchTrie MatchTrie;
 
   std::unique_ptr<llvm::MemoryBuffer> Database;
+  JSONCommandLineSyntax Syntax;
   llvm::SourceMgr SM;
   llvm::yaml::Stream YAMLStream;
 };

diff --git a/clang/lib/Tooling/JSONCompilationDatabase.cpp b/clang/lib/Tooling/JSONCompilationDatabase.cpp
index 299fbdc..07292da 100644 (file)
--- a/clang/lib/Tooling/JSONCompilationDatabase.cpp
+++ b/clang/lib/Tooling/JSONCompilationDatabase.cpp
@@ -16,7 +16,10 @@
 #include "clang/Tooling/CompilationDatabasePluginRegistry.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/StringSaver.h"
 #include <system_error>
 
 namespace clang {
@@ -111,8 +114,31 @@ class CommandLineArgumentParser {
   std::vector<std::string> CommandLine;
 };
 
-std::vector<std::string> unescapeCommandLine(
-    StringRef EscapedCommandLine) {
+std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
+                                             StringRef EscapedCommandLine) {
+  if (Syntax == JSONCommandLineSyntax::AutoDetect) {
+    llvm::Triple Triple(llvm::sys::getProcessTriple());
+    if (Triple.getOS() == llvm::Triple::OSType::Win32) {
+      // Assume Windows command line parsing on Win32 unless the triple
+      // explicitly
+      // tells us otherwise.
+      if (!Triple.hasEnvironment() ||
+          Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
+        Syntax = JSONCommandLineSyntax::Windows;
+      else
+        Syntax = JSONCommandLineSyntax::Gnu;
+    }
+  }
+
+  if (Syntax == JSONCommandLineSyntax::Windows) {
+    llvm::BumpPtrAllocator Alloc;
+    llvm::StringSaver Saver(Alloc);
+    llvm::SmallVector<const char *, 64> T;
+    llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
+    std::vector<std::string> Result(T.begin(), T.end());
+    return Result;
+  }
+  assert(Syntax == JSONCommandLineSyntax::Gnu);
   CommandLineArgumentParser parser(EscapedCommandLine);
   return parser.parse();
 }
@@ -123,7 +149,8 @@ class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
     SmallString<1024> JSONDatabasePath(Directory);
     llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
     std::unique_ptr<CompilationDatabase> Database(
-        JSONCompilationDatabase::loadFromFile(JSONDatabasePath, ErrorMessage));
+        JSONCompilationDatabase::loadFromFile(
+            JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect));
     if (!Database)
       return nullptr;
     return Database;
@@ -143,7 +170,8 @@ volatile int JSONAnchorSource = 0;
 
 std::unique_ptr<JSONCompilationDatabase>
 JSONCompilationDatabase::loadFromFile(StringRef FilePath,
-                                      std::string &ErrorMessage) {
+                                      std::string &ErrorMessage,
+                                      JSONCommandLineSyntax Syntax) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
       llvm::MemoryBuffer::getFile(FilePath);
   if (std::error_code Result = DatabaseBuffer.getError()) {
@@ -151,7 +179,7 @@ JSONCompilationDatabase::loadFromFile(StringRef FilePath,
     return nullptr;
   }
   std::unique_ptr<JSONCompilationDatabase> Database(
-      new JSONCompilationDatabase(std::move(*DatabaseBuffer)));
+      new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
   if (!Database->parse(ErrorMessage))
     return nullptr;
   return Database;
@@ -159,11 +187,12 @@ JSONCompilationDatabase::loadFromFile(StringRef FilePath,
 
 std::unique_ptr<JSONCompilationDatabase>
 JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
-                                        std::string &ErrorMessage) {
+                                        std::string &ErrorMessage,
+                                        JSONCommandLineSyntax Syntax) {
   std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
       llvm::MemoryBuffer::getMemBuffer(DatabaseString));
   std::unique_ptr<JSONCompilationDatabase> Database(
-      new JSONCompilationDatabase(std::move(DatabaseBuffer)));
+      new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
   if (!Database->parse(ErrorMessage))
     return nullptr;
   return Database;
@@ -211,10 +240,11 @@ JSONCompilationDatabase::getAllCompileCommands() const {
 }
 
 static std::vector<std::string>
-nodeToCommandLine(const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
+nodeToCommandLine(JSONCommandLineSyntax Syntax,
+                  const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
   SmallString<1024> Storage;
   if (Nodes.size() == 1) {
-    return unescapeCommandLine(Nodes[0]->getValue(Storage));
+    return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
   }
   std::vector<std::string> Arguments;
   for (auto *Node : Nodes) {
@@ -230,9 +260,9 @@ void JSONCompilationDatabase::getCommands(
     SmallString<8> DirectoryStorage;
     SmallString<32> FilenameStorage;
     Commands.emplace_back(
-      std::get<0>(CommandsRef[I])->getValue(DirectoryStorage),
-      std::get<1>(CommandsRef[I])->getValue(FilenameStorage),
-      nodeToCommandLine(std::get<2>(CommandsRef[I])));
+        std::get<0>(CommandsRef[I])->getValue(DirectoryStorage),
+        std::get<1>(CommandsRef[I])->getValue(FilenameStorage),
+        nodeToCommandLine(Syntax, std::get<2>(CommandsRef[I])));
   }
 }
 

diff --git a/clang/unittests/Tooling/CompilationDatabaseTest.cpp b/clang/unittests/Tooling/CompilationDatabaseTest.cpp
index 13d6023..4853983 100644 (file)
--- a/clang/unittests/Tooling/CompilationDatabaseTest.cpp
+++ b/clang/unittests/Tooling/CompilationDatabaseTest.cpp
@@ -21,9 +21,10 @@ namespace tooling {
 
 static void expectFailure(StringRef JSONDatabase, StringRef Explanation) {
   std::string ErrorMessage;
-  EXPECT_EQ(nullptr, JSONCompilationDatabase::loadFromBuffer(JSONDatabase,
-                                                             ErrorMessage))
-    << "Expected an error because of: " << Explanation.str();
+  EXPECT_EQ(nullptr,
+            JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                                    JSONCommandLineSyntax::Gnu))
+      << "Expected an error because of: " << Explanation.str();
 }
 
 TEST(JSONCompilationDatabase, ErrsOnInvalidFormat) {
@@ -46,9 +47,11 @@ TEST(JSONCompilationDatabase, ErrsOnInvalidFormat) {
 }
 
 static std::vector<std::string> getAllFiles(StringRef JSONDatabase,
-                                            std::string &ErrorMessage) {
+                                            std::string &ErrorMessage,
+                                            JSONCommandLineSyntax Syntax) {
   std::unique_ptr<CompilationDatabase> Database(
-      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage));
+      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                              Syntax));
   if (!Database) {
     ADD_FAILURE() << ErrorMessage;
     return std::vector<std::string>();
@@ -56,10 +59,12 @@ static std::vector<std::string> getAllFiles(StringRef JSONDatabase,
   return Database->getAllFiles();
 }
 
-static std::vector<CompileCommand> getAllCompileCommands(StringRef JSONDatabase,
-                                                    std::string &ErrorMessage) {
+static std::vector<CompileCommand>
+getAllCompileCommands(JSONCommandLineSyntax Syntax, StringRef JSONDatabase,
+                      std::string &ErrorMessage) {
   std::unique_ptr<CompilationDatabase> Database(
-      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage));
+      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                              Syntax));
   if (!Database) {
     ADD_FAILURE() << ErrorMessage;
     return std::vector<CompileCommand>();
@@ -70,7 +75,8 @@ static std::vector<CompileCommand> getAllCompileCommands(StringRef JSONDatabase,
 TEST(JSONCompilationDatabase, GetAllFiles) {
   std::string ErrorMessage;
   EXPECT_EQ(std::vector<std::string>(),
-            getAllFiles("[]", ErrorMessage)) << ErrorMessage;
+            getAllFiles("[]", ErrorMessage, JSONCommandLineSyntax::Gnu))
+      << ErrorMessage;
 
   std::vector<std::string> expected_files;
   SmallString<16> PathStorage;
@@ -78,20 +84,23 @@ TEST(JSONCompilationDatabase, GetAllFiles) {
   expected_files.push_back(PathStorage.str());
   llvm::sys::path::native("//net/dir/file2", PathStorage);
   expected_files.push_back(PathStorage.str());
-  EXPECT_EQ(expected_files, getAllFiles(
-    "[{\"directory\":\"//net/dir\","
-      "\"command\":\"command\","
-      "\"file\":\"file1\"},"
-    " {\"directory\":\"//net/dir\","
-      "\"command\":\"command\","
-      "\"file\":\"file2\"}]",
-    ErrorMessage)) << ErrorMessage;
+  EXPECT_EQ(expected_files,
+            getAllFiles("[{\"directory\":\"//net/dir\","
+                        "\"command\":\"command\","
+                        "\"file\":\"file1\"},"
+                        " {\"directory\":\"//net/dir\","
+                        "\"command\":\"command\","
+                        "\"file\":\"file2\"}]",
+                        ErrorMessage, JSONCommandLineSyntax::Gnu))
+      << ErrorMessage;
 }
 
 TEST(JSONCompilationDatabase, GetAllCompileCommands) {
   std::string ErrorMessage;
-  EXPECT_EQ(0u,
-            getAllCompileCommands("[]", ErrorMessage).size()) << ErrorMessage;
+  EXPECT_EQ(
+      0u, getAllCompileCommands(JSONCommandLineSyntax::Gnu, "[]", ErrorMessage)
+              .size())
+      << ErrorMessage;
 
   StringRef Directory1("//net/dir1");
   StringRef FileName1("file1");
@@ -101,12 +110,16 @@ TEST(JSONCompilationDatabase, GetAllCompileCommands) {
   StringRef Command2("command2");
 
   std::vector<CompileCommand> Commands = getAllCompileCommands(
-      ("[{\"directory\":\"" + Directory1 + "\"," +
-             "\"command\":\"" + Command1 + "\","
-             "\"file\":\"" + FileName1 + "\"},"
-       " {\"directory\":\"" + Directory2 + "\"," +
-             "\"command\":\"" + Command2 + "\","
-             "\"file\":\"" + FileName2 + "\"}]").str(),
+      JSONCommandLineSyntax::Gnu,
+      ("[{\"directory\":\"" + Directory1 + "\"," + "\"command\":\"" + Command1 +
+       "\","
+       "\"file\":\"" +
+       FileName1 + "\"},"
+                   " {\"directory\":\"" +
+       Directory2 + "\"," + "\"command\":\"" + Command2 + "\","
+                                                          "\"file\":\"" +
+       FileName2 + "\"}]")
+          .str(),
       ErrorMessage);
   EXPECT_EQ(2U, Commands.size()) << ErrorMessage;
   EXPECT_EQ(Directory1, Commands[0].Directory) << ErrorMessage;
@@ -120,12 +133,16 @@ TEST(JSONCompilationDatabase, GetAllCompileCommands) {
 
   // Check that order is preserved.
   Commands = getAllCompileCommands(
-      ("[{\"directory\":\"" + Directory2 + "\"," +
-             "\"command\":\"" + Command2 + "\","
-             "\"file\":\"" + FileName2 + "\"},"
-       " {\"directory\":\"" + Directory1 + "\"," +
-             "\"command\":\"" + Command1 + "\","
-             "\"file\":\"" + FileName1 + "\"}]").str(),
+      JSONCommandLineSyntax::Gnu,
+      ("[{\"directory\":\"" + Directory2 + "\"," + "\"command\":\"" + Command2 +
+       "\","
+       "\"file\":\"" +
+       FileName2 + "\"},"
+                   " {\"directory\":\"" +
+       Directory1 + "\"," + "\"command\":\"" + Command1 + "\","
+                                                          "\"file\":\"" +
+       FileName1 + "\"}]")
+          .str(),
       ErrorMessage);
   EXPECT_EQ(2U, Commands.size()) << ErrorMessage;
   EXPECT_EQ(Directory2, Commands[0].Directory) << ErrorMessage;
@@ -142,7 +159,8 @@ static CompileCommand findCompileArgsInJsonDatabase(StringRef FileName,
                                                     StringRef JSONDatabase,
                                                     std::string &ErrorMessage) {
   std::unique_ptr<CompilationDatabase> Database(
-      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage));
+      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                              JSONCommandLineSyntax::Gnu));
   if (!Database)
     return CompileCommand();
   std::vector<CompileCommand> Commands = Database->getCompileCommands(FileName);