From 9e60a2ad734e1d598f93e7148c6339795b57359c Mon Sep 17 00:00:00 2001 From: Zachary Turner Date: Thu, 18 Aug 2016 19:31:48 +0000 Subject: [PATCH] Resubmit "[Tooling] Parse compilation database command lines on Windows." This patch introduced the ability to decide at runtime whether to parse JSON compilation database command lines using Gnu syntax or Windows syntax. However, there were many existing unit tests written that hardcoded Gnu-specific paths. These tests were now failing because the auto-detection logic was choosing to parse them using Windows rules. This resubmission of the patch fixes this by introducing an enum which defines the syntax mode, which defaults to auto-detect, but for which the unit tests force Gnu style parsing. Reviewed By: alexfh Differential Revision: https://reviews.llvm.org/D23628 llvm-svn: 279120 --- .../clang/Tooling/JSONCompilationDatabase.h | 13 ++-- clang/lib/Tooling/JSONCompilationDatabase.cpp | 54 ++++++++++---- .../unittests/Tooling/CompilationDatabaseTest.cpp | 82 +++++++++++++--------- 3 files changed, 101 insertions(+), 48 deletions(-) diff --git a/clang/include/clang/Tooling/JSONCompilationDatabase.h b/clang/include/clang/Tooling/JSONCompilationDatabase.h index 2a13fc1..ee64166c 100644 --- a/clang/include/clang/Tooling/JSONCompilationDatabase.h +++ b/clang/include/clang/Tooling/JSONCompilationDatabase.h @@ -55,6 +55,7 @@ namespace tooling { /// /// JSON compilation databases can for example be generated in CMake projects /// by setting the flag -DCMAKE_EXPORT_COMPILE_COMMANDS. +enum class JSONCommandLineSyntax { Windows, Gnu, AutoDetect }; class JSONCompilationDatabase : public CompilationDatabase { public: /// \brief Loads a JSON compilation database from the specified file. @@ -62,13 +63,15 @@ public: /// Returns NULL and sets ErrorMessage if the database could not be /// loaded from the given file. static std::unique_ptr - loadFromFile(StringRef FilePath, std::string &ErrorMessage); + loadFromFile(StringRef FilePath, std::string &ErrorMessage, + JSONCommandLineSyntax Syntax); /// \brief Loads a JSON compilation database from a data buffer. /// /// Returns NULL and sets ErrorMessage if the database could not be loaded. static std::unique_ptr - loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage); + loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage, + JSONCommandLineSyntax Syntax); /// \brief Returns all compile comamnds in which the specified file was /// compiled. @@ -89,8 +92,9 @@ public: private: /// \brief Constructs a JSON compilation database on a memory buffer. - JSONCompilationDatabase(std::unique_ptr Database) - : Database(std::move(Database)), + JSONCompilationDatabase(std::unique_ptr Database, + JSONCommandLineSyntax Syntax) + : Database(std::move(Database)), Syntax(Syntax), YAMLStream(this->Database->getBuffer(), SM) {} /// \brief Parses the database file and creates the index. @@ -123,6 +127,7 @@ private: FileMatchTrie MatchTrie; std::unique_ptr Database; + JSONCommandLineSyntax Syntax; llvm::SourceMgr SM; llvm::yaml::Stream YAMLStream; }; diff --git a/clang/lib/Tooling/JSONCompilationDatabase.cpp b/clang/lib/Tooling/JSONCompilationDatabase.cpp index 299fbdc..07292da 100644 --- a/clang/lib/Tooling/JSONCompilationDatabase.cpp +++ b/clang/lib/Tooling/JSONCompilationDatabase.cpp @@ -16,7 +16,10 @@ #include "clang/Tooling/CompilationDatabasePluginRegistry.h" #include "clang/Tooling/Tooling.h" #include "llvm/ADT/SmallString.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Path.h" +#include "llvm/Support/StringSaver.h" #include namespace clang { @@ -111,8 +114,31 @@ class CommandLineArgumentParser { std::vector CommandLine; }; -std::vector unescapeCommandLine( - StringRef EscapedCommandLine) { +std::vector unescapeCommandLine(JSONCommandLineSyntax Syntax, + StringRef EscapedCommandLine) { + if (Syntax == JSONCommandLineSyntax::AutoDetect) { + llvm::Triple Triple(llvm::sys::getProcessTriple()); + if (Triple.getOS() == llvm::Triple::OSType::Win32) { + // Assume Windows command line parsing on Win32 unless the triple + // explicitly + // tells us otherwise. + if (!Triple.hasEnvironment() || + Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC) + Syntax = JSONCommandLineSyntax::Windows; + else + Syntax = JSONCommandLineSyntax::Gnu; + } + } + + if (Syntax == JSONCommandLineSyntax::Windows) { + llvm::BumpPtrAllocator Alloc; + llvm::StringSaver Saver(Alloc); + llvm::SmallVector T; + llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T); + std::vector Result(T.begin(), T.end()); + return Result; + } + assert(Syntax == JSONCommandLineSyntax::Gnu); CommandLineArgumentParser parser(EscapedCommandLine); return parser.parse(); } @@ -123,7 +149,8 @@ class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin { SmallString<1024> JSONDatabasePath(Directory); llvm::sys::path::append(JSONDatabasePath, "compile_commands.json"); std::unique_ptr Database( - JSONCompilationDatabase::loadFromFile(JSONDatabasePath, ErrorMessage)); + JSONCompilationDatabase::loadFromFile( + JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect)); if (!Database) return nullptr; return Database; @@ -143,7 +170,8 @@ volatile int JSONAnchorSource = 0; std::unique_ptr JSONCompilationDatabase::loadFromFile(StringRef FilePath, - std::string &ErrorMessage) { + std::string &ErrorMessage, + JSONCommandLineSyntax Syntax) { llvm::ErrorOr> DatabaseBuffer = llvm::MemoryBuffer::getFile(FilePath); if (std::error_code Result = DatabaseBuffer.getError()) { @@ -151,7 +179,7 @@ JSONCompilationDatabase::loadFromFile(StringRef FilePath, return nullptr; } std::unique_ptr Database( - new JSONCompilationDatabase(std::move(*DatabaseBuffer))); + new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax)); if (!Database->parse(ErrorMessage)) return nullptr; return Database; @@ -159,11 +187,12 @@ JSONCompilationDatabase::loadFromFile(StringRef FilePath, std::unique_ptr JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString, - std::string &ErrorMessage) { + std::string &ErrorMessage, + JSONCommandLineSyntax Syntax) { std::unique_ptr DatabaseBuffer( llvm::MemoryBuffer::getMemBuffer(DatabaseString)); std::unique_ptr Database( - new JSONCompilationDatabase(std::move(DatabaseBuffer))); + new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax)); if (!Database->parse(ErrorMessage)) return nullptr; return Database; @@ -211,10 +240,11 @@ JSONCompilationDatabase::getAllCompileCommands() const { } static std::vector -nodeToCommandLine(const std::vector &Nodes) { +nodeToCommandLine(JSONCommandLineSyntax Syntax, + const std::vector &Nodes) { SmallString<1024> Storage; if (Nodes.size() == 1) { - return unescapeCommandLine(Nodes[0]->getValue(Storage)); + return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage)); } std::vector Arguments; for (auto *Node : Nodes) { @@ -230,9 +260,9 @@ void JSONCompilationDatabase::getCommands( SmallString<8> DirectoryStorage; SmallString<32> FilenameStorage; Commands.emplace_back( - std::get<0>(CommandsRef[I])->getValue(DirectoryStorage), - std::get<1>(CommandsRef[I])->getValue(FilenameStorage), - nodeToCommandLine(std::get<2>(CommandsRef[I]))); + std::get<0>(CommandsRef[I])->getValue(DirectoryStorage), + std::get<1>(CommandsRef[I])->getValue(FilenameStorage), + nodeToCommandLine(Syntax, std::get<2>(CommandsRef[I]))); } } diff --git a/clang/unittests/Tooling/CompilationDatabaseTest.cpp b/clang/unittests/Tooling/CompilationDatabaseTest.cpp index 13d6023..4853983 100644 --- a/clang/unittests/Tooling/CompilationDatabaseTest.cpp +++ b/clang/unittests/Tooling/CompilationDatabaseTest.cpp @@ -21,9 +21,10 @@ namespace tooling { static void expectFailure(StringRef JSONDatabase, StringRef Explanation) { std::string ErrorMessage; - EXPECT_EQ(nullptr, JSONCompilationDatabase::loadFromBuffer(JSONDatabase, - ErrorMessage)) - << "Expected an error because of: " << Explanation.str(); + EXPECT_EQ(nullptr, + JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage, + JSONCommandLineSyntax::Gnu)) + << "Expected an error because of: " << Explanation.str(); } TEST(JSONCompilationDatabase, ErrsOnInvalidFormat) { @@ -46,9 +47,11 @@ TEST(JSONCompilationDatabase, ErrsOnInvalidFormat) { } static std::vector getAllFiles(StringRef JSONDatabase, - std::string &ErrorMessage) { + std::string &ErrorMessage, + JSONCommandLineSyntax Syntax) { std::unique_ptr Database( - JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage)); + JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage, + Syntax)); if (!Database) { ADD_FAILURE() << ErrorMessage; return std::vector(); @@ -56,10 +59,12 @@ static std::vector getAllFiles(StringRef JSONDatabase, return Database->getAllFiles(); } -static std::vector getAllCompileCommands(StringRef JSONDatabase, - std::string &ErrorMessage) { +static std::vector +getAllCompileCommands(JSONCommandLineSyntax Syntax, StringRef JSONDatabase, + std::string &ErrorMessage) { std::unique_ptr Database( - JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage)); + JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage, + Syntax)); if (!Database) { ADD_FAILURE() << ErrorMessage; return std::vector(); @@ -70,7 +75,8 @@ static std::vector getAllCompileCommands(StringRef JSONDatabase, TEST(JSONCompilationDatabase, GetAllFiles) { std::string ErrorMessage; EXPECT_EQ(std::vector(), - getAllFiles("[]", ErrorMessage)) << ErrorMessage; + getAllFiles("[]", ErrorMessage, JSONCommandLineSyntax::Gnu)) + << ErrorMessage; std::vector expected_files; SmallString<16> PathStorage; @@ -78,20 +84,23 @@ TEST(JSONCompilationDatabase, GetAllFiles) { expected_files.push_back(PathStorage.str()); llvm::sys::path::native("//net/dir/file2", PathStorage); expected_files.push_back(PathStorage.str()); - EXPECT_EQ(expected_files, getAllFiles( - "[{\"directory\":\"//net/dir\"," - "\"command\":\"command\"," - "\"file\":\"file1\"}," - " {\"directory\":\"//net/dir\"," - "\"command\":\"command\"," - "\"file\":\"file2\"}]", - ErrorMessage)) << ErrorMessage; + EXPECT_EQ(expected_files, + getAllFiles("[{\"directory\":\"//net/dir\"," + "\"command\":\"command\"," + "\"file\":\"file1\"}," + " {\"directory\":\"//net/dir\"," + "\"command\":\"command\"," + "\"file\":\"file2\"}]", + ErrorMessage, JSONCommandLineSyntax::Gnu)) + << ErrorMessage; } TEST(JSONCompilationDatabase, GetAllCompileCommands) { std::string ErrorMessage; - EXPECT_EQ(0u, - getAllCompileCommands("[]", ErrorMessage).size()) << ErrorMessage; + EXPECT_EQ( + 0u, getAllCompileCommands(JSONCommandLineSyntax::Gnu, "[]", ErrorMessage) + .size()) + << ErrorMessage; StringRef Directory1("//net/dir1"); StringRef FileName1("file1"); @@ -101,12 +110,16 @@ TEST(JSONCompilationDatabase, GetAllCompileCommands) { StringRef Command2("command2"); std::vector Commands = getAllCompileCommands( - ("[{\"directory\":\"" + Directory1 + "\"," + - "\"command\":\"" + Command1 + "\"," - "\"file\":\"" + FileName1 + "\"}," - " {\"directory\":\"" + Directory2 + "\"," + - "\"command\":\"" + Command2 + "\"," - "\"file\":\"" + FileName2 + "\"}]").str(), + JSONCommandLineSyntax::Gnu, + ("[{\"directory\":\"" + Directory1 + "\"," + "\"command\":\"" + Command1 + + "\"," + "\"file\":\"" + + FileName1 + "\"}," + " {\"directory\":\"" + + Directory2 + "\"," + "\"command\":\"" + Command2 + "\"," + "\"file\":\"" + + FileName2 + "\"}]") + .str(), ErrorMessage); EXPECT_EQ(2U, Commands.size()) << ErrorMessage; EXPECT_EQ(Directory1, Commands[0].Directory) << ErrorMessage; @@ -120,12 +133,16 @@ TEST(JSONCompilationDatabase, GetAllCompileCommands) { // Check that order is preserved. Commands = getAllCompileCommands( - ("[{\"directory\":\"" + Directory2 + "\"," + - "\"command\":\"" + Command2 + "\"," - "\"file\":\"" + FileName2 + "\"}," - " {\"directory\":\"" + Directory1 + "\"," + - "\"command\":\"" + Command1 + "\"," - "\"file\":\"" + FileName1 + "\"}]").str(), + JSONCommandLineSyntax::Gnu, + ("[{\"directory\":\"" + Directory2 + "\"," + "\"command\":\"" + Command2 + + "\"," + "\"file\":\"" + + FileName2 + "\"}," + " {\"directory\":\"" + + Directory1 + "\"," + "\"command\":\"" + Command1 + "\"," + "\"file\":\"" + + FileName1 + "\"}]") + .str(), ErrorMessage); EXPECT_EQ(2U, Commands.size()) << ErrorMessage; EXPECT_EQ(Directory2, Commands[0].Directory) << ErrorMessage; @@ -142,7 +159,8 @@ static CompileCommand findCompileArgsInJsonDatabase(StringRef FileName, StringRef JSONDatabase, std::string &ErrorMessage) { std::unique_ptr Database( - JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage)); + JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage, + JSONCommandLineSyntax::Gnu)); if (!Database) return CompileCommand(); std::vector Commands = Database->getCompileCommands(FileName); -- 2.7.4