From e9fb1506b83d001082dc535af12acf45050a527c Mon Sep 17 00:00:00 2001 From: Sam McCall Date: Tue, 23 Jun 2020 17:21:56 +0200 Subject: [PATCH] [clangd] Config: Fragments and parsing from YAML Summary: This is a piece from the design in https://tinyurl.com/clangd-config https://reviews.llvm.org/D82335 is a draft with all the bits linked together. It doesn't do anything yet, it's just a library + tests. It deliberately implements a minimal set of actual configuration options. Reviewers: kadircet, adamcz Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, usaxena95, cfe-commits Tags: #clang Differential Revision: https://reviews.llvm.org/D82386 --- clang-tools-extra/clangd/CMakeLists.txt | 1 + clang-tools-extra/clangd/ConfigFragment.h | 118 ++++++++++++ clang-tools-extra/clangd/ConfigYAML.cpp | 213 +++++++++++++++++++++ clang-tools-extra/clangd/unittests/CMakeLists.txt | 1 + .../clangd/unittests/ConfigYAMLTests.cpp | 160 ++++++++++++++++ 5 files changed, 493 insertions(+) create mode 100644 clang-tools-extra/clangd/ConfigFragment.h create mode 100644 clang-tools-extra/clangd/ConfigYAML.cpp create mode 100644 clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt index 3ff609a..de3dd81 100644 --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -36,6 +36,7 @@ add_clang_library(clangDaemon CollectMacros.cpp CompileCommands.cpp Compiler.cpp + ConfigYAML.cpp Diagnostics.cpp DraftStore.cpp ExpectedTypes.cpp diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h new file mode 100644 index 0000000..2cb6467 --- /dev/null +++ b/clang-tools-extra/clangd/ConfigFragment.h @@ -0,0 +1,118 @@ +//===--- ConfigFragment.h - Unit of user-specified configuration -*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Various clangd features have configurable behaviour (or can be disabled). +// The configuration system allows users to control this: +// - in a user config file, a project config file, via LSP, or via flags +// - specifying different settings for different files +// +// This file defines the config::Fragment structure which models one piece of +// configuration as obtained from a source like a file. +// +// This is distinct from how the config is interpreted (CompiledFragment), +// combined (Provider) and exposed to the rest of clangd (Config). +// +//===----------------------------------------------------------------------===// +// +// To add a new configuration option, you must: +// - add its syntactic form to Fragment +// - update ConfigYAML.cpp to parse it +// - add its semantic form to Config (in Config.h) +// - update ConfigCompile.cpp to map Fragment -> Config +// - make use of the option inside clangd +// - document the new option (config.md in the llvm/clangd-www repository) +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGFRAGMENT_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGFRAGMENT_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include +#include + +namespace clang { +namespace clangd { +namespace config { + +/// An entity written in config along, with its optional location in the file. +template struct Located { + Located(T Value, llvm::SMRange Range = {}) + : Range(Range), Value(std::move(Value)) {} + + llvm::SMRange Range; + T &operator->() { return Value; } + const T &operator->() const { return Value; } + T &operator*() { return Value; } + const T &operator*() const { return Value; } + +private: + T Value; +}; + +/// Used to report problems in parsing or interpreting a config. +/// Errors reflect structurally invalid config that should be user-visible. +/// Warnings reflect e.g. unknown properties that are recoverable. +using DiagnosticCallback = llvm::function_ref; + +/// A chunk of configuration obtained from a config file, LSP, or elsewhere. +struct Fragment { + /// Parses fragments from a YAML file (one from each --- delimited document). + /// Documents that contained fatal errors are omitted from the results. + /// BufferName is used for the SourceMgr and diagnostics. + static std::vector parseYAML(llvm::StringRef YAML, + llvm::StringRef BufferName, + DiagnosticCallback); + + /// These fields are not part of the user-specified configuration, but + /// instead are populated by the parser to describe the configuration source. + struct SourceInfo { + /// Retains a buffer of the original source this fragment was parsed from. + /// Locations within Located objects point into this SourceMgr. + /// Shared because multiple fragments are often parsed from one (YAML) file. + /// May be null, then all locations should be ignored. + std::shared_ptr Manager; + /// The start of the original source for this fragment. + /// Only valid if SourceManager is set. + llvm::SMLoc Location; + }; + SourceInfo Source; + + /// Conditions restrict when a Fragment applies. + /// + /// Each separate condition must match (combined with AND). + /// When one condition has multiple values, any may match (combined with OR). + /// + /// Conditions based on a file's path use the following form: + /// - if the fragment came from a project directory, the path is relative + /// - if the fragment is global (e.g. user config), the path is absolute + /// - paths always use forward-slashes (UNIX-style) + /// If no file is being processed, these conditions will not match. + struct ConditionBlock { + /// The file being processed must fully match a regular expression. + std::vector> PathMatch; + /// An unrecognized key was found while parsing the condition. + /// The condition will evaluate to false. + bool HasUnrecognizedCondition = false; + }; + ConditionBlock Condition; + + struct CompileFlagsBlock { + std::vector> Add; + } CompileFlags; +}; + +} // namespace config +} // namespace clangd +} // namespace clang + +#endif diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp new file mode 100644 index 0000000..ee3331d --- /dev/null +++ b/clang-tools-extra/clangd/ConfigYAML.cpp @@ -0,0 +1,213 @@ +//===--- ConfigYAML.cpp - Loading configuration fragments from YAML files -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ConfigFragment.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/YAMLParser.h" +#include + +namespace clang { +namespace clangd { +namespace config { +namespace { +using llvm::yaml::BlockScalarNode; +using llvm::yaml::MappingNode; +using llvm::yaml::Node; +using llvm::yaml::ScalarNode; +using llvm::yaml::SequenceNode; + +class Parser { + llvm::SourceMgr &SM; + +public: + Parser(llvm::SourceMgr &SM) : SM(SM) {} + + // Tries to parse N into F, returning false if it failed and we couldn't + // meaningfully recover (e.g. YAML syntax error broke the stream). + // The private parse() helpers follow the same pattern. + bool parse(Fragment &F, Node &N) { + DictParser Dict("Config", this); + Dict.handle("If", [&](Node &N) { return parse(F.Condition, N); }); + Dict.handle("CompileFlags", + [&](Node &N) { return parse(F.CompileFlags, N); }); + return Dict.parse(N); + } + +private: + bool parse(Fragment::ConditionBlock &F, Node &N) { + DictParser Dict("Condition", this); + Dict.unrecognized( + [&](llvm::StringRef) { F.HasUnrecognizedCondition = true; }); + Dict.handle("PathMatch", [&](Node &N) { + if (auto Values = scalarValues(N)) + F.PathMatch = std::move(*Values); + return !N.failed(); + }); + return Dict.parse(N); + } + + bool parse(Fragment::CompileFlagsBlock &F, Node &N) { + DictParser Dict("CompileFlags", this); + Dict.handle("Add", [&](Node &N) { + if (auto Values = scalarValues(N)) + F.Add = std::move(*Values); + return !N.failed(); + }); + return Dict.parse(N); + } + + // Helper for parsing mapping nodes (dictionaries). + // We don't use YamlIO as we want to control over unknown keys. + class DictParser { + llvm::StringRef Description; + std::vector>> Keys; + std::function Unknown; + Parser *Outer; + + public: + DictParser(llvm::StringRef Description, Parser *Outer) + : Description(Description), Outer(Outer) {} + + // Parse is called when Key is encountered, and passed the associated value. + // It should emit diagnostics if the value is invalid (e.g. wrong type). + // If Key is seen twice, Parse runs only once and an error is reported. + void handle(llvm::StringLiteral Key, std::function Parse) { + for (const auto &Entry : Keys) + assert(Entry.first != Key && "duplicate key handler"); + Keys.emplace_back(Key, std::move(Parse)); + } + + // Fallback is called when a Key is not matched by any handle(). + // A warning is also automatically emitted. + void unrecognized(std::function Fallback) { + Unknown = std::move(Fallback); + } + + // Process a mapping node and call handlers for each key/value pair. + bool parse(Node &N) const { + if (N.getType() != Node::NK_Mapping) { + Outer->error(Description + " should be a dictionary", N); + return false; + } + llvm::SmallSet Seen; + for (auto &KV : llvm::cast(N)) { + auto *K = KV.getKey(); + if (!K) // YAMLParser emitted an error. + return false; + auto Key = Outer->scalarValue(*K, "Dictionary key"); + if (!Key) + continue; + if (!Seen.insert(**Key).second) { + Outer->warning("Duplicate key " + **Key + " is ignored", *K); + continue; + } + auto *Value = KV.getValue(); + if (!Value) // YAMLParser emitted an error. + return false; + bool Matched = false; + for (const auto &Handler : Keys) { + if (Handler.first == **Key) { + if (!Handler.second(*Value)) + return false; + Matched = true; + break; + } + } + if (!Matched) { + Outer->warning("Unknown " + Description + " key " + **Key, *K); + if (Unknown) + Unknown(**Key); + } + } + return true; + } + }; + + // Try to parse a single scalar value from the node, warn on failure. + llvm::Optional> scalarValue(Node &N, + llvm::StringRef Desc) { + llvm::SmallString<256> Buf; + if (auto *S = llvm::dyn_cast(&N)) + return Located(S->getValue(Buf).str(), N.getSourceRange()); + if (auto *BS = llvm::dyn_cast(&N)) + return Located(BS->getValue().str(), N.getSourceRange()); + warning(Desc + " should be scalar", N); + return llvm::None; + } + + // Try to parse a list of single scalar values, or just a single value. + llvm::Optional>> scalarValues(Node &N) { + std::vector> Result; + if (auto *S = llvm::dyn_cast(&N)) { + llvm::SmallString<256> Buf; + Result.emplace_back(S->getValue(Buf).str(), N.getSourceRange()); + } else if (auto *S = llvm::dyn_cast(&N)) { + Result.emplace_back(S->getValue().str(), N.getSourceRange()); + } else if (auto *S = llvm::dyn_cast(&N)) { + for (auto &Child : *S) { + if (auto Value = scalarValue(Child, "List item")) + Result.push_back(std::move(*Value)); + } + } else { + warning("Expected scalar or list of scalars", N); + return llvm::None; + } + return Result; + } + + // Report a "hard" error, reflecting a config file that can never be valid. + void error(const llvm::Twine &Msg, const Node &N) { + SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Error, Msg, + N.getSourceRange()); + } + + // Report a "soft" error that could be caused by e.g. version skew. + void warning(const llvm::Twine &Msg, const Node &N) { + SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Warning, Msg, + N.getSourceRange()); + } +}; + +} // namespace + +std::vector Fragment::parseYAML(llvm::StringRef YAML, + llvm::StringRef BufferName, + DiagnosticCallback Diags) { + // The YAML document may contain multiple conditional fragments. + // The SourceManager is shared for all of them. + auto SM = std::make_shared(); + auto Buf = llvm::MemoryBuffer::getMemBufferCopy(YAML, BufferName); + // Adapt DiagnosticCallback to function-pointer interface. + // Callback receives both errors we emit and those from the YAML parser. + SM->setDiagHandler( + [](const llvm::SMDiagnostic &Diag, void *Ctx) { + (*reinterpret_cast(Ctx))(Diag); + }, + &Diags); + std::vector Result; + for (auto &Doc : llvm::yaml::Stream(*Buf, *SM)) { + if (Node *N = Doc.parseBlockNode()) { + Fragment Fragment; + Fragment.Source.Manager = SM; + Fragment.Source.Location = N->getSourceRange().Start; + if (Parser(*SM).parse(Fragment, *N)) + Result.push_back(std::move(Fragment)); + } + } + // Hack: stash the buffer in the SourceMgr to keep it alive. + // SM has two entries: "main" non-owning buffer, and ignored owning buffer. + SM->AddNewSourceBuffer(std::move(Buf), llvm::SMLoc()); + return Result; +} + +} // namespace config +} // namespace clangd +} // namespace clang diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt index 03e0966..7cfe379 100644 --- a/clang-tools-extra/clangd/unittests/CMakeLists.txt +++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -41,6 +41,7 @@ add_unittest(ClangdUnitTests ClangdTests CollectMacrosTests.cpp CompileCommandsTests.cpp CompilerTests.cpp + ConfigYAMLTests.cpp DexTests.cpp DiagnosticsTests.cpp DraftStoreTests.cpp diff --git a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp new file mode 100644 index 0000000..db94e7b --- /dev/null +++ b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp @@ -0,0 +1,160 @@ +//===-- ConfigYAMLTests.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Annotations.h" +#include "ConfigFragment.h" +#include "Matchers.h" +#include "Protocol.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/SourceMgr.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "gtest/internal/gtest-internal.h" + +namespace clang { +namespace clangd { +namespace config { +template void PrintTo(const Located &V, std::ostream *OS) { + *OS << ::testing::PrintToString(*V); +} + +namespace { +using ::testing::AllOf; +using ::testing::ElementsAre; +using ::testing::IsEmpty; + +MATCHER_P(Val, Value, "") { + if (*arg == Value) + return true; + *result_listener << "value is " << *arg; + return false; +} + +Position toPosition(llvm::SMLoc L, const llvm::SourceMgr &SM) { + auto LineCol = SM.getLineAndColumn(L); + Position P; + P.line = LineCol.first - 1; + P.character = LineCol.second - 1; + return P; +} + +Range toRange(llvm::SMRange R, const llvm::SourceMgr &SM) { + return Range{toPosition(R.Start, SM), toPosition(R.End, SM)}; +} + +struct CapturedDiags { + std::function callback() { + return [this](const llvm::SMDiagnostic &D) { + Diagnostics.emplace_back(); + Diag &Out = Diagnostics.back(); + Out.Message = D.getMessage().str(); + Out.Kind = D.getKind(); + Out.Pos.line = D.getLineNo() - 1; + Out.Pos.character = D.getColumnNo(); // Zero-based - bug in SourceMgr? + if (!D.getRanges().empty()) { + const auto &R = D.getRanges().front(); + Out.Range.emplace(); + Out.Range->start.line = Out.Range->end.line = Out.Pos.line; + Out.Range->start.character = R.first; + Out.Range->end.character = R.second; + } + }; + } + struct Diag { + std::string Message; + llvm::SourceMgr::DiagKind Kind; + Position Pos; + llvm::Optional Range; + + friend void PrintTo(const Diag &D, std::ostream *OS) { + *OS << (D.Kind == llvm::SourceMgr::DK_Error ? "error: " : "warning: ") + << D.Message << "@" << llvm::to_string(D.Pos); + } + }; + std::vector Diagnostics; +}; + +MATCHER_P(DiagMessage, M, "") { return arg.Message == M; } +MATCHER_P(DiagKind, K, "") { return arg.Kind == K; } +MATCHER_P(DiagPos, P, "") { return arg.Pos == P; } +MATCHER_P(DiagRange, R, "") { return arg.Range == R; } + +TEST(ParseYAML, SyntacticForms) { + CapturedDiags Diags; + const char *YAML = R"yaml( +If: + PathMatch: + - 'abc' +CompileFlags: { Add: [foo, bar] } +--- +CompileFlags: + Add: | + b + az + )yaml"; + auto Results = Fragment::parseYAML(YAML, "config.yaml", Diags.callback()); + EXPECT_THAT(Diags.Diagnostics, IsEmpty()); + ASSERT_EQ(Results.size(), 2u); + EXPECT_FALSE(Results.front().Condition.HasUnrecognizedCondition); + EXPECT_THAT(Results.front().Condition.PathMatch, ElementsAre(Val("abc"))); + EXPECT_THAT(Results.front().CompileFlags.Add, + ElementsAre(Val("foo"), Val("bar"))); + + EXPECT_THAT(Results.back().CompileFlags.Add, ElementsAre(Val("b\naz\n"))); +} + +TEST(ParseYAML, Locations) { + CapturedDiags Diags; + Annotations YAML(R"yaml( +If: + PathMatch: [['???bad***regex(((']] + )yaml"); + auto Results = + Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback()); + EXPECT_THAT(Diags.Diagnostics, IsEmpty()); + ASSERT_EQ(Results.size(), 1u); + ASSERT_NE(Results.front().Source.Manager, nullptr); + EXPECT_EQ(toRange(Results.front().Condition.PathMatch.front().Range, + *Results.front().Source.Manager), + YAML.range()); +} + +TEST(ParseYAML, Diagnostics) { + CapturedDiags Diags; + Annotations YAML(R"yaml( +If: + [[UnknownCondition]]: "foo" +CompileFlags: + Add: 'first' +--- +CompileFlags: {^ +)yaml"); + auto Results = + Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback()); + + ASSERT_THAT( + Diags.Diagnostics, + ElementsAre(AllOf(DiagMessage("Unknown Condition key UnknownCondition"), + DiagKind(llvm::SourceMgr::DK_Warning), + DiagPos(YAML.range().start), DiagRange(YAML.range())), + AllOf(DiagMessage("Unexpected token. Expected Key, Flow " + "Entry, or Flow Mapping End."), + DiagKind(llvm::SourceMgr::DK_Error), + DiagPos(YAML.point()), DiagRange(llvm::None)))); + + ASSERT_EQ(Results.size(), 2u); + EXPECT_THAT(Results.front().CompileFlags.Add, ElementsAre(Val("first"))); + EXPECT_TRUE(Results.front().Condition.HasUnrecognizedCondition); + EXPECT_THAT(Results.back().CompileFlags.Add, IsEmpty()); +} + +} // namespace +} // namespace config +} // namespace clangd +} // namespace clang -- 2.7.4