From 477e4b974653f92960c0bf569d88da7baacef68a Mon Sep 17 00:00:00 2001 From: Stephen Kelly Date: Sun, 14 Mar 2021 16:18:54 +0000 Subject: [PATCH] [AST] Add generator for source location introspection Generate a json file containing descriptions of AST classes and their public accessors which return SourceLocation or SourceRange. Use the JSON file to generate a C++ API and implementation for accessing the source locations and method names for accessing them for a given AST node. This new API can be used to implement 'srcloc' output in clang-query: http://ce.steveire.com/z/m_kTIo The JSON file can also be used to generate bindings for other languages, such as Python and Javascript: https://steveire.wordpress.com/2019/04/30/the-future-of-ast-matching In this first version of this feature, only the accessors for Stmt classes are generated, not Decls, TypeLocs etc. Those can be added after this change is reviewed, as this change is mostly about infrastructure of these code generators. Also in this version, the platforms/cmake configurations are excluded as much as possible so that support can be added iteratively. Currently a break on any platform causes a revert of the entire feature. This way, the `OR WIN32` can be removed in a future commit and if it breaks the buildbots, only that commit gets reverted, making the entire process easier to manage. Differential Revision: https://reviews.llvm.org/D93164 --- clang/include/clang/Tooling/NodeIntrospection.h | 85 +++++++++ clang/lib/Tooling/CMakeLists.txt | 99 ++++++++++ clang/lib/Tooling/DumpTool/APIData.h | 32 ++++ clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp | 170 +++++++++++++++++ clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h | 48 +++++ clang/lib/Tooling/DumpTool/CMakeLists.txt | 16 ++ clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp | 139 ++++++++++++++ .../lib/Tooling/DumpTool/generate_cxx_src_locs.py | 201 +++++++++++++++++++++ clang/lib/Tooling/NodeIntrospection.cpp | 61 +++++++ clang/unittests/CMakeLists.txt | 1 + clang/unittests/Introspection/CMakeLists.txt | 25 +++ .../unittests/Introspection/IntrospectionTest.cpp | 100 ++++++++++ llvm/utils/gn/secondary/clang/lib/Tooling/BUILD.gn | 14 ++ .../secondary/clang/lib/Tooling/DumpTool/BUILD.gn | 20 ++ llvm/utils/gn/secondary/clang/unittests/BUILD.gn | 1 + .../clang/unittests/Introspection/BUILD.gn | 20 ++ 16 files changed, 1032 insertions(+) create mode 100644 clang/include/clang/Tooling/NodeIntrospection.h create mode 100644 clang/lib/Tooling/DumpTool/APIData.h create mode 100644 clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp create mode 100644 clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h create mode 100644 clang/lib/Tooling/DumpTool/CMakeLists.txt create mode 100644 clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp create mode 100755 clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py create mode 100644 clang/lib/Tooling/NodeIntrospection.cpp create mode 100644 clang/unittests/Introspection/CMakeLists.txt create mode 100644 clang/unittests/Introspection/IntrospectionTest.cpp create mode 100644 llvm/utils/gn/secondary/clang/lib/Tooling/DumpTool/BUILD.gn create mode 100644 llvm/utils/gn/secondary/clang/unittests/Introspection/BUILD.gn diff --git a/clang/include/clang/Tooling/NodeIntrospection.h b/clang/include/clang/Tooling/NodeIntrospection.h new file mode 100644 index 0000000..abaa58b --- /dev/null +++ b/clang/include/clang/Tooling/NodeIntrospection.h @@ -0,0 +1,85 @@ +//===- NodeIntrospection.h ------------------------------------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the implementation of the NodeIntrospection. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_NODEINTROSPECTION_H +#define LLVM_CLANG_TOOLING_NODEINTROSPECTION_H + +#include "clang/AST/ASTTypeTraits.h" +#include "clang/AST/DeclarationName.h" + +#include +#include + +namespace clang { + +class Stmt; + +namespace tooling { + +class LocationCall { +public: + enum LocationCallFlags { NoFlags, ReturnsPointer, IsCast }; + LocationCall(std::shared_ptr on, std::string name, + LocationCallFlags flags = NoFlags) + : m_on(on), m_name(name), m_flags(flags) {} + LocationCall(std::shared_ptr on, std::string name, + std::vector const &args, + LocationCallFlags flags = NoFlags) + : m_on(on), m_name(name), m_flags(flags) {} + + LocationCall *on() const { return m_on.get(); } + StringRef name() const { return m_name; } + std::vector const &args() const { return m_args; } + bool returnsPointer() const { return m_flags & ReturnsPointer; } + bool isCast() const { return m_flags & IsCast; } + +private: + std::shared_ptr m_on; + std::string m_name; + std::vector m_args; + LocationCallFlags m_flags; +}; + +class LocationCallFormatterCpp { +public: + static std::string format(LocationCall *Call); +}; + +namespace internal { +struct RangeLessThan { + bool operator()( + std::pair> const &LHS, + std::pair> const &RHS) const; +}; +} // namespace internal + +template >> +using UniqueMultiMap = std::set, Comp>; + +using SourceLocationMap = + UniqueMultiMap>; +using SourceRangeMap = + UniqueMultiMap, + internal::RangeLessThan>; + +struct NodeLocationAccessors { + SourceLocationMap LocationAccessors; + SourceRangeMap RangeAccessors; +}; + +namespace NodeIntrospection { +NodeLocationAccessors GetLocations(clang::Stmt const *Object); +NodeLocationAccessors GetLocations(clang::DynTypedNode const &Node); +} // namespace NodeIntrospection +} // namespace tooling +} // namespace clang +#endif diff --git a/clang/lib/Tooling/CMakeLists.txt b/clang/lib/Tooling/CMakeLists.txt index 7a58af5..c7fe324 100644 --- a/clang/lib/Tooling/CMakeLists.txt +++ b/clang/lib/Tooling/CMakeLists.txt @@ -8,10 +8,107 @@ add_subdirectory(Core) add_subdirectory(Inclusions) add_subdirectory(Refactoring) add_subdirectory(ASTDiff) +add_subdirectory(DumpTool) add_subdirectory(Syntax) add_subdirectory(DependencyScanning) add_subdirectory(Transformer) +find_package(Python3 COMPONENTS Interpreter) + +# Replace the last lib component of the current binary directory with include +string(FIND ${CMAKE_CURRENT_BINARY_DIR} "/lib/" PATH_LIB_START REVERSE) +if(PATH_LIB_START EQUAL -1) + message(FATAL_ERROR "Couldn't find lib component in binary directory") +endif() +math(EXPR PATH_LIB_END "${PATH_LIB_START}+5") +string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD) +string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} ${PATH_LIB_END} -1 PATH_TAIL) +string(CONCAT BINARY_INCLUDE_DIR ${PATH_HEAD} "/include/clang/" ${PATH_TAIL}) + +if (NOT Python3_EXECUTABLE + OR WIN32 + OR APPLE + OR GENERATOR_IS_MULTI_CONFIG + OR NOT LLVM_NATIVE_ARCH IN_LIST LLVM_TARGETS_TO_BUILD + ) + file(GENERATE OUTPUT ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc + CONTENT " +namespace clang { +namespace tooling { + +NodeLocationAccessors NodeIntrospection::GetLocations(clang::Stmt const *) { + return {}; +} +NodeLocationAccessors +NodeIntrospection::GetLocations(clang::DynTypedNode const &) { + return {}; +} +} // namespace tooling +} // namespace clang +" + ) +else() + # The generation of ASTNodeAPI.json takes a long time in a + # Debug build due to parsing AST.h. Disable the processing + # but setting CLANG_TOOLING_BUILD_AST_INTROSPECTION as an + # internal hidden setting to override. + # When the processing is disabled, a trivial/empty JSON + # file is generated by clang-ast-dump and generate_cxx_src_locs.py + # generates the same API, but with a trivial implementation. + option(CLANG_TOOLING_BUILD_AST_INTROSPECTION "Enable AST introspection" TRUE) + + set(skip_expensive_processing $,$>>) + + file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ASTTU.cpp + CONTENT " +#include +") + + add_custom_command( + COMMENT Generate ASTNodeAPI.json + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json + DEPENDS clang-ast-dump clang-resource-headers + COMMAND + $ + # Skip this in debug mode because parsing AST.h is too slow + --skip-processing=${skip_expensive_processing} + --astheader=${CMAKE_CURRENT_BINARY_DIR}/ASTTU.cpp + -I ${CMAKE_BINARY_DIR}/lib/clang/${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}/include + -I ${CMAKE_SOURCE_DIR}/../clang/include + -I ${CMAKE_BINARY_DIR}/tools/clang/include/ + -I ${CMAKE_BINARY_DIR}/include + -I ${CMAKE_SOURCE_DIR}/include + --json-output-path ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json + ) + + add_custom_target(run-ast-api-dump-tool + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json + ) + + add_custom_command( + COMMENT Generate NodeIntrospection.inc + OUTPUT ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json ${CMAKE_CURRENT_SOURCE_DIR}/DumpTool/generate_cxx_src_locs.py + COMMAND + ${CMAKE_COMMAND} -E make_directory + ${CMAKE_CURRENT_BINARY_DIR}/generated/ + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/DumpTool/generate_cxx_src_locs.py + --json-input-path ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json + --output-file NodeIntrospection.inc + --empty-implementation ${skip_expensive_processing} + COMMAND + ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/NodeIntrospection.inc + ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc + ) + + add_custom_target(run-ast-api-generate-tool + DEPENDS + ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc + ) +endif() + add_clang_library(clangTooling AllTUsExecution.cpp ArgumentsAdjusters.cpp @@ -27,6 +124,8 @@ add_clang_library(clangTooling Refactoring.cpp RefactoringCallbacks.cpp StandaloneExecution.cpp + NodeIntrospection.cpp + ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc Tooling.cpp DEPENDS diff --git a/clang/lib/Tooling/DumpTool/APIData.h b/clang/lib/Tooling/DumpTool/APIData.h new file mode 100644 index 0000000..0ec53f6 --- /dev/null +++ b/clang/lib/Tooling/DumpTool/APIData.h @@ -0,0 +1,32 @@ +//===- APIData.h ---------------------------------------------*- C++ -*----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_TOOLING_DUMPTOOL_APIDATA_H +#define LLVM_CLANG_LIB_TOOLING_DUMPTOOL_APIDATA_H + +#include +#include + +namespace clang { +namespace tooling { + +struct ClassData { + + bool isEmpty() const { + return ASTClassLocations.empty() && ASTClassRanges.empty(); + } + + std::vector ASTClassLocations; + std::vector ASTClassRanges; + // TODO: Extend this with locations available via typelocs etc. +}; + +} // namespace tooling +} // namespace clang + +#endif diff --git a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp new file mode 100644 index 0000000..ff279d9 --- /dev/null +++ b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp @@ -0,0 +1,170 @@ +//===- ASTSrcLocProcessor.cpp --------------------------------*- C++ -*----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ASTSrcLocProcessor.h" + +#include "clang/Frontend/CompilerInstance.h" +#include "llvm/Support/JSON.h" + +using namespace clang::tooling; +using namespace llvm; +using namespace clang::ast_matchers; + +ASTSrcLocProcessor::ASTSrcLocProcessor(StringRef JsonPath) + : JsonPath(JsonPath) { + + MatchFinder::MatchFinderOptions FinderOptions; + + Finder = std::make_unique(std::move(FinderOptions)); + Finder->addMatcher( + cxxRecordDecl( + isDefinition(), + isSameOrDerivedFrom( + // TODO: Extend this with other clades + namedDecl(hasName("clang::Stmt")).bind("nodeClade")), + optionally(isDerivedFrom(cxxRecordDecl().bind("derivedFrom")))) + .bind("className"), + this); +} + +std::unique_ptr +ASTSrcLocProcessor::createASTConsumer(clang::CompilerInstance &Compiler, + StringRef File) { + return Finder->newASTConsumer(); +} + +llvm::json::Object toJSON(llvm::StringMap> const &Obj) { + using llvm::json::toJSON; + + llvm::json::Object JsonObj; + for (const auto &Item : Obj) { + JsonObj[Item.first()] = Item.second; + } + return JsonObj; +} + +llvm::json::Object toJSON(llvm::StringMap const &Obj) { + using llvm::json::toJSON; + + llvm::json::Object JsonObj; + for (const auto &Item : Obj) { + JsonObj[Item.first()] = Item.second; + } + return JsonObj; +} + +llvm::json::Object toJSON(ClassData const &Obj) { + llvm::json::Object JsonObj; + + if (!Obj.ASTClassLocations.empty()) + JsonObj["sourceLocations"] = Obj.ASTClassLocations; + if (!Obj.ASTClassRanges.empty()) + JsonObj["sourceRanges"] = Obj.ASTClassRanges; + return JsonObj; +} + +llvm::json::Object toJSON(llvm::StringMap const &Obj) { + using llvm::json::toJSON; + + llvm::json::Object JsonObj; + for (const auto &Item : Obj) { + if (!Item.second.isEmpty()) + JsonObj[Item.first()] = ::toJSON(Item.second); + } + return JsonObj; +} + +void WriteJSON(std::string JsonPath, + llvm::StringMap const &ClassInheritance, + llvm::StringMap> const &ClassesInClade, + llvm::StringMap const &ClassEntries) { + llvm::json::Object JsonObj; + + using llvm::json::toJSON; + + JsonObj["classInheritance"] = ::toJSON(ClassInheritance); + JsonObj["classesInClade"] = ::toJSON(ClassesInClade); + JsonObj["classEntries"] = ::toJSON(ClassEntries); + + std::error_code EC; + llvm::raw_fd_ostream JsonOut(JsonPath, EC, llvm::sys::fs::F_Text); + if (EC) + return; + + llvm::json::Value JsonVal(std::move(JsonObj)); + JsonOut << formatv("{0:2}", JsonVal); +} + +void ASTSrcLocProcessor::generate() { + WriteJSON(JsonPath, ClassInheritance, ClassesInClade, ClassEntries); +} + +std::vector +CaptureMethods(std::string TypeString, const clang::CXXRecordDecl *ASTClass, + const MatchFinder::MatchResult &Result) { + + auto publicAccessor = [](auto... InnerMatcher) { + return cxxMethodDecl(isPublic(), parameterCountIs(0), isConst(), + InnerMatcher...); + }; + + auto BoundNodesVec = + match(findAll(publicAccessor(ofClass(equalsNode(ASTClass)), + returns(asString(TypeString))) + .bind("classMethod")), + *ASTClass, *Result.Context); + + std::vector Methods; + for (const auto &BN : BoundNodesVec) { + if (const auto *Node = BN.getNodeAs("classMethod")) { + // Only record the getBeginLoc etc on Stmt etc, because it will call + // more-derived implementations pseudo-virtually. + if ((ASTClass->getName() != "Stmt" && ASTClass->getName() != "Decl") && + (Node->getName() == "getBeginLoc" || Node->getName() == "getEndLoc" || + Node->getName() == "getSourceRange")) { + continue; + } + // Only record the getExprLoc on Expr, because it will call + // more-derived implementations pseudo-virtually. + if (ASTClass->getName() != "Expr" && Node->getName() == "getExprLoc") { + continue; + } + Methods.push_back(Node->getName().str()); + } + } + return Methods; +} + +void ASTSrcLocProcessor::run(const MatchFinder::MatchResult &Result) { + + if (const auto *ASTClass = + Result.Nodes.getNodeAs("className")) { + + StringRef ClassName = ASTClass->getName(); + + ClassData CD; + + const auto *NodeClade = + Result.Nodes.getNodeAs("nodeClade"); + StringRef CladeName = NodeClade->getName(); + + if (const auto *DerivedFrom = + Result.Nodes.getNodeAs("derivedFrom")) + ClassInheritance[ClassName] = DerivedFrom->getName(); + + CD.ASTClassLocations = + CaptureMethods("class clang::SourceLocation", ASTClass, Result); + CD.ASTClassRanges = + CaptureMethods("class clang::SourceRange", ASTClass, Result); + + if (!CD.isEmpty()) { + ClassEntries[ClassName] = CD; + ClassesInClade[CladeName].push_back(ClassName); + } + } +} diff --git a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h new file mode 100644 index 0000000..0099475 --- /dev/null +++ b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h @@ -0,0 +1,48 @@ +//===- ASTSrcLocProcessor.h ---------------------------------*- C++ -*-----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_DUMPTOOL_ASTSRCLOCPROCESSOR_H +#define LLVM_CLANG_TOOLING_DUMPTOOL_ASTSRCLOCPROCESSOR_H + +#include "APIData.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "llvm/ADT/StringRef.h" +#include +#include +#include + +namespace clang { + +class CompilerInstance; + +namespace tooling { + +class ASTSrcLocProcessor : public ast_matchers::MatchFinder::MatchCallback { +public: + explicit ASTSrcLocProcessor(StringRef JsonPath); + + std::unique_ptr createASTConsumer(CompilerInstance &Compiler, + StringRef File); + + void generate(); + +private: + void run(const ast_matchers::MatchFinder::MatchResult &Result) override; + + llvm::StringMap ClassInheritance; + llvm::StringMap> ClassesInClade; + llvm::StringMap ClassEntries; + + std::string JsonPath; + std::unique_ptr Finder; +}; + +} // namespace tooling +} // namespace clang + +#endif diff --git a/clang/lib/Tooling/DumpTool/CMakeLists.txt b/clang/lib/Tooling/DumpTool/CMakeLists.txt new file mode 100644 index 0000000..712985b --- /dev/null +++ b/clang/lib/Tooling/DumpTool/CMakeLists.txt @@ -0,0 +1,16 @@ + +add_clang_executable(clang-ast-dump + ASTSrcLocProcessor.cpp + ClangSrcLocDump.cpp +) + +target_link_libraries(clang-ast-dump + PRIVATE + clangAST + clangASTMatchers + clangBasic + clangDriver + clangFrontend + clangSerialization + clangToolingCore +) diff --git a/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp b/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp new file mode 100644 index 0000000..28337ea --- /dev/null +++ b/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp @@ -0,0 +1,139 @@ +//===- ClangSrcLocDump.cpp ------------------------------------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Diagnostic.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Job.h" +#include "clang/Driver/Options.h" +#include "clang/Driver/Tool.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/JSON.h" + +#include "ASTSrcLocProcessor.h" + +using namespace clang::tooling; +using namespace clang; +using namespace llvm; + +static cl::list IncludeDirectories( + "I", cl::desc("Include directories to use while compiling"), + cl::value_desc("directory"), cl::Required, cl::OneOrMore, cl::Prefix); + +static cl::opt + AstHeaderFile("astheader", cl::desc("AST header to parse API from"), + cl::Required, cl::value_desc("AST header file")); + +static cl::opt + SkipProcessing("skip-processing", + cl::desc("Avoid processing the AST header file"), + cl::Required, cl::value_desc("bool")); + +static cl::opt JsonOutputPath("json-output-path", + cl::desc("json output path"), + cl::Required, + cl::value_desc("path")); + +class ASTSrcLocGenerationAction : public clang::ASTFrontendAction { +public: + ASTSrcLocGenerationAction() : Processor(JsonOutputPath) {} + + ~ASTSrcLocGenerationAction() { Processor.generate(); } + + std::unique_ptr + CreateASTConsumer(clang::CompilerInstance &Compiler, + llvm::StringRef File) override { + return Processor.createASTConsumer(Compiler, File); + } + +private: + ASTSrcLocProcessor Processor; +}; + +int main(int argc, const char **argv) { + + cl::ParseCommandLineOptions(argc, argv); + + if (SkipProcessing) { + std::error_code EC; + llvm::raw_fd_ostream JsonOut(JsonOutputPath, EC, llvm::sys::fs::F_Text); + if (EC) + return 1; + JsonOut << formatv("{0:2}", llvm::json::Value(llvm::json::Object())); + return 0; + } + + std::vector Args; + Args.push_back("-cc1"); + + llvm::transform(IncludeDirectories, std::back_inserter(Args), + [](const std::string &IncDir) { return "-I" + IncDir; }); + + Args.push_back("-fsyntax-only"); + Args.push_back(AstHeaderFile); + + std::vector Argv(Args.size(), nullptr); + llvm::transform(Args, Argv.begin(), + [](const std::string &Arg) { return Arg.c_str(); }); + + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + unsigned MissingArgIndex, MissingArgCount; + auto Opts = driver::getDriverOptTable(); + auto ParsedArgs = Opts.ParseArgs(llvm::makeArrayRef(Argv).slice(1), + MissingArgIndex, MissingArgCount); + ParseDiagnosticArgs(*DiagOpts, ParsedArgs); + TextDiagnosticPrinter DiagnosticPrinter(llvm::errs(), &*DiagOpts); + DiagnosticsEngine Diagnostics( + IntrusiveRefCntPtr(new DiagnosticIDs()), &*DiagOpts, + &DiagnosticPrinter, false); + + FileManager Files(FileSystemOptions(), vfs::getRealFileSystem()); + + auto Driver = std::make_unique( + "clang", llvm::sys::getDefaultTargetTriple(), Diagnostics, + "ast-api-dump-tool", &Files.getVirtualFileSystem()); + + auto Comp = Driver->BuildCompilation(llvm::makeArrayRef(Argv)); + if (!Comp) + return 1; + + const auto &Jobs = Comp->getJobs(); + if (Jobs.size() != 1 || !isa(*Jobs.begin())) { + SmallString<256> error_msg; + llvm::raw_svector_ostream error_stream(error_msg); + Jobs.Print(error_stream, "; ", true); + return 1; + } + + const auto &Cmd = cast(*Jobs.begin()); + const llvm::opt::ArgStringList &CC1Args = Cmd.getArguments(); + + auto Invocation = std::make_unique(); + CompilerInvocation::CreateFromArgs(*Invocation, CC1Args, Diagnostics); + + CompilerInstance Compiler(std::make_shared()); + Compiler.setInvocation(std::move(Invocation)); + + Compiler.createDiagnostics(&DiagnosticPrinter, false); + if (!Compiler.hasDiagnostics()) + return 1; + + Compiler.createSourceManager(Files); + + ASTSrcLocGenerationAction ScopedToolAction; + Compiler.ExecuteAction(ScopedToolAction); + + Files.clearStatCache(); + + return 0; +} diff --git a/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py b/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py new file mode 100755 index 0000000..fd02e4c --- /dev/null +++ b/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import sys +import json + +import argparse + +class Generator(object): + + implementationContent = '' + + def GeneratePrologue(self): + + self.implementationContent += \ + """ +/*===- Generated file -------------------------------------------*- C++ -*-===*\ +|* *| +|* Introspection of available AST node SourceLocations *| +|* *| +|* Automatically generated file, do not edit! *| +|* *| +\*===----------------------------------------------------------------------===*/ + +namespace clang { +namespace tooling { + +using LocationAndString = SourceLocationMap::value_type; +using RangeAndString = SourceRangeMap::value_type; +""" + + def GenerateBaseGetLocationsDeclaration(self, CladeName): + self.implementationContent += \ + """ +void GetLocationsImpl(std::shared_ptr const& Prefix, + clang::{0} const *Object, SourceLocationMap &Locs, + SourceRangeMap &Rngs); +""".format(CladeName) + + def GenerateSrcLocMethod(self, ClassName, ClassData): + + self.implementationContent += \ + """ +static void GetLocations{0}(std::shared_ptr const& Prefix, + clang::{0} const &Object, + SourceLocationMap &Locs, SourceRangeMap &Rngs) +{{ +""".format(ClassName) + + if 'sourceLocations' in ClassData: + for locName in ClassData['sourceLocations']: + self.implementationContent += \ + """ + Locs.insert(LocationAndString(Object.{0}(), + std::make_shared(Prefix, "{0}"))); +""".format(locName) + + self.implementationContent += '\n' + + if 'sourceRanges' in ClassData: + for rngName in ClassData['sourceRanges']: + self.implementationContent += \ + """ + Rngs.insert(RangeAndString(Object.{0}(), + std::make_shared(Prefix, "{0}"))); +""".format(rngName) + + self.implementationContent += '\n' + + self.implementationContent += '}\n' + + def GenerateFiles(self, OutputFile): + with open(os.path.join(os.getcwd(), + OutputFile), 'w') as f: + f.write(self.implementationContent) + + def GenerateBaseGetLocationsFunction(self, ASTClassNames, CladeName): + + MethodReturnType = 'NodeLocationAccessors' + + Signature = \ + 'GetLocations(clang::{0} const *Object)'.format(CladeName) + ImplSignature = \ + """ +GetLocationsImpl(std::shared_ptr const& Prefix, + clang::{0} const *Object, SourceLocationMap &Locs, + SourceRangeMap &Rngs) +""".format(CladeName) + + self.implementationContent += \ + 'void {0} {{ GetLocations{1}(Prefix, *Object, Locs, Rngs);'.format( + ImplSignature, + CladeName) + + for ASTClassName in ASTClassNames: + if ASTClassName != CladeName: + self.implementationContent += \ + """ +if (auto Derived = llvm::dyn_cast(Object)) {{ + GetLocations{0}(Prefix, *Derived, Locs, Rngs); +}} +""".format(ASTClassName) + + self.implementationContent += '}' + + self.implementationContent += \ + """ +{0} NodeIntrospection::{1} {{ + NodeLocationAccessors Result; + std::shared_ptr Prefix; + + GetLocationsImpl(Prefix, Object, Result.LocationAccessors, + Result.RangeAccessors); +""".format(MethodReturnType, + Signature) + + self.implementationContent += 'return Result; }' + + def GenerateDynNodeVisitor(self, CladeNames): + MethodReturnType = 'NodeLocationAccessors' + + Signature = \ + 'GetLocations(clang::DynTypedNode const &Node)' + + self.implementationContent += MethodReturnType \ + + ' NodeIntrospection::' + Signature + '{' + + for CladeName in CladeNames: + self.implementationContent += \ + """ + if (const auto *N = Node.get<{0}>()) + return GetLocations(const_cast<{0} *>(N));""".format(CladeName) + + self.implementationContent += '\nreturn {}; }' + + def GenerateEpilogue(self): + + self.implementationContent += ''' + } +} +''' + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument('--json-input-path', + help='Read API description from FILE', metavar='FILE') + parser.add_argument('--output-file', help='Generate output in FILEPATH', + metavar='FILEPATH') + parser.add_argument('--empty-implementation', + help='Generate empty implementation', + action="store", type=int) + + options = parser.parse_args() + + if options.empty_implementation: + with open(os.path.join(os.getcwd(), + options.output_file), 'w') as f: + f.write(""" +namespace clang { +namespace tooling { + +NodeLocationAccessors NodeIntrospection::GetLocations(clang::Stmt const *) { + return {}; +} +NodeLocationAccessors +NodeIntrospection::GetLocations(clang::DynTypedNode const &) { + return {}; +} +} // namespace tooling +} // namespace clang + """) + sys.exit(0) + + with open(options.json_input_path) as f: + jsonData = json.load(f) + + g = Generator() + + g.GeneratePrologue() + + if 'classesInClade' in jsonData: + for (CladeName, ClassNameData) in jsonData['classesInClade'].items(): + g.GenerateBaseGetLocationsDeclaration(CladeName) + + for (ClassName, ClassAccessors) in jsonData['classEntries'].items(): + if ClassAccessors: + g.GenerateSrcLocMethod(ClassName, ClassAccessors) + + for (CladeName, ClassNameData) in jsonData['classesInClade'].items(): + g.GenerateBaseGetLocationsFunction(ClassNameData, CladeName) + + g.GenerateDynNodeVisitor(jsonData['classesInClade'].keys()) + + g.GenerateEpilogue() + + g.GenerateFiles(options.output_file) + +if __name__ == '__main__': + main() diff --git a/clang/lib/Tooling/NodeIntrospection.cpp b/clang/lib/Tooling/NodeIntrospection.cpp new file mode 100644 index 0000000..89e8c19 --- /dev/null +++ b/clang/lib/Tooling/NodeIntrospection.cpp @@ -0,0 +1,61 @@ +//===- NodeIntrospection.h -----------------------------------*- C++ -*----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the implementation of the NodeIntrospection. +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/NodeIntrospection.h" + +#include "clang/AST/AST.h" + +namespace clang { + +namespace tooling { + +std::string LocationCallFormatterCpp::format(LocationCall *Call) { + SmallVector vec; + while (Call) { + vec.push_back(Call); + Call = Call->on(); + } + std::string result; + for (auto *VecCall : llvm::reverse(llvm::makeArrayRef(vec).drop_front())) { + result += + (VecCall->name() + "()" + (VecCall->returnsPointer() ? "->" : ".")) + .str(); + } + result += (vec.back()->name() + "()").str(); + return result; +} + +namespace internal { +bool RangeLessThan::operator()( + std::pair> const &LHS, + std::pair> const &RHS) const { + if (!LHS.first.isValid() || !RHS.first.isValid()) + return false; + + if (LHS.first.getBegin() < RHS.first.getBegin()) + return true; + else if (LHS.first.getBegin() != RHS.first.getBegin()) + return false; + + if (LHS.first.getEnd() < RHS.first.getEnd()) + return true; + else if (LHS.first.getEnd() != RHS.first.getEnd()) + return false; + + return LHS.second->name() < RHS.second->name(); +} +} // namespace internal + +} // namespace tooling +} // namespace clang + +#include "clang/Tooling/NodeIntrospection.inc" diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt index bb635df..4e08739 100644 --- a/clang/unittests/CMakeLists.txt +++ b/clang/unittests/CMakeLists.txt @@ -29,6 +29,7 @@ add_subdirectory(ASTMatchers) add_subdirectory(AST) add_subdirectory(CrossTU) add_subdirectory(Tooling) +add_subdirectory(Introspection) add_subdirectory(Format) add_subdirectory(Frontend) add_subdirectory(Rewrite) diff --git a/clang/unittests/Introspection/CMakeLists.txt b/clang/unittests/Introspection/CMakeLists.txt new file mode 100644 index 0000000..3dd8aec --- /dev/null +++ b/clang/unittests/Introspection/CMakeLists.txt @@ -0,0 +1,25 @@ +set(LLVM_LINK_COMPONENTS + FrontendOpenMP + Support + ) + +add_clang_unittest(IntrospectionTests + IntrospectionTest.cpp + ) + +clang_target_link_libraries(IntrospectionTests + PRIVATE + clangAST + clangASTMatchers + clangTooling + clangBasic + clangSerialization + clangFrontend + ) +target_compile_definitions(IntrospectionTests PRIVATE + SKIP_INTROSPECTION_GENERATION=$,$>> +) +target_link_libraries(IntrospectionTests + PRIVATE + LLVMTestingSupport +) diff --git a/clang/unittests/Introspection/IntrospectionTest.cpp b/clang/unittests/Introspection/IntrospectionTest.cpp new file mode 100644 index 0000000..25fc17a --- /dev/null +++ b/clang/unittests/Introspection/IntrospectionTest.cpp @@ -0,0 +1,100 @@ +//===- unittest/Introspection/IntrospectionTest.cpp ----------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Tests for AST location API introspection. +// +//===----------------------------------------------------------------------===// + +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/Tooling/NodeIntrospection.h" +#include "clang/Tooling/Tooling.h" +#include "gmock/gmock-matchers.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace clang; +using namespace clang::ast_matchers; +using namespace clang::tooling; + +using ::testing::UnorderedElementsAre; +using ::testing::Pair; + +#if SKIP_INTROSPECTION_GENERATION + +TEST(Introspection, NonFatalAPI) { + auto AST = buildASTFromCode("void foo() {} void bar() { foo(); }", "foo.cpp", + std::make_shared()); + auto &Ctx = AST->getASTContext(); + auto &TU = *Ctx.getTranslationUnitDecl(); + + auto BoundNodes = ast_matchers::match( + decl(hasDescendant( + callExpr(callee(functionDecl(hasName("foo")))).bind("fooCall"))), + TU, Ctx); + + EXPECT_EQ(BoundNodes.size(), 1u); + + auto *FooCall = BoundNodes[0].getNodeAs("fooCall"); + + auto result = NodeIntrospection::GetLocations(FooCall); + + EXPECT_EQ(result.LocationAccessors.size(), 0u); + EXPECT_EQ(result.RangeAccessors.size(), 0u); +} + +#else + +TEST(Introspection, SourceLocations) { + auto AST = buildASTFromCode("void foo() {} void bar() { foo(); }", "foo.cpp", + std::make_shared()); + auto &Ctx = AST->getASTContext(); + auto &TU = *Ctx.getTranslationUnitDecl(); + + auto BoundNodes = ast_matchers::match( + decl(hasDescendant( + callExpr(callee(functionDecl(hasName("foo")))).bind("fooCall"))), + TU, Ctx); + + EXPECT_EQ(BoundNodes.size(), 1u); + + auto *FooCall = BoundNodes[0].getNodeAs("fooCall"); + + auto result = NodeIntrospection::GetLocations(FooCall); + + std::map ExpectedLocations; + llvm::transform(result.LocationAccessors, + std::inserter(ExpectedLocations, ExpectedLocations.end()), + [](const auto &Accessor) { + return std::make_pair( + LocationCallFormatterCpp::format(Accessor.second.get()), + Accessor.first); + }); + + EXPECT_THAT(ExpectedLocations, + UnorderedElementsAre( + Pair("getBeginLoc()", FooCall->getBeginLoc()), + Pair("getEndLoc()", FooCall->getEndLoc()), + Pair("getExprLoc()", FooCall->getExprLoc()), + Pair("getRParenLoc()", FooCall->getRParenLoc()))); + + std::map ExpectedRanges; + llvm::transform(result.RangeAccessors, + std::inserter(ExpectedRanges, ExpectedRanges.end()), + [](const auto &Accessor) { + return std::make_pair( + LocationCallFormatterCpp::format(Accessor.second.get()), + Accessor.first); + }); + + EXPECT_THAT(ExpectedRanges, + UnorderedElementsAre( + Pair("getSourceRange()", FooCall->getSourceRange()))); +} +#endif diff --git a/llvm/utils/gn/secondary/clang/lib/Tooling/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Tooling/BUILD.gn index 5bc4241..3e9cf28 100644 --- a/llvm/utils/gn/secondary/clang/lib/Tooling/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/lib/Tooling/BUILD.gn @@ -1,7 +1,19 @@ +# FIXME: The cmake build runs DumpTool:clang-ast-dump to generate a json +# file and feeds it into this step in non-debug builds or if an option is set. +action("node_introspection_inc") { + script = "DumpTool/generate_cxx_src_locs.py" + outputs = [ "$target_gen_dir/clang/Tooling/NodeIntrospection.inc" ] + args = [ + "--empty-implementation=1", + "--output-file=" + rebase_path(outputs[0], root_build_dir), + ] +} + static_library("Tooling") { output_name = "clangTooling" configs += [ "//llvm/utils/gn/build:clang_code" ] deps = [ + ":node_introspection_inc", "//clang/include/clang/Driver:Options", "//clang/lib/AST", "//clang/lib/ASTMatchers", @@ -13,6 +25,7 @@ static_library("Tooling") { "//clang/lib/Rewrite", "//clang/lib/Tooling/Core", ] + include_dirs = [ target_gen_dir ] sources = [ "AllTUsExecution.cpp", "ArgumentsAdjusters.cpp", @@ -25,6 +38,7 @@ static_library("Tooling") { "GuessTargetAndModeCompilationDatabase.cpp", "InterpolatingCompilationDatabase.cpp", "JSONCompilationDatabase.cpp", + "NodeIntrospection.cpp", "Refactoring.cpp", "RefactoringCallbacks.cpp", "StandaloneExecution.cpp", diff --git a/llvm/utils/gn/secondary/clang/lib/Tooling/DumpTool/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Tooling/DumpTool/BUILD.gn new file mode 100644 index 0000000..7d9d211 --- /dev/null +++ b/llvm/utils/gn/secondary/clang/lib/Tooling/DumpTool/BUILD.gn @@ -0,0 +1,20 @@ +executable("clang-ast-dump") { + configs += [ "//llvm/utils/gn/build:clang_code" ] + deps = [ + "//clang/lib/AST", + "//clang/lib/ASTMatchers", + "//clang/lib/Basic", + "//clang/lib/Driver", + "//clang/lib/Format", + "//clang/lib/Frontend", + "//clang/lib/Lex", + "//clang/lib/Rewrite", + "//clang/lib/Serialization", + "//clang/lib/Tooling/Core", + ] + + sources = [ + "ASTSrcLocProcessor.cpp", + "ClangSrcLocDump.cpp", + ] +} diff --git a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn index 23f5d2d..1b8904c 100644 --- a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn @@ -12,6 +12,7 @@ group("unittests") { "Format:FormatTests", "Frontend:FrontendTests", "Index:IndexTests", + "Introspection:IntrospectionTests", "Lex:LexTests", "Rename:ClangRenameTests", "Rewrite:RewriteTests", diff --git a/llvm/utils/gn/secondary/clang/unittests/Introspection/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Introspection/BUILD.gn new file mode 100644 index 0000000..d42138c --- /dev/null +++ b/llvm/utils/gn/secondary/clang/unittests/Introspection/BUILD.gn @@ -0,0 +1,20 @@ +import("//llvm/utils/unittest/unittest.gni") + +unittest("IntrospectionTests") { + configs += [ "//llvm/utils/gn/build:clang_code" ] + deps = [ + "//clang/lib/AST", + "//clang/lib/ASTMatchers", + "//clang/lib/Basic", + "//clang/lib/Frontend", + "//clang/lib/Serialization", + "//clang/lib/Tooling", + "//llvm/lib/Support", + "//llvm/lib/Testing/Support", + ] + + + defines = [ "SKIP_INTROSPECTION_GENERATION" ] + + sources = [ "IntrospectionTest.cpp" ] +} -- 2.7.4