From 3bf14cebee461f4f6be4164f366a301e4f789f81 Mon Sep 17 00:00:00 2001 From: Edwin Vane Date: Thu, 22 Aug 2013 13:07:14 +0000 Subject: [PATCH] Introducing new tool clang-replace Introducing new tool 'clang-replace' that finds files containing serialized Replacements and applies those changes after deduplication and detecting conflicts. Currently the tool does not apply changes. It stops just after the deduplication and conflict report phase. Forthcoming patches will complete functionality. Both build systems updated for new tool. Includes a conflict test case. clang-replace added to Doxygen build. Differential Revision: http://llvm-reviews.chandlerc.com/D1424 llvm-svn: 189008 --- clang-tools-extra/CMakeLists.txt | 1 + clang-tools-extra/Makefile | 2 +- .../clang-replace/ApplyReplacements.cpp | 193 +++++++++++++++++++++ .../clang-replace/ApplyReplacements.h | 77 ++++++++ clang-tools-extra/clang-replace/CMakeLists.txt | 19 ++ clang-tools-extra/clang-replace/Makefile | 16 ++ .../clang-replace/tool/CMakeLists.txt | 17 ++ .../clang-replace/tool/ClangReplaceMain.cpp | 50 ++++++ clang-tools-extra/clang-replace/tool/Makefile | 28 +++ clang-tools-extra/docs/Doxyfile | 2 +- clang-tools-extra/test/CMakeLists.txt | 2 +- clang-tools-extra/test/clang-replace/conflict.cpp | 7 + .../test/clang-replace/conflict/common.h | 17 ++ .../test/clang-replace/conflict/expected.txt | 11 ++ .../test/clang-replace/conflict/file1.yaml | 16 ++ .../test/clang-replace/conflict/file2.yaml | 16 ++ .../test/clang-replace/conflict/file3.yaml | 8 + 17 files changed, 479 insertions(+), 3 deletions(-) create mode 100644 clang-tools-extra/clang-replace/ApplyReplacements.cpp create mode 100644 clang-tools-extra/clang-replace/ApplyReplacements.h create mode 100644 clang-tools-extra/clang-replace/CMakeLists.txt create mode 100644 clang-tools-extra/clang-replace/Makefile create mode 100644 clang-tools-extra/clang-replace/tool/CMakeLists.txt create mode 100644 clang-tools-extra/clang-replace/tool/ClangReplaceMain.cpp create mode 100644 clang-tools-extra/clang-replace/tool/Makefile create mode 100644 clang-tools-extra/test/clang-replace/conflict.cpp create mode 100644 clang-tools-extra/test/clang-replace/conflict/common.h create mode 100644 clang-tools-extra/test/clang-replace/conflict/expected.txt create mode 100644 clang-tools-extra/test/clang-replace/conflict/file1.yaml create mode 100644 clang-tools-extra/test/clang-replace/conflict/file2.yaml create mode 100644 clang-tools-extra/test/clang-replace/conflict/file3.yaml diff --git a/clang-tools-extra/CMakeLists.txt b/clang-tools-extra/CMakeLists.txt index fdd2426f..706f78d 100644 --- a/clang-tools-extra/CMakeLists.txt +++ b/clang-tools-extra/CMakeLists.txt @@ -1,6 +1,7 @@ add_subdirectory(remove-cstr-calls) add_subdirectory(tool-template) add_subdirectory(cpp11-migrate) +add_subdirectory(clang-replace) add_subdirectory(modularize) add_subdirectory(clang-tidy) diff --git a/clang-tools-extra/Makefile b/clang-tools-extra/Makefile index 33897d6..2bdbd1c 100644 --- a/clang-tools-extra/Makefile +++ b/clang-tools-extra/Makefile @@ -12,7 +12,7 @@ CLANG_LEVEL := ../.. include $(CLANG_LEVEL)/../../Makefile.config PARALLEL_DIRS := remove-cstr-calls tool-template modularize -DIRS := cpp11-migrate clang-tidy unittests +DIRS := cpp11-migrate clang-tidy clang-replace unittests include $(CLANG_LEVEL)/Makefile diff --git a/clang-tools-extra/clang-replace/ApplyReplacements.cpp b/clang-tools-extra/clang-replace/ApplyReplacements.cpp new file mode 100644 index 0000000..2b738268 --- /dev/null +++ b/clang-tools-extra/clang-replace/ApplyReplacements.cpp @@ -0,0 +1,193 @@ +//===-- Core/ApplyChangeDescriptions.cpp ----------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// \brief This file provides the implementation for finding and applying change +/// description files. +/// +//===----------------------------------------------------------------------===// +#include "ApplyReplacements.h" +#include "clang/Basic/LangOptions.h" +#include "clang/Basic/SourceManager.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace clang; + + +static void eatDiagnostics(const SMDiagnostic &, void *) {} + +namespace clang { +namespace replace { + +llvm::error_code +collectReplacementsFromDirectory(const llvm::StringRef Directory, + TUReplacements &TUs, + clang::DiagnosticsEngine &Diagnostics) { + using namespace llvm::sys::fs; + using namespace llvm::sys::path; + + error_code ErrorCode; + + for (recursive_directory_iterator I(Directory, ErrorCode), E; + I != E && !ErrorCode; I.increment(ErrorCode)) { + if (filename(I->path())[0] == '.') { + // Indicate not to descend into directories beginning with '.' + I.no_push(); + continue; + } + + if (extension(I->path()) != ".yaml") + continue; + + OwningPtr Out; + error_code BufferError = MemoryBuffer::getFile(I->path(), Out); + if (BufferError) { + errs() << "Error reading " << I->path() << ": " << BufferError.message() + << "\n"; + continue; + } + + yaml::Input YIn(Out->getBuffer()); + YIn.setDiagHandler(&eatDiagnostics); + tooling::TranslationUnitReplacements TU; + YIn >> TU; + if (YIn.error()) { + // File doesn't appear to be a header change description. Ignore it. + continue; + } + + // Only keep files that properly parse. + TUs.push_back(TU); + } + + return ErrorCode; +} + +/// \brief Dumps information for a sequence of conflicting Replacements. +/// +/// \param[in] File FileEntry for the file the conflicting Replacements are +/// for. +/// \param[in] ConflictingReplacements List of conflicting Replacements. +/// \param[in] SM SourceManager used for reporting. +static void reportConflict( + const FileEntry *File, + const llvm::ArrayRef ConflictingReplacements, + SourceManager &SM) { + FileID FID = SM.translateFile(File); + if (FID.isInvalid()) + FID = SM.createFileID(File, SourceLocation(), SrcMgr::C_User); + + // FIXME: Output something a little more user-friendly (e.g. unified diff?) + errs() << "The following changes conflict:\n"; + for (const tooling::Replacement *I = ConflictingReplacements.begin(), + *E = ConflictingReplacements.end(); + I != E; ++I) { + if (I->getLength() == 0) { + errs() << " Insert at " << SM.getLineNumber(FID, I->getOffset()) << ":" + << SM.getColumnNumber(FID, I->getOffset()) << " " + << I->getReplacementText() << "\n"; + } else { + if (I->getReplacementText().empty()) + errs() << " Remove "; + else + errs() << " Replace "; + + errs() << SM.getLineNumber(FID, I->getOffset()) << ":" + << SM.getColumnNumber(FID, I->getOffset()) << "-" + << SM.getLineNumber(FID, I->getOffset() + I->getLength() - 1) + << ":" + << SM.getColumnNumber(FID, I->getOffset() + I->getLength() - 1); + + if (I->getReplacementText().empty()) + errs() << "\n"; + else + errs() << " with \"" << I->getReplacementText() << "\"\n"; + } + } +} + +/// \brief Deduplicates and tests for conflicts among the replacements for each +/// file in \c Replacements. Any conflicts found are reported. +/// +/// \param[in,out] Replacements Container of all replacements grouped by file +/// to be deduplicated and checked for conflicts. +/// \param[in] SM SourceManager required for conflict reporting +/// +/// \returns \li true if conflicts were detected +/// \li false if no conflicts were detected +static bool deduplicateAndDetectConflicts(FileToReplacementsMap &Replacements, + SourceManager &SM) { + bool conflictsFound = false; + + for (FileToReplacementsMap::iterator I = Replacements.begin(), + E = Replacements.end(); + I != E; ++I) { + + const FileEntry *Entry = SM.getFileManager().getFile(I->getKey()); + if (!Entry) { + errs() << "Described file '" << I->getKey() + << "' doesn't exist. Ignoring...\n"; + continue; + } + + std::vector Conflicts; + tooling::deduplicate(I->getValue(), Conflicts); + + if (Conflicts.empty()) + continue; + + conflictsFound = true; + + errs() << "There are conflicting changes to " << I->getKey() << ":\n"; + + for (std::vector::const_iterator + ConflictI = Conflicts.begin(), + ConflictE = Conflicts.end(); + ConflictI != ConflictE; ++ConflictI) { + ArrayRef ConflictingReplacements( + &I->getValue()[ConflictI->getOffset()], ConflictI->getLength()); + reportConflict(Entry, ConflictingReplacements, SM); + } + } + + return conflictsFound; +} + +bool mergeAndDeduplicate(const TUReplacements &TUs, + FileToReplacementsMap &GroupedReplacements, + clang::DiagnosticsEngine &Diagnostics) { + + // FIXME: Use Diagnostics for output + + // Group all replacements by target file. + for (TUReplacements::const_iterator TUI = TUs.begin(), TUE = TUs.end(); + TUI != TUE; ++TUI) + for (std::vector::const_iterator + RI = TUI->Replacements.begin(), + RE = TUI->Replacements.end(); + RI != RE; ++RI) + GroupedReplacements[RI->getFilePath()].push_back(*RI); + + FileManager Files((FileSystemOptions())); + SourceManager SM(Diagnostics, Files); + + // Ask clang to deduplicate and report conflicts. + if (deduplicateAndDetectConflicts(GroupedReplacements, SM)) + return false; + + return true; +} + +} // end namespace replace +} // end namespace clang diff --git a/clang-tools-extra/clang-replace/ApplyReplacements.h b/clang-tools-extra/clang-replace/ApplyReplacements.h new file mode 100644 index 0000000..7b4608c --- /dev/null +++ b/clang-tools-extra/clang-replace/ApplyReplacements.h @@ -0,0 +1,77 @@ +//===-- Core/ApplyChangeDescriptions.h --------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// \brief This file provides the interface for finding and applying change +/// description files. +/// +//===----------------------------------------------------------------------===// + +#ifndef CPP11_MIGRATE_APPLYCHANGEDESCRIPTIONS_H +#define CPP11_MIGRATE_APPLYCHANGEDESCRIPTIONS_H + +#include "clang/Tooling/ReplacementsYaml.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/system_error.h" +#include + +namespace clang { + +class DiagnosticsEngine; + +namespace replace { + +/// \brief Collection of TranslationUnitReplacements. +typedef std::vector +TUReplacements; + +/// \brief Map mapping file name to Replacements targeting that file. +typedef llvm::StringMap > +FileToReplacementsMap; + +/// \brief Recursively descends through a directory structure rooted at \p +/// Directory and attempts to deserialize *.yaml files as +/// TranslationUnitReplacements. All docs that successfully deserialize are +/// added to \p TUs. +/// +/// Directories starting with '.' are ignored during traversal. +/// +/// \param[in] Directory Directory to begin search for serialized +/// TranslationUnitReplacements. +/// \param[out] TUs Collection of all found and deserialized +/// TranslationUnitReplacements. +/// \param[in] Diagnostics DiagnosticsEngine used for error output. +/// +/// \returns An error_code indicating success or failure in navigating the +/// directory structure. +llvm::error_code +collectReplacementsFromDirectory(const llvm::StringRef Directory, + TUReplacements &TUs, + clang::DiagnosticsEngine &Diagnostics); + +/// \brief Deduplicate, check for conflicts, and apply all Replacements stored +/// in \c TUs. If conflicts occur, no Replacements are applied. +/// +/// \param[in] TUs Collection of TranslationUnitReplacements to merge, +/// deduplicate, and test for conflicts. +/// \param[out] GroupedReplacements Container grouping all Replacements by the +/// file they target. +/// \param[in] Diagnostics DiagnosticsEngine used for error/warning output. +/// +/// \returns \li true If all changes were applied successfully. +/// \li false If there were conflicts. +bool mergeAndDeduplicate(const TUReplacements &TUs, + FileToReplacementsMap &GroupedReplacements, + clang::DiagnosticsEngine &Diagnostics); + +} // end namespace replace +} // end namespace clang + +#endif // CPP11_MIGRATE_APPLYCHANGEDESCRIPTIONS_H diff --git a/clang-tools-extra/clang-replace/CMakeLists.txt b/clang-tools-extra/clang-replace/CMakeLists.txt new file mode 100644 index 0000000..65b2aa3 --- /dev/null +++ b/clang-tools-extra/clang-replace/CMakeLists.txt @@ -0,0 +1,19 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + asmparser + bitreader + support + mc + ) + +add_clang_library(clangReplace + ApplyReplacements.cpp + ) +target_link_libraries(clangReplace + clangTooling + clangBasic + clangRewriteFrontend + ) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +add_subdirectory(tool) diff --git a/clang-tools-extra/clang-replace/Makefile b/clang-tools-extra/clang-replace/Makefile new file mode 100644 index 0000000..b337f39 --- /dev/null +++ b/clang-tools-extra/clang-replace/Makefile @@ -0,0 +1,16 @@ +##===- clang-replace/Makefile ------------------------------*- Makefile -*-===## +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +##===----------------------------------------------------------------------===## + +CLANG_LEVEL := ../../.. +LIBRARYNAME := clangReplace +include $(CLANG_LEVEL)/../../Makefile.config + +DIRS = tool + +include $(CLANG_LEVEL)/Makefile diff --git a/clang-tools-extra/clang-replace/tool/CMakeLists.txt b/clang-tools-extra/clang-replace/tool/CMakeLists.txt new file mode 100644 index 0000000..ff47688 --- /dev/null +++ b/clang-tools-extra/clang-replace/tool/CMakeLists.txt @@ -0,0 +1,17 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + asmparser + bitreader + support + mc + ) + +add_clang_executable(clang-replace + ClangReplaceMain.cpp + ) +target_link_libraries(clang-replace + clangReplace + ) + +install(TARGETS clang-replace + RUNTIME DESTINATION bin) diff --git a/clang-tools-extra/clang-replace/tool/ClangReplaceMain.cpp b/clang-tools-extra/clang-replace/tool/ClangReplaceMain.cpp new file mode 100644 index 0000000..f085ed8 --- /dev/null +++ b/clang-tools-extra/clang-replace/tool/ClangReplaceMain.cpp @@ -0,0 +1,50 @@ +//===-- ClangReplaceMain.cpp - Main file for clang-replace tool -----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// \brief This file provides the main function for the clang-replace tool. +/// +//===----------------------------------------------------------------------===// + +#include "ApplyReplacements.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "llvm/Support/CommandLine.h" + +using namespace llvm; +using namespace clang; +using namespace clang::replace; + +static cl::opt Directory(cl::Positional, cl::Required, + cl::desc("")); + +int main(int argc, char **argv) { + cl::ParseCommandLineOptions(argc, argv); + + IntrusiveRefCntPtr DiagOpts(new DiagnosticOptions()); + DiagnosticsEngine Diagnostics( + IntrusiveRefCntPtr(new DiagnosticIDs()), + DiagOpts.getPtr()); + + TUReplacements TUs; + + error_code ErrorCode = + collectReplacementsFromDirectory(Directory, TUs, Diagnostics); + + if (ErrorCode) { + errs() << "Trouble iterating over directory '" << Directory + << "': " << ErrorCode.message() << "\n"; + return false; + } + + FileToReplacementsMap GroupedReplacements; + if (mergeAndDeduplicate(TUs, GroupedReplacements, Diagnostics)) + return 0; + return 1; +} diff --git a/clang-tools-extra/clang-replace/tool/Makefile b/clang-tools-extra/clang-replace/tool/Makefile new file mode 100644 index 0000000..83787cf --- /dev/null +++ b/clang-tools-extra/clang-replace/tool/Makefile @@ -0,0 +1,28 @@ +##===- clang-replace/tool/Makefile -------------------------*- Makefile -*-===## +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +##===----------------------------------------------------------------------===## + +CLANG_LEVEL := ../../../.. +include $(CLANG_LEVEL)/../../Makefile.config + +TOOLNAME = clang-replace + +# No plugins, optimize startup time. +TOOL_NO_EXPORTS = 1 + +SOURCES = ClangReplaceMain.cpp + +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc mcparser option +USEDLIBS = clangReplace.a clangFormat.a clangTooling.a clangFrontend.a \ + clangSerialization.a clangDriver.a clangRewriteFrontend.a \ + clangRewriteCore.a clangParse.a clangSema.a clangAnalysis.a \ + clangAST.a clangASTMatchers.a clangEdit.a clangLex.a clangBasic.a + +include $(CLANG_LEVEL)/Makefile + +CPP.Flags += -I$(PROJ_SRC_DIR)/.. diff --git a/clang-tools-extra/docs/Doxyfile b/clang-tools-extra/docs/Doxyfile index 174b25b..1c5e058 100644 --- a/clang-tools-extra/docs/Doxyfile +++ b/clang-tools-extra/docs/Doxyfile @@ -648,7 +648,7 @@ WARN_LOGFILE = # directories like "/usr/src/myproject". Separate the files or directories # with spaces. -INPUT = ../cpp11-migrate +INPUT = ../cpp11-migrate ../clang-replace # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is diff --git a/clang-tools-extra/test/CMakeLists.txt b/clang-tools-extra/test/CMakeLists.txt index 4aa6105..49fd4b9 100644 --- a/clang-tools-extra/test/CMakeLists.txt +++ b/clang-tools-extra/test/CMakeLists.txt @@ -27,7 +27,7 @@ set(CLANG_TOOLS_TEST_DEPS clang clang-headers FileCheck count not # Individual tools we test. - remove-cstr-calls cpp11-migrate modularize clang-tidy + remove-cstr-calls clang-replace cpp11-migrate modularize clang-tidy # Unit tests ExtraToolsUnitTests diff --git a/clang-tools-extra/test/clang-replace/conflict.cpp b/clang-tools-extra/test/clang-replace/conflict.cpp new file mode 100644 index 0000000..5b79401 --- /dev/null +++ b/clang-tools-extra/test/clang-replace/conflict.cpp @@ -0,0 +1,7 @@ +// RUN: mkdir -p %T/conflict +// RUN: sed "s#\$(path)#%/S/conflict#" %S/conflict/file1.yaml > %T/conflict/file1.yaml +// RUN: sed "s#\$(path)#%/S/conflict#" %S/conflict/file2.yaml > %T/conflict/file2.yaml +// RUN: sed "s#\$(path)#%/S/conflict#" %S/conflict/file3.yaml > %T/conflict/file3.yaml +// RUN: sed "s#\$(path)#%/S/conflict#" %S/conflict/expected.txt > %T/conflict/expected.txt +// RUN: not clang-replace %T/conflict > %T/conflict/output.txt 2>&1 +// RUN: diff -b %T/conflict/output.txt %T/conflict/expected.txt diff --git a/clang-tools-extra/test/clang-replace/conflict/common.h b/clang-tools-extra/test/clang-replace/conflict/common.h new file mode 100644 index 0000000..630a39a --- /dev/null +++ b/clang-tools-extra/test/clang-replace/conflict/common.h @@ -0,0 +1,17 @@ +#ifndef COMMON_H +#define COMMON_H + +extern void ext(int (&)[5]); + +void func(int t) { + int ints[5]; + for (unsigned i = 0; i < 5; ++i) { + ints[i] = t; + } + + int *i = 0; + + ext(ints); +} + +#endif // COMMON_H diff --git a/clang-tools-extra/test/clang-replace/conflict/expected.txt b/clang-tools-extra/test/clang-replace/conflict/expected.txt new file mode 100644 index 0000000..32dddd3 --- /dev/null +++ b/clang-tools-extra/test/clang-replace/conflict/expected.txt @@ -0,0 +1,11 @@ +There are conflicting changes to $(path)/common.h: +The following changes conflict: + Replace 8:8-8:33 with "auto & i : ints" + Replace 8:8-8:33 with "int & elem : ints" +The following changes conflict: + Replace 9:5-9:11 with "elem" + Replace 9:5-9:11 with "i" +The following changes conflict: + Remove 12:3-12:14 + Insert at 12:12 (int*) + Replace 12:12-12:12 with "nullptr" diff --git a/clang-tools-extra/test/clang-replace/conflict/file1.yaml b/clang-tools-extra/test/clang-replace/conflict/file1.yaml new file mode 100644 index 0000000..1bb0a2a --- /dev/null +++ b/clang-tools-extra/test/clang-replace/conflict/file1.yaml @@ -0,0 +1,16 @@ +--- +MainSourceFile: "source1.cpp" +Replacements: + - FilePath: "$(path)/common.h" + Offset: 106 + Length: 26 + ReplacementText: "auto & i : ints" + - FilePath: "$(path)/common.h" + Offset: 140 + Length: 7 + ReplacementText: "i" + - FilePath: "$(path)/common.h" + Offset: 160 + Length: 12 + ReplacementText: "" +... diff --git a/clang-tools-extra/test/clang-replace/conflict/file2.yaml b/clang-tools-extra/test/clang-replace/conflict/file2.yaml new file mode 100644 index 0000000..fb48e3f --- /dev/null +++ b/clang-tools-extra/test/clang-replace/conflict/file2.yaml @@ -0,0 +1,16 @@ +--- +MainSourceFile: "source2.cpp" +Replacements: + - FilePath: "$(path)/common.h" + Offset: 106 + Length: 26 + ReplacementText: "int & elem : ints" + - FilePath: "$(path)/common.h" + Offset: 140 + Length: 7 + ReplacementText: "elem" + - FilePath: "$(path)/common.h" + Offset: 169 + Length: 1 + ReplacementText: "nullptr" +... diff --git a/clang-tools-extra/test/clang-replace/conflict/file3.yaml b/clang-tools-extra/test/clang-replace/conflict/file3.yaml new file mode 100644 index 0000000..59e71eb --- /dev/null +++ b/clang-tools-extra/test/clang-replace/conflict/file3.yaml @@ -0,0 +1,8 @@ +--- +MainSourceFile: "source1.cpp" +Replacements: + - FilePath: "$(path)/common.h" + Offset: 169 + Length: 0 + ReplacementText: "(int*)" +... -- 2.7.4