From fcab930cd32e6d977816fa7094a0de0bc0d7dd1f Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Fri, 10 Mar 2023 21:21:09 -0800 Subject: [PATCH] [clang][deps] Handle response files in dep scanner Extract the code the driver uses to expand response files and reuse it in the dependency scanner. rdar://106155880 Differential Revision: https://reviews.llvm.org/D145838 --- clang/include/clang/Basic/DiagnosticDriverKinds.td | 3 ++ clang/include/clang/Driver/Driver.h | 10 ++++ clang/lib/Driver/Driver.cpp | 55 ++++++++++++++++++++++ .../DependencyScanningWorker.cpp | 28 ++++++++--- clang/test/ClangScanDeps/response-file.c | 41 ++++++++++++++++ clang/tools/driver/driver.cpp | 44 ++--------------- 6 files changed, 134 insertions(+), 47 deletions(-) create mode 100644 clang/test/ClangScanDeps/response-file.c diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td index 3b32227..2c6bb0aa 100644 --- a/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -716,4 +716,7 @@ def err_drv_riscv_unsupported_with_linker_relaxation : Error< def err_drv_loongarch_invalid_mfpu_EQ : Error< "invalid argument '%0' to -mfpu=; must be one of: 64, 32, none, 0 (alias for none)">; + +def err_drv_expand_response_file : Error< + "failed to expand response file: %0">; } diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h index 691deb7..c85658d 100644 --- a/clang/include/clang/Driver/Driver.h +++ b/clang/include/clang/Driver/Driver.h @@ -808,6 +808,16 @@ llvm::StringRef getDriverMode(StringRef ProgName, ArrayRef Args); /// Checks whether the value produced by getDriverMode is for CL mode. bool IsClangCL(StringRef DriverMode); +/// Expand response files from a clang driver or cc1 invocation. +/// +/// \param Args The arguments that will be expanded. +/// \param ClangCLMode Whether clang is in CL mode. +/// \param Alloc Allocator for new arguments. +/// \param FS Filesystem to use when expanding files. +llvm::Error expandResponseFiles(SmallVectorImpl &Args, + bool ClangCLMode, llvm::BumpPtrAllocator &Alloc, + llvm::vfs::FileSystem *FS = nullptr); + } // end namespace driver } // end namespace clang diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index b991a18..6533d4e 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -6434,3 +6434,58 @@ llvm::StringRef clang::driver::getDriverMode(StringRef ProgName, } bool driver::IsClangCL(StringRef DriverMode) { return DriverMode.equals("cl"); } + +llvm::Error driver::expandResponseFiles(SmallVectorImpl &Args, + bool ClangCLMode, + llvm::BumpPtrAllocator &Alloc, + llvm::vfs::FileSystem *FS) { + // Parse response files using the GNU syntax, unless we're in CL mode. There + // are two ways to put clang in CL compatibility mode: ProgName is either + // clang-cl or cl, or --driver-mode=cl is on the command line. The normal + // command line parsing can't happen until after response file parsing, so we + // have to manually search for a --driver-mode=cl argument the hard way. + // Finally, our -cc1 tools don't care which tokenization mode we use because + // response files written by clang will tokenize the same way in either mode. + enum { Default, POSIX, Windows } RSPQuoting = Default; + for (const char *F : Args) { + if (strcmp(F, "--rsp-quoting=posix") == 0) + RSPQuoting = POSIX; + else if (strcmp(F, "--rsp-quoting=windows") == 0) + RSPQuoting = Windows; + } + + // Determines whether we want nullptr markers in Args to indicate response + // files end-of-lines. We only use this for the /LINK driver argument with + // clang-cl.exe on Windows. + bool MarkEOLs = ClangCLMode; + + llvm::cl::TokenizerCallback Tokenizer; + if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) + Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; + else + Tokenizer = &llvm::cl::TokenizeGNUCommandLine; + + if (MarkEOLs && Args.size() > 1 && StringRef(Args[1]).startswith("-cc1")) + MarkEOLs = false; + + llvm::cl::ExpansionContext ECtx(Alloc, Tokenizer); + ECtx.setMarkEOLs(MarkEOLs); + if (FS) + ECtx.setVFS(FS); + + if (llvm::Error Err = ECtx.expandResponseFiles(Args)) + return Err; + + // If -cc1 came from a response file, remove the EOL sentinels. + auto FirstArg = llvm::find_if(llvm::drop_begin(Args), + [](const char *A) { return A != nullptr; }); + if (FirstArg != Args.end() && StringRef(*FirstArg).startswith("-cc1")) { + // If -cc1 came from a response file, remove the EOL sentinels. + if (MarkEOLs) { + auto newEnd = std::remove(Args.begin(), Args.end(), nullptr); + Args.resize(newEnd - Args.begin()); + } + } + + return llvm::Error::success(); +} diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 10a26ab..5869418 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Basic/DiagnosticDriver.h" #include "clang/Basic/DiagnosticFrontend.h" #include "clang/CodeGen/ObjectFilePCHContainerOperations.h" #include "clang/Driver/Compilation.h" @@ -22,6 +23,8 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" #include "clang/Tooling/Tooling.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Error.h" #include "llvm/TargetParser/Host.h" #include @@ -366,16 +369,29 @@ llvm::Error DependencyScanningWorker::computeDependencies( } static bool forEachDriverJob( - ArrayRef Args, DiagnosticsEngine &Diags, FileManager &FM, + ArrayRef ArgStrs, DiagnosticsEngine &Diags, FileManager &FM, llvm::function_ref Callback) { + SmallVector Argv; + Argv.reserve(ArgStrs.size()); + for (const std::string &Arg : ArgStrs) + Argv.push_back(Arg.c_str()); + + llvm::vfs::FileSystem *FS = &FM.getVirtualFileSystem(); + std::unique_ptr Driver = std::make_unique( - Args[0], llvm::sys::getDefaultTargetTriple(), Diags, - "clang LLVM compiler", &FM.getVirtualFileSystem()); + Argv[0], llvm::sys::getDefaultTargetTriple(), Diags, + "clang LLVM compiler", FS); Driver->setTitle("clang_based_tool"); - std::vector Argv; - for (const std::string &Arg : Args) - Argv.push_back(Arg.c_str()); + llvm::BumpPtrAllocator Alloc; + bool CLMode = driver::IsClangCL( + driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1))); + + if (llvm::Error E = driver::expandResponseFiles(Argv, CLMode, Alloc, FS)) { + Diags.Report(diag::err_drv_expand_response_file) + << llvm::toString(std::move(E)); + return false; + } const std::unique_ptr Compilation( Driver->BuildCompilation(llvm::ArrayRef(Argv))); diff --git a/clang/test/ClangScanDeps/response-file.c b/clang/test/ClangScanDeps/response-file.c new file mode 100644 index 0000000..c08105c --- /dev/null +++ b/clang/test/ClangScanDeps/response-file.c @@ -0,0 +1,41 @@ +// Check that the scanner can handle a response file input. + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -format experimental-full -compilation-database %t/cdb.json > %t/deps.json + +// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s + +// CHECK: "command-line": [ +// CHECK: "-fsyntax-only" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "tu.c" +// CHECK: "-I" +// CHECK-NEXT: "include" +// CHECK: ], +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: "[[PREFIX]]/include/header.h" +// CHECK-NEXT: ] + +//--- cdb.json.template +[{ + "file": "DIR/t.c", + "directory": "DIR", + "command": "clang @DIR/args.txt" +}] + +//--- args.txt +@args_nested.txt +-fsyntax-only tu.c + +//--- args_nested.txt +-I include + +//--- include/header.h + +//--- tu.c +#include "header.h" diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp index 3ec2bcc..a05e095 100644 --- a/clang/tools/driver/driver.cpp +++ b/clang/tools/driver/driver.cpp @@ -391,55 +391,17 @@ int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContext) { const char *ProgName = ToolContext.NeedsPrependArg ? ToolContext.PrependArg : ToolContext.Path; - // Parse response files using the GNU syntax, unless we're in CL mode. There - // are two ways to put clang in CL compatibility mode: ProgName is either - // clang-cl or cl, or --driver-mode=cl is on the command line. The normal - // command line parsing can't happen until after response file parsing, so we - // have to manually search for a --driver-mode=cl argument the hard way. - // Finally, our -cc1 tools don't care which tokenization mode we use because - // response files written by clang will tokenize the same way in either mode. bool ClangCLMode = IsClangCL(getDriverMode(ProgName, llvm::ArrayRef(Args).slice(1))); - enum { Default, POSIX, Windows } RSPQuoting = Default; - for (const char *F : Args) { - if (strcmp(F, "--rsp-quoting=posix") == 0) - RSPQuoting = POSIX; - else if (strcmp(F, "--rsp-quoting=windows") == 0) - RSPQuoting = Windows; - } - // Determines whether we want nullptr markers in Args to indicate response - // files end-of-lines. We only use this for the /LINK driver argument with - // clang-cl.exe on Windows. - bool MarkEOLs = ClangCLMode; - - llvm::cl::TokenizerCallback Tokenizer; - if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) - Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; - else - Tokenizer = &llvm::cl::TokenizeGNUCommandLine; - - if (MarkEOLs && Args.size() > 1 && StringRef(Args[1]).startswith("-cc1")) - MarkEOLs = false; - llvm::cl::ExpansionContext ECtx(A, Tokenizer); - ECtx.setMarkEOLs(MarkEOLs); - if (llvm::Error Err = ECtx.expandResponseFiles(Args)) { + if (llvm::Error Err = expandResponseFiles(Args, ClangCLMode, A)) { llvm::errs() << toString(std::move(Err)) << '\n'; return 1; } - // Handle -cc1 integrated tools, even if -cc1 was expanded from a response - // file. - auto FirstArg = llvm::find_if(llvm::drop_begin(Args), - [](const char *A) { return A != nullptr; }); - if (FirstArg != Args.end() && StringRef(*FirstArg).startswith("-cc1")) { - // If -cc1 came from a response file, remove the EOL sentinels. - if (MarkEOLs) { - auto newEnd = std::remove(Args.begin(), Args.end(), nullptr); - Args.resize(newEnd - Args.begin()); - } + // Handle -cc1 integrated tools. + if (Args.size() >= 2 && StringRef(Args[1]).startswith("-cc1")) return ExecuteCC1Tool(Args, ToolContext); - } // Handle options that need handling before the real command line parsing in // Driver::BuildCompilation() -- 2.7.4