From: Francis Visoiu Mistrih Date: Mon, 30 Sep 2019 16:05:12 +0000 (-0700) Subject: [dsymutil] Add support for linking remarks X-Git-Tag: llvmorg-11-init~5209 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=1ee18f275e60462d8c5e149fc74eaea9c671d4a8;p=platform%2Fupstream%2Fllvm.git [dsymutil] Add support for linking remarks This adds support to dsymutil for linking remark files and placing them in the final .dSYM bundle. The result will be placed in: * a.out.dSYM/Contents/Resources/Remarks/a.out or * a.out.dSYM/Contents/Resources/Remarks/a.out- for universal binaries When multi-threaded, this runs a third thread which loops over all the object files and parses remarks as it finds __remarks sections. Testing this involves running dsymutil on pre-built binaries and object files, then running llvm-bcanalyzer on the final result to check for remarks. Differential Revision: https://reviews.llvm.org/D69142 --- diff --git a/llvm/test/tools/dsymutil/Inputs/basic1.c b/llvm/test/tools/dsymutil/Inputs/basic1.c index cedf83ae..a41edc9 100644 --- a/llvm/test/tools/dsymutil/Inputs/basic1.c +++ b/llvm/test/tools/dsymutil/Inputs/basic1.c @@ -19,6 +19,13 @@ Archive compilation (after basic compilation): ar -q libbasic.a basic2.macho.x86_64.o basic3.macho.x86_64.o clang basic1.macho.x86_64.o -lbasic -o basic-archive.macho.x86_64 -Wl,-dead_strip -L. + + Remarks compilation: + for FILE in basic1.c basic2.c basic3.c; do + clang -gline-tables-only -c $FILE -fsave-optimization-record=bitstream -foptimization-remarks-file=/tmp/${FILE%.c}.macho.remarks.x86_64.opt.bitstream -mllvm -remarks-section -o ${FILE%.c}.macho.remarks.x86_64.o + done + clang basic1.macho.remarks.x86_64.o basic2.macho.remarks.x86_64.o basic3.macho.remarks.x86_64.o -o basic.macho.remarks.x86_64 -Wl,-dead_strip + */ int foo(int); diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/basic.macho.remarks.x86_64 b/llvm/test/tools/dsymutil/Inputs/remarks/basic.macho.remarks.x86_64 new file mode 100755 index 0000000..59e53c8 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/basic.macho.remarks.x86_64 differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/basic1.macho.remarks.x86_64.o b/llvm/test/tools/dsymutil/Inputs/remarks/basic1.macho.remarks.x86_64.o new file mode 100644 index 0000000..f8aa441 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/basic1.macho.remarks.x86_64.o differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/basic1.macho.remarks.x86_64.opt.bitstream b/llvm/test/tools/dsymutil/Inputs/remarks/basic1.macho.remarks.x86_64.opt.bitstream new file mode 100644 index 0000000..ac700c3 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/basic1.macho.remarks.x86_64.opt.bitstream differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/basic2.macho.remarks.x86_64.o b/llvm/test/tools/dsymutil/Inputs/remarks/basic2.macho.remarks.x86_64.o new file mode 100644 index 0000000..880926b Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/basic2.macho.remarks.x86_64.o differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/basic2.macho.remarks.x86_64.opt.bitstream b/llvm/test/tools/dsymutil/Inputs/remarks/basic2.macho.remarks.x86_64.opt.bitstream new file mode 100644 index 0000000..ee71291 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/basic2.macho.remarks.x86_64.opt.bitstream differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/basic3.macho.remarks.x86_64.o b/llvm/test/tools/dsymutil/Inputs/remarks/basic3.macho.remarks.x86_64.o new file mode 100644 index 0000000..9508c97 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/basic3.macho.remarks.x86_64.o differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/basic3.macho.remarks.x86_64.opt.bitstream b/llvm/test/tools/dsymutil/Inputs/remarks/basic3.macho.remarks.x86_64.opt.bitstream new file mode 100644 index 0000000..e29471c Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/basic3.macho.remarks.x86_64.opt.bitstream differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.i386.opt.bitstream b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.i386.opt.bitstream new file mode 100644 index 0000000..3abbc03 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.i386.opt.bitstream differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86 b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86 new file mode 100755 index 0000000..a0ebc45 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86 differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86.c b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86.c new file mode 100644 index 0000000..8153fc2 --- /dev/null +++ b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86.c @@ -0,0 +1,14 @@ +/* This file was used to generate: + * - fat.macho.remarks.x86.o + * - fat.macho.remarks.x86.opt.bitstream + * - fat.macho.remarks.x86 + */ + +/* for ARCH in x86_64 x86_64h i386 + * do + * clang -gline-tables-only -c fat.macho.remarks.x86.c -fsave-optimization-record=bitstream -foptimization-record-file=fat.macho.remarks."$ARCH".opt.bitstream -mllvm -remarks-section -arch "$ARCH" + * done + * lipo -create -output fat.macho.remarks.x86.o fat.macho.remarks.x86_64.o fat.macho.remarks.x86_64h.o fat.macho.remarks.i386.o + * clang -gline-tables-only fat.macho.remarks.x86.o -arch x86_64 -arch x86_64h -arch i386 -o fat.macho.remarks.x86 + */ +int main(void) { return 0; } diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86.o b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86.o new file mode 100644 index 0000000..2f5b8f3 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86.o differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86.opt.bitstream b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86.opt.bitstream new file mode 100644 index 0000000..72e5999 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86.opt.bitstream differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86_64.opt.bitstream b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86_64.opt.bitstream new file mode 100644 index 0000000..c710786 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86_64.opt.bitstream differ diff --git a/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86_64h.opt.bitstream b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86_64h.opt.bitstream new file mode 100644 index 0000000..c710786 Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/remarks/fat.macho.remarks.x86_64h.opt.bitstream differ diff --git a/llvm/test/tools/dsymutil/X86/basic-linking-bundle.test b/llvm/test/tools/dsymutil/X86/basic-linking-bundle.test index 11c2130..650ed1a 100644 --- a/llvm/test/tools/dsymutil/X86/basic-linking-bundle.test +++ b/llvm/test/tools/dsymutil/X86/basic-linking-bundle.test @@ -7,6 +7,9 @@ RUN: dsymutil -oso-prepend-path=%p/.. %t/basic.macho.x86_64 Check that the object file in the bundle exists and is sane: RUN: llvm-dwarfdump -a %t/basic.macho.x86_64.dSYM/Contents/Resources/DWARF/basic.macho.x86_64 | FileCheck %S/basic-linking-x86.test +Check that we don't create an empty Remarks directory if there are no remarks. +RUN: not ls %t/basic.macho.x86_64.dSYM/Contents/Resources/Remarks + Check that llvm-dwarfdump -a recognizes the bundle as a dSYM: RUN: llvm-dwarfdump -a %t/basic.macho.x86_64.dSYM | FileCheck %S/basic-linking-x86.test diff --git a/llvm/test/tools/dsymutil/X86/remarks-linking-bundle.test b/llvm/test/tools/dsymutil/X86/remarks-linking-bundle.test new file mode 100644 index 0000000..92849b1 --- /dev/null +++ b/llvm/test/tools/dsymutil/X86/remarks-linking-bundle.test @@ -0,0 +1,59 @@ +RUN: rm -rf %t +RUN: mkdir -p %t +RUN: cat %p/../Inputs/remarks/basic.macho.remarks.x86_64 > %t/basic.macho.remarks.x86_64 + +RUN: dsymutil -oso-prepend-path=%p/../Inputs -remarks-prepend-path=%p/../Inputs %t/basic.macho.remarks.x86_64 + +Check that the remark file in the bundle exists and is sane: +RUN: llvm-bcanalyzer %t/basic.macho.remarks.x86_64.dSYM/Contents/Resources/Remarks/basic.macho.remarks.x86_64 | FileCheck %s + +Now emit it in a different format: YAML. +RUN: dsymutil -remarks-output-format=yaml -oso-prepend-path=%p/../Inputs -remarks-prepend-path=%p/../Inputs %t/basic.macho.remarks.x86_64 +RUN: cat %t/basic.macho.remarks.x86_64.dSYM/Contents/Resources/Remarks/basic.macho.remarks.x86_64 | FileCheck %s --check-prefix=CHECK-YAML + +CHECK: %t/fat.macho.remarks.x86 + +RUN: dsymutil -oso-prepend-path=%p/../Inputs -remarks-prepend-path=%p/../Inputs %t/fat.macho.remarks.x86 + +Check that the remark files in the bundle exist and are all sane: +RUN: llvm-bcanalyzer %t/fat.macho.remarks.x86.dSYM/Contents/Resources/Remarks/fat.macho.remarks.x86-x86_64h | FileCheck %s +RUN: llvm-bcanalyzer %t/fat.macho.remarks.x86.dSYM/Contents/Resources/Remarks/fat.macho.remarks.x86-x86_64 | FileCheck %s +RUN: llvm-bcanalyzer %t/fat.macho.remarks.x86.dSYM/Contents/Resources/Remarks/fat.macho.remarks.x86-i386 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-i386 + +CHECK: HELP: -oso-prepend-path HELP: -o HELP: -papertrail +HELP: -remarks-output-format +HELP: -remarks-prepend-path HELP: -symbol-map HELP: -symtab HELP: -toolchain diff --git a/llvm/tools/dsymutil/CMakeLists.txt b/llvm/tools/dsymutil/CMakeLists.txt index f88e6db..489ad9b 100644 --- a/llvm/tools/dsymutil/CMakeLists.txt +++ b/llvm/tools/dsymutil/CMakeLists.txt @@ -12,6 +12,7 @@ set(LLVM_LINK_COMPONENTS MC Object Option + Remarks Support Target ) diff --git a/llvm/tools/dsymutil/DwarfLinker.cpp b/llvm/tools/dsymutil/DwarfLinker.cpp index e835ce1..624c343 100644 --- a/llvm/tools/dsymutil/DwarfLinker.cpp +++ b/llvm/tools/dsymutil/DwarfLinker.cpp @@ -63,6 +63,8 @@ #include "llvm/Object/MachO.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Object/SymbolicFile.h" +#include "llvm/Remarks/RemarkFormat.h" +#include "llvm/Remarks/RemarkLinker.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" @@ -2540,6 +2542,42 @@ static Error copySwiftInterfaces( return Error::success(); } +static Error emitRemarks(const LinkOptions &Options, StringRef BinaryPath, + StringRef ArchName, + const remarks::RemarkLinker &RL) { + // Make sure we don't create the directories and the file if there is nothing + // to serialize. + if (RL.empty()) + return Error::success(); + + SmallString<128> InputPath; + SmallString<128> Path; + // Create the "Remarks" directory in the "Resources" directory. + sys::path::append(Path, *Options.ResourceDir, "Remarks"); + if (std::error_code EC = sys::fs::create_directories(Path.str(), true, + sys::fs::perms::all_all)) + return errorCodeToError(EC); + + // Append the file name. + // For fat binaries, also append a dash and the architecture name. + sys::path::append(Path, sys::path::filename(BinaryPath)); + if (Options.NumDebugMaps > 1) { + // More than one debug map means we have a fat binary. + Path += '-'; + Path += ArchName; + } + + std::error_code EC; + raw_fd_ostream OS(Options.NoOutput ? "-" : Path.str(), EC, sys::fs::OF_None); + if (EC) + return errorCodeToError(EC); + + if (Error E = RL.serialize(OS, Options.RemarksFormat)) + return E; + + return Error::success(); +} + bool DwarfLinker::link(const DebugMap &Map) { if (!createStreamer(Map.getTriple(), OutFile)) return false; @@ -2803,6 +2841,18 @@ bool DwarfLinker::link(const DebugMap &Map) { } }; + remarks::RemarkLinker RL; + if (!Options.RemarksPrependPath.empty()) + RL.setExternalFilePrependPath(Options.RemarksPrependPath); + auto RemarkLinkLambda = [&](size_t i) { + // Link remarks from one object file. + auto &LinkContext = ObjectContexts[i]; + if (const object::ObjectFile *Obj = LinkContext.ObjectFile) + if (Error E = RL.link(*Obj)) + return E; + return Error(Error::success()); + }; + auto AnalyzeAll = [&]() { for (unsigned i = 0, e = NumObjects; i != e; ++i) { AnalyzeLambda(i); @@ -2828,6 +2878,26 @@ bool DwarfLinker::link(const DebugMap &Map) { EmitLambda(); }; + auto EmitRemarksLambda = [&]() { + StringRef ArchName = Map.getTriple().getArchName(); + return emitRemarks(Options, Map.getBinaryPath(), ArchName, RL); + }; + + // Instead of making error handling a lot more complicated using futures, + // write to one llvm::Error instance if something went wrong. + // We're assuming RemarkLinkAllError is alive longer than the thread + // executing RemarkLinkAll. + auto RemarkLinkAll = [&](Error &RemarkLinkAllError) { + // Allow assigning to the error only within the lambda. + ErrorAsOutParameter EAO(&RemarkLinkAllError); + for (unsigned i = 0, e = NumObjects; i != e; ++i) + if ((RemarkLinkAllError = RemarkLinkLambda(i))) + return; + + if ((RemarkLinkAllError = EmitRemarksLambda())) + return; + }; + // To limit memory usage in the single threaded case, analyze and clone are // run sequentially so the LinkContext is freed after processing each object // in endDebugObject. @@ -2835,13 +2905,29 @@ bool DwarfLinker::link(const DebugMap &Map) { for (unsigned i = 0, e = NumObjects; i != e; ++i) { AnalyzeLambda(i); CloneLambda(i); + + if (Error E = RemarkLinkLambda(i)) + return error(toString(std::move(E))); } EmitLambda(); + + if (Error E = EmitRemarksLambda()) + return error(toString(std::move(E))); + } else { - ThreadPool pool(2); + // This should not be constructed on the single-threaded path to avoid fatal + // errors from unchecked llvm::Error objects. + Error RemarkLinkAllError = Error::success(); + + ThreadPool pool(3); pool.async(AnalyzeAll); pool.async(CloneAll); + pool.async(RemarkLinkAll, std::ref(RemarkLinkAllError)); pool.wait(); + + // Report errors from RemarkLinkAll, if any. + if (Error E = std::move(RemarkLinkAllError)) + return error(toString(std::move(E))); } if (Options.NoOutput) diff --git a/llvm/tools/dsymutil/LinkUtils.h b/llvm/tools/dsymutil/LinkUtils.h index 0cd5600..0f31334 100644 --- a/llvm/tools/dsymutil/LinkUtils.h +++ b/llvm/tools/dsymutil/LinkUtils.h @@ -12,6 +12,7 @@ #include "SymbolMap.h" #include "llvm/ADT/Twine.h" +#include "llvm/Remarks/RemarkFormat.h" #include "llvm/Support/WithColor.h" #include @@ -68,6 +69,21 @@ struct LinkOptions { /// Symbol map translator. SymbolMapTranslator Translator; + /// Fields used for linking and placing remarks into the .dSYM bundle. + /// @{ + + /// Number of debug maps processed in total. + unsigned NumDebugMaps = 0; + + /// -remarks-prepend-path: prepend a path to all the external remark file + /// paths found in remark metadata. + std::string RemarksPrependPath; + + /// The output format of the remarks. + remarks::Format RemarksFormat = remarks::Format::Bitstream; + + /// @} + LinkOptions() = default; }; diff --git a/llvm/tools/dsymutil/Options.td b/llvm/tools/dsymutil/Options.td index c2114c8..8eaa365a 100644 --- a/llvm/tools/dsymutil/Options.td +++ b/llvm/tools/dsymutil/Options.td @@ -144,3 +144,15 @@ def: Separate<["-"], "j">, Alias, HelpText<"Alias for --num-threads">, Group; + +def remarks_prepend_path: Separate<["--", "-"], "remarks-prepend-path">, + MetaVarName<"">, + HelpText<"Specify a directory to prepend to the paths of the external remark files.">, + Group; +def: Joined<["--", "-"], "remarks-prepend-path=">, Alias; + +def remarks_output_format: Separate<["--", "-"], "remarks-output-format">, + MetaVarName<"">, + HelpText<"Specify the format to be used when serializing the linked remarks.">, + Group; +def: Joined<["--", "-"], "remarks-output-format=">, Alias; diff --git a/llvm/tools/dsymutil/dsymutil.cpp b/llvm/tools/dsymutil/dsymutil.cpp index 88a8593..2801178 100644 --- a/llvm/tools/dsymutil/dsymutil.cpp +++ b/llvm/tools/dsymutil/dsymutil.cpp @@ -266,6 +266,18 @@ static Expected getOptions(opt::InputArgList &Args) { if (getenv("RC_DEBUG_OPTIONS")) Options.PaperTrailWarnings = true; + if (opt::Arg *RemarksPrependPath = Args.getLastArg(OPT_remarks_prepend_path)) + Options.LinkOpts.RemarksPrependPath = RemarksPrependPath->getValue(); + + if (opt::Arg *RemarksOutputFormat = + Args.getLastArg(OPT_remarks_output_format)) { + if (Expected FormatOrErr = + remarks::parseFormat(RemarksOutputFormat->getValue())) + Options.LinkOpts.RemarksFormat = *FormatOrErr; + else + return FormatOrErr.takeError(); + } + if (Error E = verifyOptions(Options)) return std::move(E); return Options; @@ -507,6 +519,10 @@ int main(int argc, char **argv) { return 1; } + // Remember the number of debug maps that are being processed to decide how + // to name the remark files. + Options.LinkOpts.NumDebugMaps = DebugMapPtrsOrErr->size(); + if (Options.LinkOpts.Update) { // The debug map should be empty. Add one object file corresponding to // the input file.