From: Francis Visoiu Mistrih Date: Tue, 30 Jul 2019 16:01:40 +0000 (+0000) Subject: [Remarks] Add two serialization modes for remarks: separate and standalone X-Git-Tag: llvmorg-11-init~13282 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=5ed3d146f87e3103ea5bbdb27bca8d6b1b1f714a;p=platform%2Fupstream%2Fllvm.git [Remarks] Add two serialization modes for remarks: separate and standalone The default mode is separate, where the metadata is serialized separately from the remarks. Another mode is the standalone mode, where the metadata is serialized before the remarks, on the same stream. llvm-svn: 367328 --- diff --git a/llvm/docs/Remarks.rst b/llvm/docs/Remarks.rst index c573eb2..f4e5d8f 100644 --- a/llvm/docs/Remarks.rst +++ b/llvm/docs/Remarks.rst @@ -147,6 +147,26 @@ Other tools that support remarks: .. option:: -opt-remarks-format= .. option:: -opt-remarks-with-hotness +Serialization modes +=================== + +There are two modes available for serializing remarks: + +``Separate`` + + In this mode, the remarks and the metadata are serialized separately. The + client is responsible for parsing the metadata first, then use the metadata + to correctly parse the remarks. + +``Standalone`` + + In this mode, the remarks and the metadata are serialized to the same + stream. The metadata will always come before the remarks. + + The compiler does not support emitting standalone remarks. This mode is + more suited for post-processing tools like linkers, that can merge the + remarks for one whole project. + .. _yamlremarks: YAML remarks diff --git a/llvm/include/llvm/Remarks/RemarkSerializer.h b/llvm/include/llvm/Remarks/RemarkSerializer.h index a8e083f..cea2afb 100644 --- a/llvm/include/llvm/Remarks/RemarkSerializer.h +++ b/llvm/include/llvm/Remarks/RemarkSerializer.h @@ -14,12 +14,23 @@ #define LLVM_REMARKS_REMARK_SERIALIZER_H #include "llvm/Remarks/Remark.h" +#include "llvm/Remarks/RemarkFormat.h" #include "llvm/Remarks/RemarkStringTable.h" #include "llvm/Support/raw_ostream.h" namespace llvm { namespace remarks { +enum class SerializerMode { + Separate, // A mode where the metadata is serialized separately from the + // remarks. Typically, this is used when the remarks need to be + // streamed to a side file and the metadata is embedded into the + // final result of the compilation. + Standalone // A mode where everything can be retrieved in the same + // file/buffer. Typically, this is used for storing remarks for + // later use. +}; + struct MetaSerializer; /// This is the base class for a remark serializer. @@ -27,11 +38,14 @@ struct MetaSerializer; struct RemarkSerializer { /// The open raw_ostream that the remark diagnostics are emitted to. raw_ostream &OS; + /// The serialization mode. + SerializerMode Mode; /// The string table containing all the unique strings used in the output. /// The table can be serialized to be consumed after the compilation. Optional StrTab; - RemarkSerializer(raw_ostream &OS) : OS(OS), StrTab() {} + RemarkSerializer(raw_ostream &OS, SerializerMode Mode) + : OS(OS), Mode(Mode), StrTab() {} /// This is just an interface. virtual ~RemarkSerializer() = default; @@ -57,12 +71,12 @@ struct MetaSerializer { /// Create a remark serializer. Expected> -createRemarkSerializer(Format RemarksFormat, raw_ostream &OS); +createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, raw_ostream &OS); /// Create a remark serializer that uses a pre-filled string table. Expected> -createRemarkSerializer(Format RemarksFormat, raw_ostream &OS, - remarks::StringTable StrTab); +createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS, remarks::StringTable StrTab); } // end namespace remarks } // end namespace llvm diff --git a/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h b/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h index 5d68bb4..f57ae4c 100644 --- a/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h +++ b/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h @@ -34,12 +34,15 @@ struct YAMLRemarkSerializer : public RemarkSerializer { /// The YAML streamer. yaml::Output YAMLOutput; - YAMLRemarkSerializer(raw_ostream &OS); + YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode); void emit(const Remark &Remark) override; std::unique_ptr metaSerializer(raw_ostream &OS, Optional ExternalFilename = None) override; + +protected: + bool DidEmitMeta = false; }; struct YAMLMetaSerializer : public MetaSerializer { @@ -55,12 +58,14 @@ struct YAMLMetaSerializer : public MetaSerializer { /// like the regular YAML remark but instead of string entries it's using /// numbers that map to an index in the string table. struct YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer { - YAMLStrTabRemarkSerializer(raw_ostream &OS) : YAMLRemarkSerializer(OS) { + YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode) + : YAMLRemarkSerializer(OS, Mode) { // Having a string table set up enables the serializer to use it. StrTab.emplace(); } - YAMLStrTabRemarkSerializer(raw_ostream &OS, StringTable StrTabIn) - : YAMLRemarkSerializer(OS) { + YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode, + StringTable StrTabIn) + : YAMLRemarkSerializer(OS, Mode) { StrTab = std::move(StrTabIn); } std::unique_ptr diff --git a/llvm/lib/IR/RemarkStreamer.cpp b/llvm/lib/IR/RemarkStreamer.cpp index 450453f..4bbbfe8 100644 --- a/llvm/lib/IR/RemarkStreamer.cpp +++ b/llvm/lib/IR/RemarkStreamer.cpp @@ -136,7 +136,7 @@ llvm::setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, return make_error(std::move(E)); Expected> RemarkSerializer = - remarks::createRemarkSerializer(*Format, RemarksFile->os()); + remarks::createRemarkSerializer(*Format, remarks::SerializerMode::Separate, RemarksFile->os()); if (Error E = RemarkSerializer.takeError()) return make_error(std::move(E)); diff --git a/llvm/lib/Remarks/RemarkSerializer.cpp b/llvm/lib/Remarks/RemarkSerializer.cpp index aa68b49..73cec4f 100644 --- a/llvm/lib/Remarks/RemarkSerializer.cpp +++ b/llvm/lib/Remarks/RemarkSerializer.cpp @@ -17,22 +17,23 @@ using namespace llvm; using namespace llvm::remarks; Expected> -remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS) { +remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS) { switch (RemarksFormat) { case Format::Unknown: return createStringError(std::errc::invalid_argument, "Unknown remark serializer format."); case Format::YAML: - return llvm::make_unique(OS); + return llvm::make_unique(OS, Mode); case Format::YAMLStrTab: - return llvm::make_unique(OS); + return llvm::make_unique(OS, Mode); } llvm_unreachable("Unknown remarks::Format enum"); } Expected> -remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS, - remarks::StringTable StrTab) { +remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS, remarks::StringTable StrTab) { switch (RemarksFormat) { case Format::Unknown: return createStringError(std::errc::invalid_argument, @@ -42,7 +43,8 @@ remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS, "Unable to use a string table with the yaml " "format. Use 'yaml-strtab' instead."); case Format::YAMLStrTab: - return llvm::make_unique(OS, std::move(StrTab)); + return llvm::make_unique(OS, Mode, + std::move(StrTab)); } llvm_unreachable("Unknown remarks::Format enum"); } diff --git a/llvm/lib/Remarks/YAMLRemarkSerializer.cpp b/llvm/lib/Remarks/YAMLRemarkSerializer.cpp index 725ac15..f8ed740 100644 --- a/llvm/lib/Remarks/YAMLRemarkSerializer.cpp +++ b/llvm/lib/Remarks/YAMLRemarkSerializer.cpp @@ -149,10 +149,19 @@ template <> struct MappingTraits { LLVM_YAML_IS_SEQUENCE_VECTOR(Argument) -YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS) - : RemarkSerializer(OS), YAMLOutput(OS, reinterpret_cast(this)) {} +YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode) + : RemarkSerializer(OS, Mode), YAMLOutput(OS, reinterpret_cast(this)) {} void YAMLRemarkSerializer::emit(const Remark &Remark) { + // In standalone mode, emit the metadata first and set DidEmitMeta to avoid + // emitting it again. + if (Mode == SerializerMode::Standalone) { + std::unique_ptr MetaSerializer = + metaSerializer(OS, /*ExternalFilename=*/None); + MetaSerializer->emit(); + DidEmitMeta = true; + } + // Again, YAMLTraits expect a non-const object for inputting, but we're not // using that here. auto R = const_cast(&Remark); diff --git a/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp b/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp index 811c1b2..761a46f 100644 --- a/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp +++ b/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp @@ -21,20 +21,21 @@ using namespace llvm; -static void check(const remarks::Remark &R, StringRef ExpectedR, - StringRef ExpectedMeta, bool UseStrTab = false, +static void check(remarks::SerializerMode Mode, const remarks::Remark &R, + StringRef ExpectedR, Optional ExpectedMeta, + bool UseStrTab = false, Optional StrTab = None) { std::string Buf; raw_string_ostream OS(Buf); Expected> MaybeS = [&] { if (UseStrTab) { if (StrTab) - return createRemarkSerializer(remarks::Format::YAMLStrTab, OS, + return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS, std::move(*StrTab)); else - return createRemarkSerializer(remarks::Format::YAMLStrTab, OS); + return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS); } else - return createRemarkSerializer(remarks::Format::YAML, OS); + return createRemarkSerializer(remarks::Format::YAML, Mode, OS); }(); EXPECT_FALSE(errorToBool(MaybeS.takeError())); std::unique_ptr S = std::move(*MaybeS); @@ -42,11 +43,27 @@ static void check(const remarks::Remark &R, StringRef ExpectedR, S->emit(R); EXPECT_EQ(OS.str(), ExpectedR); - Buf.clear(); - std::unique_ptr MS = - S->metaSerializer(OS, StringRef(EXTERNALFILETESTPATH)); - MS->emit(); - EXPECT_EQ(OS.str(), ExpectedMeta); + if (ExpectedMeta) { + Buf.clear(); + std::unique_ptr MS = + S->metaSerializer(OS, StringRef(EXTERNALFILETESTPATH)); + MS->emit(); + EXPECT_EQ(OS.str(), *ExpectedMeta); + } +} + +static void check(const remarks::Remark &R, StringRef ExpectedR, + StringRef ExpectedMeta, bool UseStrTab = false, + Optional StrTab = None) { + return check(remarks::SerializerMode::Separate, R, ExpectedR, ExpectedMeta, + UseStrTab, std::move(StrTab)); +} + +static void checkStandalone(const remarks::Remark &R, StringRef ExpectedR, + Optional StrTab = None) { + bool UseStrTab = StrTab.hasValue(); + return check(remarks::SerializerMode::Standalone, R, ExpectedR, + /*ExpectedMeta=*/None, UseStrTab, std::move(StrTab)); } TEST(YAMLRemarks, SerializerRemark) { @@ -83,6 +100,40 @@ TEST(YAMLRemarks, SerializerRemark) { 38)); } +TEST(YAMLRemarks, SerializerRemarkStandalone) { + remarks::Remark R; + R.RemarkType = remarks::Type::Missed; + R.PassName = "pass"; + R.RemarkName = "name"; + R.FunctionName = "func"; + R.Loc = remarks::RemarkLocation{"path", 3, 4}; + R.Hotness = 5; + R.Args.emplace_back(); + R.Args.back().Key = "key"; + R.Args.back().Val = "value"; + R.Args.emplace_back(); + R.Args.back().Key = "keydebug"; + R.Args.back().Val = "valuedebug"; + R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7}; + checkStandalone( + R, + StringRef("REMARKS\0" + "\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0" + "--- !Missed\n" + "Pass: pass\n" + "Name: name\n" + "DebugLoc: { File: path, Line: 3, Column: 4 }\n" + "Function: func\n" + "Hotness: 5\n" + "Args:\n" + " - key: value\n" + " - keydebug: valuedebug\n" + " DebugLoc: { File: argpath, Line: 6, Column: 7 }\n" + "...\n", + 301)); +} + TEST(YAMLRemarks, SerializerRemarkStrTab) { remarks::Remark R; R.RemarkType = remarks::Type::Missed; @@ -156,3 +207,42 @@ TEST(YAMLRemarks, SerializerRemarkParsedStrTab) { /*UseStrTab=*/true, remarks::StringTable(remarks::ParsedStringTable(StrTab))); } + +TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandalone) { + StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45); + remarks::ParsedStringTable ParsedStrTab(StrTab); + remarks::StringTable PreFilledStrTab(ParsedStrTab); + remarks::Remark R; + R.RemarkType = remarks::Type::Missed; + R.PassName = "pass"; + R.RemarkName = "name"; + R.FunctionName = "func"; + R.Loc = remarks::RemarkLocation{"path", 3, 4}; + R.Hotness = 5; + R.Args.emplace_back(); + R.Args.back().Key = "key"; + R.Args.back().Val = "value"; + R.Args.emplace_back(); + R.Args.back().Key = "keydebug"; + R.Args.back().Val = "valuedebug"; + R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7}; + checkStandalone( + R, + StringRef("REMARKS\0" + "\0\0\0\0\0\0\0\0" + "\x2d\0\0\0\0\0\0\0" + "pass\0name\0func\0path\0value\0valuedebug\0argpath\0" + "--- !Missed\n" + "Pass: 0\n" + "Name: 1\n" + "DebugLoc: { File: 3, Line: 3, Column: 4 }\n" + "Function: 2\n" + "Hotness: 5\n" + "Args:\n" + " - key: 4\n" + " - keydebug: 5\n" + " DebugLoc: { File: 6, Line: 6, Column: 7 }\n" + "...\n", + 315), + std::move(PreFilledStrTab)); +}