From 146d5791d9b8b20eaca0ca3434f81359046edfbe Mon Sep 17 00:00:00 2001 From: Dean Michael Berris Date: Fri, 31 Aug 2018 08:04:56 +0000 Subject: [PATCH] [XRay] FDR Record Producer/Consumer Implementation Summary: This patch defines two new base types called `RecordProducer` and `RecordConsumer` which have default implementations for convenience (particularly for testing). A `RecordProducer` implementation has one member function called `produce()` which serves as a factory constructor for `Record` instances. This code exercises the `RecordInitializer` code path in the implementation for `FileBasedRecordProducer`. A `RecordConsumer` has a single member function called `consume(...)` which, as the name implies, consumes instances of `std::unique_ptr`. We have two implementations, one of which is used in the test to generate a vector of `std::unique_ptr` similar to how the `LogBuilder` implementation works. We introduce a test in `FDRProducerConsumerTest` which ensures that records we write through the `FDRTraceWriter` can be loaded by the `FileBasedRecordProducer`. The record(s) loaded this way are written again through the `FDRTraceWriter` into a separate string, which we then compare. This ensures that the read-in bytes to create the `Record` instances in memory can be replicated when written out through the `FDRTraceWriter`. This change depends on D51210 and is part of the refactoring of D50441 into smaller, more focused changes. Reviewers: eizan, kpw Subscribers: mgorny, hiraditya, llvm-commits Differential Revision: https://reviews.llvm.org/D51289 llvm-svn: 341180 --- llvm/include/llvm/XRay/FDRRecordConsumer.h | 55 +++++++++ llvm/include/llvm/XRay/FDRRecordProducer.h | 46 ++++++++ llvm/lib/XRay/CMakeLists.txt | 2 + llvm/lib/XRay/FDRRecordProducer.cpp | 125 +++++++++++++++++++++ llvm/lib/XRay/LogBuilderConsumer.cpp | 38 +++++++ llvm/unittests/XRay/CMakeLists.txt | 1 + llvm/unittests/XRay/FDRProducerConsumerTest.cpp | 141 ++++++++++++++++++++++++ 7 files changed, 408 insertions(+) create mode 100644 llvm/include/llvm/XRay/FDRRecordConsumer.h create mode 100644 llvm/include/llvm/XRay/FDRRecordProducer.h create mode 100644 llvm/lib/XRay/FDRRecordProducer.cpp create mode 100644 llvm/lib/XRay/LogBuilderConsumer.cpp create mode 100644 llvm/unittests/XRay/FDRProducerConsumerTest.cpp diff --git a/llvm/include/llvm/XRay/FDRRecordConsumer.h b/llvm/include/llvm/XRay/FDRRecordConsumer.h new file mode 100644 index 0000000..e856e15 --- /dev/null +++ b/llvm/include/llvm/XRay/FDRRecordConsumer.h @@ -0,0 +1,55 @@ +//===- FDRRecordConsumer.h - XRay Flight Data Recorder Mode Records -------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_INCLUDE_LLVM_XRAY_FDRRECORDCONSUMER_H_ +#define LLVM_INCLUDE_LLVM_XRAY_FDRRECORDCONSUMER_H_ + +#include "llvm/Support/Error.h" +#include "llvm/XRay/FDRRecords.h" +#include +#include +#include + +namespace llvm { +namespace xray { + +class RecordConsumer { +public: + virtual Error consume(std::unique_ptr R) = 0; + virtual ~RecordConsumer() = default; +}; + +// This consumer will collect all the records into a vector of records, in +// arrival order. +class LogBuilderConsumer : public RecordConsumer { + std::vector> &Records; + +public: + explicit LogBuilderConsumer(std::vector> &R) + : RecordConsumer(), Records(R) {} + + Error consume(std::unique_ptr R) override; +}; + +// A PipelineConsumer applies a set of visitors to every consumed Record, in the +// order by which the visitors are added to the pipeline in the order of +// appearance. +class PipelineConsumer : public RecordConsumer { + std::vector Visitors; + +public: + PipelineConsumer(std::initializer_list V) + : RecordConsumer(), Visitors(V) {} + + Error consume(std::unique_ptr R) override; +}; + +} // namespace xray +} // namespace llvm + +#endif // LLVM_INCLUDE_LLVM_XRAY_FDRRECORDCONSUMER_H_ diff --git a/llvm/include/llvm/XRay/FDRRecordProducer.h b/llvm/include/llvm/XRay/FDRRecordProducer.h new file mode 100644 index 0000000..e67e228 --- /dev/null +++ b/llvm/include/llvm/XRay/FDRRecordProducer.h @@ -0,0 +1,46 @@ +//===- FDRRecordProducer.h - XRay FDR Mode Record Producer ----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_INCLUDE_LLVM_XRAY_FDRRECORDPRODUCER_H_ +#define LLVM_INCLUDE_LLVM_XRAY_FDRRECORDPRODUCER_H_ + +#include "llvm/Support/Error.h" +#include "llvm/XRay/FDRRecords.h" +#include "llvm/XRay/XRayRecord.h" +#include + +namespace llvm { +namespace xray { + +class RecordProducer { +public: + /// All producer implementations must yield either an Error or a non-nullptr + /// unique_ptr. + virtual Expected> produce() = 0; + virtual ~RecordProducer() = default; +}; + +class FileBasedRecordProducer : public RecordProducer { + const XRayFileHeader &Header; + DataExtractor &E; + uint32_t &OffsetPtr; + +public: + FileBasedRecordProducer(const XRayFileHeader &FH, DataExtractor &DE, + uint32_t &OP) + : Header(FH), E(DE), OffsetPtr(OP) {} + + /// This producer encapsulates the logic for loading a File-backed + /// RecordProducer hidden behind a DataExtractor. + Expected> produce() override; +}; + +} // namespace xray +} // namespace llvm + +#endif // LLVM_INCLUDE_LLVM_XRAY_FDRRECORDPRODUCER_H_ diff --git a/llvm/lib/XRay/CMakeLists.txt b/llvm/lib/XRay/CMakeLists.txt index 5a09a24..a726db2 100644 --- a/llvm/lib/XRay/CMakeLists.txt +++ b/llvm/lib/XRay/CMakeLists.txt @@ -1,8 +1,10 @@ add_llvm_library(LLVMXRay + FDRRecordProducer.cpp FDRRecords.cpp FDRTraceWriter.cpp FileHeaderReader.cpp InstrumentationMap.cpp + LogBuilderConsumer.cpp Profile.cpp RecordInitializer.cpp Trace.cpp diff --git a/llvm/lib/XRay/FDRRecordProducer.cpp b/llvm/lib/XRay/FDRRecordProducer.cpp new file mode 100644 index 0000000..84d75cc9 --- /dev/null +++ b/llvm/lib/XRay/FDRRecordProducer.cpp @@ -0,0 +1,125 @@ +//===- FDRRecordProducer.cpp - XRay FDR Mode Record Producer --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/FDRRecordProducer.h" +#include "llvm/Support/DataExtractor.h" + +namespace llvm { +namespace xray { + +namespace { + +// Keep this in sync with the values written in the XRay FDR mode runtime in +// compiler-rt. +enum class MetadataRecordKinds : uint8_t { + NewBuffer, + EndOfBuffer, + NewCPUId, + TSCWrap, + WalltimeMarker, + CustomEventMarker, + CallArgument, + BufferExtents, + TypedEventMarker, + Pid, + // This is an end marker, used to identify the upper bound for this enum. + EnumEndMarker, +}; + +Expected> +metadataRecordType(const XRayFileHeader &Header, uint8_t T) { + if (T >= static_cast(MetadataRecordKinds::EnumEndMarker)) + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Invalid metadata record type: %d", T); + static constexpr MetadataRecordKinds Mapping[] = { + MetadataRecordKinds::NewBuffer, + MetadataRecordKinds::EndOfBuffer, + MetadataRecordKinds::NewCPUId, + MetadataRecordKinds::TSCWrap, + MetadataRecordKinds::WalltimeMarker, + MetadataRecordKinds::CustomEventMarker, + MetadataRecordKinds::CallArgument, + MetadataRecordKinds::BufferExtents, + MetadataRecordKinds::TypedEventMarker, + MetadataRecordKinds::Pid, + }; + switch (Mapping[T]) { + case MetadataRecordKinds::NewBuffer: + return make_unique(); + case MetadataRecordKinds::EndOfBuffer: + if (Header.Version >= 2) + return createStringError( + std::make_error_code(std::errc::executable_format_error), + "End of buffer records are no longer supported starting version " + "2 of the log."); + return make_unique(); + case MetadataRecordKinds::NewCPUId: + return make_unique(); + case MetadataRecordKinds::TSCWrap: + return make_unique(); + case MetadataRecordKinds::WalltimeMarker: + return make_unique(); + case MetadataRecordKinds::CustomEventMarker: + return make_unique(); + case MetadataRecordKinds::CallArgument: + return make_unique(); + case MetadataRecordKinds::BufferExtents: + return make_unique(); + case MetadataRecordKinds::TypedEventMarker: + return createStringError(std::make_error_code(std::errc::invalid_argument), + "Encountered an unsupported TypedEventMarker."); + case MetadataRecordKinds::Pid: + return make_unique(); + case MetadataRecordKinds::EnumEndMarker: + llvm_unreachable("Invalid MetadataRecordKind"); + } +} + +} // namespace + +Expected> FileBasedRecordProducer::produce() { + // At the top level, we read one byte to determine the type of the record to + // create. This byte will comprise of the following bits: + // + // - offset 0: A '1' indicates a metadata record, a '0' indicates a function + // record. + // - offsets 1-7: For metadata records, this will indicate the kind of + // metadata record should be loaded. + // + // We read first byte, then create the appropriate type of record to consume + // the rest of the bytes. + auto PreReadOffset = OffsetPtr; + uint8_t FirstByte = E.getU8(&OffsetPtr); + std::unique_ptr R; + + // For metadata records, handle especially here. + if (FirstByte & 0x01) { + auto LoadedType = FirstByte >> 1; + auto MetadataRecordOrErr = metadataRecordType(Header, LoadedType); + if (!MetadataRecordOrErr) + return joinErrors( + MetadataRecordOrErr.takeError(), + createStringError( + std::make_error_code(std::errc::executable_format_error), + "Encountered an unsupported metadata record (%d) at offset %d.", + LoadedType, PreReadOffset)); + R = std::move(MetadataRecordOrErr.get()); + } else { + R = llvm::make_unique(); + } + RecordInitializer RI(E, OffsetPtr); + + if (auto Err = R->apply(RI)) + return std::move(Err); + + assert(R != nullptr); + return std::move(R); +} + +} // namespace xray +} // namespace llvm diff --git a/llvm/lib/XRay/LogBuilderConsumer.cpp b/llvm/lib/XRay/LogBuilderConsumer.cpp new file mode 100644 index 0000000..88b7d2d --- /dev/null +++ b/llvm/lib/XRay/LogBuilderConsumer.cpp @@ -0,0 +1,38 @@ +//===- FDRRecordConsumer.h - XRay Flight Data Recorder Mode Records -------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "llvm/XRay/FDRRecordConsumer.h" + +namespace llvm { +namespace xray { + +Error LogBuilderConsumer::consume(std::unique_ptr R) { + if (!R) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Must not call RecordConsumer::consume() with a null pointer."); + Records.push_back(std::move(R)); + return Error::success(); +} + +Error PipelineConsumer::consume(std::unique_ptr R) { + if (!R) + return createStringError( + std::make_error_code(std::errc::invalid_argument), + "Must not call RecordConsumer::consume() with a null pointer."); + + // We apply all of the visitors in order, and concatenate errors + // appropriately. + Error Result = Error::success(); + for (auto *V : Visitors) + Result = joinErrors(std::move(Result), R->apply(*V)); + return Result; +} + +} // namespace xray +} // namespace llvm diff --git a/llvm/unittests/XRay/CMakeLists.txt b/llvm/unittests/XRay/CMakeLists.txt index 0927f2e..0210324 100644 --- a/llvm/unittests/XRay/CMakeLists.txt +++ b/llvm/unittests/XRay/CMakeLists.txt @@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS ) add_llvm_unittest(XRayTests + FDRProducerConsumerTest.cpp FDRTraceWriterTest.cpp GraphTest.cpp ProfileTest.cpp diff --git a/llvm/unittests/XRay/FDRProducerConsumerTest.cpp b/llvm/unittests/XRay/FDRProducerConsumerTest.cpp new file mode 100644 index 0000000..6f2aebe --- /dev/null +++ b/llvm/unittests/XRay/FDRProducerConsumerTest.cpp @@ -0,0 +1,141 @@ +//===- llvm/unittest/XRay/FDRProducerConsumerTest.cpp -----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Test for round-trip record writing and reading. +// +//===----------------------------------------------------------------------===// +#include "llvm/Support/DataExtractor.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/XRay/FDRLogBuilder.h" +#include "llvm/XRay/FDRRecordConsumer.h" +#include "llvm/XRay/FDRRecordProducer.h" +#include "llvm/XRay/FDRRecords.h" +#include "llvm/XRay/FDRTraceWriter.h" +#include "llvm/XRay/FileHeaderReader.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include +#include + +namespace llvm { +namespace xray { +namespace { + +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::Not; + +template std::unique_ptr MakeRecord(); + +template <> std::unique_ptr MakeRecord() { + return make_unique(1); +} + +template <> std::unique_ptr MakeRecord() { + return make_unique(1); +} + +template <> std::unique_ptr MakeRecord() { + return make_unique(1); +} + +template <> std::unique_ptr MakeRecord() { + return make_unique(1); +} + +template <> std::unique_ptr MakeRecord() { + return make_unique(1, 2); +} + +template <> std::unique_ptr MakeRecord() { + return make_unique(4, 1, "data"); +} + +template <> std::unique_ptr MakeRecord() { + return make_unique(1); +} + +template <> std::unique_ptr MakeRecord() { + return make_unique(1); +} + +template <> std::unique_ptr MakeRecord() { + return make_unique(RecordTypes::ENTER, 1, 2); +} + +template class RoundTripTest : public ::testing::Test { +public: + RoundTripTest() : Data(), OS(Data) { + H.Version = 3; + H.Type = 1; + H.ConstantTSC = true; + H.NonstopTSC = true; + H.CycleFrequency = 3e9; + + Writer = make_unique(OS, H); + Record = MakeRecord(); + } + +protected: + std::string Data; + raw_string_ostream OS; + XRayFileHeader H; + std::unique_ptr Writer; + std::unique_ptr Record; +}; + +TYPED_TEST_CASE_P(RoundTripTest); + +// This test ensures that the writing and reading implementations are in sync -- +// that given write(read(write(R))) == R. +TYPED_TEST_P(RoundTripTest, RoundTripsSingleValue) { + auto &R = this->Record; + ASSERT_FALSE(errorToBool(R->apply(*this->Writer))); + this->OS.flush(); + + DataExtractor DE(this->Data, true, 8); + uint32_t OffsetPtr = 0; + auto HeaderOrErr = readBinaryFormatHeader(DE, OffsetPtr); + if (!HeaderOrErr) + FAIL() << HeaderOrErr.takeError(); + + FileBasedRecordProducer P(HeaderOrErr.get(), DE, OffsetPtr); + std::vector> Records; + LogBuilderConsumer C(Records); + while (DE.isValidOffsetForDataOfSize(OffsetPtr, 1)) { + auto R = P.produce(); + if (!R) + FAIL() << R.takeError(); + if (auto E = C.consume(std::move(R.get()))) + FAIL() << E; + } + ASSERT_THAT(Records, Not(IsEmpty())); + std::string Data2; + raw_string_ostream OS2(Data2); + FDRTraceWriter Writer2(OS2, this->H); + for (auto &P : Records) + ASSERT_FALSE(errorToBool(P->apply(Writer2))); + OS2.flush(); + + EXPECT_EQ(Data2.substr(sizeof(XRayFileHeader)), + this->Data.substr(sizeof(XRayFileHeader))); + EXPECT_THAT(Records[0]->type(), Eq(R->type())); +} + +REGISTER_TYPED_TEST_CASE_P(RoundTripTest, RoundTripsSingleValue); + +using RecordTypes = + ::testing::Types; +INSTANTIATE_TYPED_TEST_CASE_P(Records, RoundTripTest, RecordTypes); + +} // namespace +} // namespace xray +} // namespace llvm -- 2.7.4