From 494fd8f84f90608e0b8efa595262d1ebb03c3d3e Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Tue, 5 Feb 2019 18:46:36 +0000 Subject: [PATCH] [Reproducers] Instrumentation Framework: Serialization This is the is serialization/deserialization part of the reproducer instrumentation framework. For all the details refer to the RFC on the mailing list: http://lists.llvm.org/pipermail/lldb-dev/2019-January/014530.html Differential revision: https://reviews.llvm.org/D57714 llvm-svn: 353195 --- .../lldb/Utility/ReproducerInstrumentation.h | 302 +++++++++++++++++++++ lldb/source/Utility/CMakeLists.txt | 1 + lldb/source/Utility/ReproducerInstrumentation.cpp | 44 +++ lldb/unittests/Utility/CMakeLists.txt | 1 + .../Utility/ReproducerInstrumentationTest.cpp | 208 ++++++++++++++ 5 files changed, 556 insertions(+) create mode 100644 lldb/include/lldb/Utility/ReproducerInstrumentation.h create mode 100644 lldb/source/Utility/ReproducerInstrumentation.cpp create mode 100644 lldb/unittests/Utility/ReproducerInstrumentationTest.cpp diff --git a/lldb/include/lldb/Utility/ReproducerInstrumentation.h b/lldb/include/lldb/Utility/ReproducerInstrumentation.h new file mode 100644 index 0000000..61e25ca --- /dev/null +++ b/lldb/include/lldb/Utility/ReproducerInstrumentation.h @@ -0,0 +1,302 @@ +//===-- ReproducerInstrumentation.h -----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_UTILITY_REPRODUCER_INSTRUMENTATION_H +#define LLDB_UTILITY_REPRODUCER_INSTRUMENTATION_H + +#include "lldb/Utility/FileSpec.h" +#include "lldb/Utility/Log.h" +#include "lldb/Utility/Logging.h" + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/ErrorHandling.h" + +#include +#include + +namespace lldb_private { +namespace repro { + +/// Mapping between serialized indices and their corresponding objects. +/// +/// This class is used during replay to map indices back to in-memory objects. +/// +/// When objects are constructed, they are added to this mapping using +/// AddObjectForIndex. +/// +/// When an object is passed to a function, its index is deserialized and +/// AddObjectForIndex returns the corresponding object. If there is no object +/// for the given index, a nullptr is returend. The latter is valid when custom +/// replay code is in place and the actual object is ignored. +class IndexToObject { +public: + /// Returns an object as a pointer for the given index or nullptr if not + /// present in the map. + template T *GetObjectForIndex(unsigned idx) { + assert(idx != 0 && "Cannot get object for sentinel"); + void *object = GetObjectForIndexImpl(idx); + return static_cast(object); + } + + /// Adds a pointer to an object to the mapping for the given index. + template void AddObjectForIndex(unsigned idx, T *object) { + AddObjectForIndexImpl( + idx, static_cast( + const_cast::type *>(object))); + } + + /// Adds a reference to an object to the mapping for the given index. + template void AddObjectForIndex(unsigned idx, T &object) { + AddObjectForIndexImpl( + idx, static_cast( + const_cast::type *>(&object))); + } + +private: + /// Helper method that does the actual lookup. The void* result is later cast + /// by the caller. + void *GetObjectForIndexImpl(unsigned idx); + + /// Helper method that does the actual insertion. + void AddObjectForIndexImpl(unsigned idx, void *object); + + /// Keeps a mapping between indices and their corresponding object. + llvm::DenseMap m_mapping; +}; + +/// We need to differentiate between pointers to fundamental and +/// non-fundamental types. See the corresponding Deserializer::Read method +/// for the reason why. +struct PointerTag {}; +struct ReferenceTag {}; +struct ValueTag {}; +struct FundamentalPointerTag {}; +struct FundamentalReferenceTag {}; + +/// Return the deserialization tag for the given type T. +template struct serializer_tag { typedef ValueTag type; }; +template struct serializer_tag { + typedef + typename std::conditional::value, + FundamentalPointerTag, PointerTag>::type type; +}; +template struct serializer_tag { + typedef typename std::conditional::value, + FundamentalReferenceTag, ReferenceTag>::type + type; +}; + +/// Deserializes data from a buffer. It is used to deserialize function indices +/// to replay, their arguments and return values. +/// +/// Fundamental types and strings are read by value. Objects are read by their +/// index, which get translated by the IndexToObject mapping maintained in +/// this class. +/// +/// Additional bookkeeping with regards to the IndexToObject is required to +/// deserialize objects. When a constructor is run or an object is returned by +/// value, we need to capture the object and add it to the index together with +/// its index. This is the job of HandleReplayResult(Void). +class Deserializer { +public: + Deserializer(llvm::StringRef buffer) : m_buffer(buffer) {} + + /// Returns true when the buffer has unread data. + bool HasData(unsigned size) { return size <= m_buffer.size(); } + + /// Deserialize and interpret value as T. + template T Deserialize() { + return Read(typename serializer_tag::type()); + } + + /// Store the returned value in the index-to-object mapping. + template void HandleReplayResult(const T &t) { + unsigned result = Deserialize(); + if (std::is_fundamental::value) + return; + // We need to make a copy as the original object might go out of scope. + m_index_to_object.AddObjectForIndex(result, new T(t)); + } + + /// Store the returned value in the index-to-object mapping. + template void HandleReplayResult(T *t) { + unsigned result = Deserialize(); + if (std::is_fundamental::value) + return; + m_index_to_object.AddObjectForIndex(result, t); + } + + /// All returned types are recorded, even when the function returns a void. + /// The latter requires special handling. + void HandleReplayResultVoid() { + unsigned result = Deserialize(); + assert(result == 0); + } + +protected: + IndexToObject &GetIndexToObject() { return m_index_to_object; } + +private: + template T Read(ValueTag) { + assert(HasData(sizeof(T))); + T t; + std::memcpy(reinterpret_cast(&t), m_buffer.data(), sizeof(T)); + m_buffer = m_buffer.drop_front(sizeof(T)); + return t; + } + + template T Read(PointerTag) { + typedef typename std::remove_pointer::type UnderlyingT; + return m_index_to_object.template GetObjectForIndex( + Deserialize()); + } + + template T Read(ReferenceTag) { + typedef typename std::remove_reference::type UnderlyingT; + // If this is a reference to a fundamental type we just read its value. + return *m_index_to_object.template GetObjectForIndex( + Deserialize()); + } + + /// This method is used to parse references to fundamental types. Because + /// they're not recorded in the object table we have serialized their value. + /// We read its value, allocate a copy on the heap, and return a pointer to + /// the copy. + template T Read(FundamentalPointerTag) { + typedef typename std::remove_pointer::type UnderlyingT; + return new UnderlyingT(Deserialize()); + } + + /// This method is used to parse references to fundamental types. Because + /// they're not recorded in the object table we have serialized their value. + /// We read its value, allocate a copy on the heap, and return a reference to + /// the copy. + template T Read(FundamentalReferenceTag) { + // If this is a reference to a fundamental type we just read its value. + typedef typename std::remove_reference::type UnderlyingT; + return *(new UnderlyingT(Deserialize())); + } + + /// Mapping of indices to objects. + IndexToObject m_index_to_object; + + /// Buffer containing the serialized data. + llvm::StringRef m_buffer; +}; + +/// Partial specialization for C-style strings. We read the string value +/// instead of treating it as pointer. +template <> const char *Deserializer::Deserialize(); +template <> char *Deserializer::Deserialize(); + +/// Helpers to auto-synthesize function replay code. It deserializes the replay +/// function's arguments one by one and finally calls the corresponding +/// function. +template struct DeserializationHelper; + +template +struct DeserializationHelper { + template struct deserialized { + static Result doit(Deserializer &deserializer, + Result (*f)(Deserialized..., Head, Tail...), + Deserialized... d) { + return DeserializationHelper:: + template deserialized::doit( + deserializer, f, d..., deserializer.Deserialize()); + } + }; +}; + +template <> struct DeserializationHelper<> { + template struct deserialized { + static Result doit(Deserializer &deserializer, Result (*f)(Deserialized...), + Deserialized... d) { + return f(d...); + } + }; +}; + +/// Maps an object to an index for serialization. Indices are unique and +/// incremented for every new object. +/// +/// Indices start at 1 in order to differentiate with an invalid index (0) in +/// the serialized buffer. +class ObjectToIndex { +public: + template unsigned GetIndexForObject(T *t) { + return GetIndexForObjectImpl((void *)t); + } + +private: + unsigned GetIndexForObjectImpl(void *object); + + std::mutex m_mutex; + llvm::DenseMap m_mapping; +}; + +/// Serializes functions, their arguments and their return type to a stream. +class Serializer { +public: + Serializer(llvm::raw_ostream &stream = llvm::outs()) : m_stream(stream) {} + + /// Recursively serialize all the given arguments. + template + void SerializeAll(const Head &head, const Tail &... tail) { + Serialize(head); + SerializeAll(tail...); + } + + void SerializeAll() {} + +private: + /// Serialize pointers. We need to differentiate between pointers to + /// fundamental types (in which case we serialize its value) and pointer to + /// objects (in which case we serialize their index). + template void Serialize(T *t) { + if (std::is_fundamental::value) { + Serialize(*t); + } else { + unsigned idx = m_tracker.GetIndexForObject(t); + Serialize(idx); + } + } + + /// Serialize references. We need to differentiate between references to + /// fundamental types (in which case we serialize its value) and references + /// to objects (in which case we serialize their index). + template void Serialize(T &t) { + if (std::is_fundamental::value) { + m_stream.write(reinterpret_cast(&t), sizeof(T)); + } else { + unsigned idx = m_tracker.GetIndexForObject(&t); + Serialize(idx); + } + } + + void Serialize(void *v) { + // FIXME: Support void* + llvm_unreachable("void* is currently unsupported."); + } + + void Serialize(const char *t) { + m_stream << t; + m_stream.write(0x0); + } + + /// Serialization stream. + llvm::raw_ostream &m_stream; + + /// Mapping of objects to indices. + ObjectToIndex m_tracker; +}; + +} // namespace repro +} // namespace lldb_private + +#endif // LLDB_UTILITY_REPRODUCER_INSTRUMENTATION_H diff --git a/lldb/source/Utility/CMakeLists.txt b/lldb/source/Utility/CMakeLists.txt index 4c981c6..03f9654 100644 --- a/lldb/source/Utility/CMakeLists.txt +++ b/lldb/source/Utility/CMakeLists.txt @@ -67,6 +67,7 @@ add_lldb_library(lldbUtility RegisterValue.cpp RegularExpression.cpp Reproducer.cpp + ReproducerInstrumentation.cpp Scalar.cpp SelectHelper.cpp SharingPtr.cpp diff --git a/lldb/source/Utility/ReproducerInstrumentation.cpp b/lldb/source/Utility/ReproducerInstrumentation.cpp new file mode 100644 index 0000000..8b53c48 --- /dev/null +++ b/lldb/source/Utility/ReproducerInstrumentation.cpp @@ -0,0 +1,44 @@ +//===-- ReproducerInstrumentation.cpp ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "lldb/Utility/ReproducerInstrumentation.h" +#include "lldb/Utility/Reproducer.h" + +using namespace lldb_private; +using namespace lldb_private::repro; + +void *IndexToObject::GetObjectForIndexImpl(unsigned idx) { + return m_mapping.lookup(idx); +} + +void IndexToObject::AddObjectForIndexImpl(unsigned idx, void *object) { + assert(idx != 0 && "Cannot add object for sentinel"); + m_mapping[idx] = object; +} + +template <> char *Deserializer::Deserialize() { + return const_cast(Deserialize()); +} + +template <> const char *Deserializer::Deserialize() { + auto pos = m_buffer.find('\0'); + if (pos == llvm::StringRef::npos) + return nullptr; + const char *str = m_buffer.data(); + m_buffer = m_buffer.drop_front(pos + 1); + return str; +} + +unsigned ObjectToIndex::GetIndexForObjectImpl(void *object) { + std::lock_guard guard(m_mutex); + unsigned index = m_mapping.size() + 1; + auto it = m_mapping.find(object); + if (it == m_mapping.end()) + m_mapping[object] = index; + return m_mapping[object]; +} diff --git a/lldb/unittests/Utility/CMakeLists.txt b/lldb/unittests/Utility/CMakeLists.txt index 1e7baef..e16320d 100644 --- a/lldb/unittests/Utility/CMakeLists.txt +++ b/lldb/unittests/Utility/CMakeLists.txt @@ -20,6 +20,7 @@ add_lldb_unittest(UtilityTests PredicateTest.cpp RegisterValueTest.cpp ReproducerTest.cpp + ReproducerInstrumentationTest.cpp ScalarTest.cpp StateTest.cpp StatusTest.cpp diff --git a/lldb/unittests/Utility/ReproducerInstrumentationTest.cpp b/lldb/unittests/Utility/ReproducerInstrumentationTest.cpp new file mode 100644 index 0000000..65ee425 --- /dev/null +++ b/lldb/unittests/Utility/ReproducerInstrumentationTest.cpp @@ -0,0 +1,208 @@ +//===-- ReproducerInstrumentationTest.cpp -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "lldb/Utility/ReproducerInstrumentation.h" + +using namespace lldb_private; +using namespace lldb_private::repro; + +namespace { +struct Foo { + int m = 1; +}; +struct Bar { + double m = 2; +}; + +bool operator==(const Foo &LHS, const Foo &RHS) { return LHS.m == RHS.m; } +bool operator==(const Bar &LHS, const Bar &RHS) { return LHS.m == RHS.m; } + +struct Pod { + bool a = true; + bool b = false; + char c = 'a'; + float d = 1.1; + int e = 2; + long long f = 3; + long g = 4; + short h = 5; + unsigned char i = 'b'; + unsigned int j = 6; + unsigned long long k = 7; + unsigned long l = 8; + unsigned short m = 9; +}; +} // namespace + +static const Pod p; + +TEST(IndexToObjectTest, ObjectForIndex) { + IndexToObject index_to_object; + Foo foo; + Bar bar; + + EXPECT_EQ(nullptr, index_to_object.GetObjectForIndex(1)); + EXPECT_EQ(nullptr, index_to_object.GetObjectForIndex(2)); + + index_to_object.AddObjectForIndex(1, foo); + index_to_object.AddObjectForIndex(2, &bar); + + EXPECT_EQ(&foo, index_to_object.GetObjectForIndex(1)); + EXPECT_EQ(&bar, index_to_object.GetObjectForIndex(2)); +} + +TEST(DeserializerTest, HasData) { + { + Deserializer deserializer(""); + EXPECT_FALSE(deserializer.HasData(1)); + } + + { + Deserializer deserializer("a"); + EXPECT_TRUE(deserializer.HasData(1)); + EXPECT_FALSE(deserializer.HasData(2)); + } +} + +TEST(SerializationRountripTest, SerializeDeserializePod) { + std::string str; + llvm::raw_string_ostream os(str); + + Serializer serializer(os); + serializer.SerializeAll(p.a, p.b, p.c, p.d, p.e, p.f, p.g, p.h, p.i, p.j, p.k, + p.l, p.m); + + llvm::StringRef buffer(os.str()); + Deserializer deserializer(buffer); + + EXPECT_EQ(p.a, deserializer.Deserialize()); + EXPECT_EQ(p.b, deserializer.Deserialize()); + EXPECT_EQ(p.c, deserializer.Deserialize()); + EXPECT_EQ(p.d, deserializer.Deserialize()); + EXPECT_EQ(p.e, deserializer.Deserialize()); + EXPECT_EQ(p.f, deserializer.Deserialize()); + EXPECT_EQ(p.g, deserializer.Deserialize()); + EXPECT_EQ(p.h, deserializer.Deserialize()); + EXPECT_EQ(p.i, deserializer.Deserialize()); + EXPECT_EQ(p.j, deserializer.Deserialize()); + EXPECT_EQ(p.k, deserializer.Deserialize()); + EXPECT_EQ(p.l, deserializer.Deserialize()); + EXPECT_EQ(p.m, deserializer.Deserialize()); +} + +TEST(SerializationRountripTest, SerializeDeserializePodPointers) { + std::string str; + llvm::raw_string_ostream os(str); + + Serializer serializer(os); + serializer.SerializeAll(&p.a, &p.b, &p.c, &p.d, &p.e, &p.f, &p.g, &p.h, &p.i, + &p.j, &p.k, &p.l, &p.m); + + llvm::StringRef buffer(os.str()); + Deserializer deserializer(buffer); + + EXPECT_EQ(p.a, *deserializer.Deserialize()); + EXPECT_EQ(p.b, *deserializer.Deserialize()); + EXPECT_EQ(p.c, *deserializer.Deserialize()); + EXPECT_EQ(p.d, *deserializer.Deserialize()); + EXPECT_EQ(p.e, *deserializer.Deserialize()); + EXPECT_EQ(p.f, *deserializer.Deserialize()); + EXPECT_EQ(p.g, *deserializer.Deserialize()); + EXPECT_EQ(p.h, *deserializer.Deserialize()); + EXPECT_EQ(p.i, *deserializer.Deserialize()); + EXPECT_EQ(p.j, *deserializer.Deserialize()); + EXPECT_EQ(p.k, *deserializer.Deserialize()); + EXPECT_EQ(p.l, *deserializer.Deserialize()); + EXPECT_EQ(p.m, *deserializer.Deserialize()); +} + +TEST(SerializationRountripTest, SerializeDeserializePodReferences) { + std::string str; + llvm::raw_string_ostream os(str); + + Serializer serializer(os); + serializer.SerializeAll(p.a, p.b, p.c, p.d, p.e, p.f, p.g, p.h, p.i, p.j, p.k, + p.l, p.m); + + llvm::StringRef buffer(os.str()); + Deserializer deserializer(buffer); + + EXPECT_EQ(p.a, deserializer.Deserialize()); + EXPECT_EQ(p.b, deserializer.Deserialize()); + EXPECT_EQ(p.c, deserializer.Deserialize()); + EXPECT_EQ(p.d, deserializer.Deserialize()); + EXPECT_EQ(p.e, deserializer.Deserialize()); + EXPECT_EQ(p.f, deserializer.Deserialize()); + EXPECT_EQ(p.g, deserializer.Deserialize()); + EXPECT_EQ(p.h, deserializer.Deserialize()); + EXPECT_EQ(p.i, deserializer.Deserialize()); + EXPECT_EQ(p.j, deserializer.Deserialize()); + EXPECT_EQ(p.k, deserializer.Deserialize()); + EXPECT_EQ(p.l, deserializer.Deserialize()); + EXPECT_EQ(p.m, deserializer.Deserialize()); +} + +TEST(SerializationRountripTest, SerializeDeserializeCString) { + const char *cstr = "string"; + + std::string str; + llvm::raw_string_ostream os(str); + + Serializer serializer(os); + serializer.SerializeAll(cstr); + + llvm::StringRef buffer(os.str()); + Deserializer deserializer(buffer); + + EXPECT_STREQ(cstr, deserializer.Deserialize()); +} + +TEST(SerializationRountripTest, SerializeDeserializeObjectPointer) { + Foo foo; + Bar bar; + + std::string str; + llvm::raw_string_ostream os(str); + + Serializer serializer(os); + serializer.SerializeAll(static_cast(1), static_cast(2)); + serializer.SerializeAll(&foo, &bar); + + llvm::StringRef buffer(os.str()); + Deserializer deserializer(buffer); + + deserializer.HandleReplayResult(&foo); + deserializer.HandleReplayResult(&bar); + + EXPECT_EQ(foo, *deserializer.Deserialize()); + EXPECT_EQ(bar, *deserializer.Deserialize()); +} + +TEST(SerializationRountripTest, SerializeDeserializeObjectReference) { + Foo foo; + Bar bar; + + std::string str; + llvm::raw_string_ostream os(str); + + Serializer serializer(os); + serializer.SerializeAll(static_cast(1), static_cast(2)); + serializer.SerializeAll(foo, bar); + + llvm::StringRef buffer(os.str()); + Deserializer deserializer(buffer); + + deserializer.HandleReplayResult(&foo); + deserializer.HandleReplayResult(&bar); + + EXPECT_EQ(foo, deserializer.Deserialize()); + EXPECT_EQ(bar, deserializer.Deserialize()); +} -- 2.7.4