list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_srcs_exclude})
endif()
-file(GLOB tf_core_platform_exclude_srcs
- "${tensorflow_source_dir}/tensorflow/core/platform/variant_coding.cc")
-list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_exclude_srcs})
-
list(APPEND tf_core_lib_srcs ${tf_core_platform_srcs})
if(UNIX)
file(GLOB_RECURSE tf_core_framework_srcs
"${tensorflow_source_dir}/tensorflow/core/framework/*.h"
"${tensorflow_source_dir}/tensorflow/core/framework/*.cc"
- "${tensorflow_source_dir}/tensorflow/core/platform/variant_coding.h"
- "${tensorflow_source_dir}/tensorflow/core/platform/variant_coding.cc"
"${tensorflow_source_dir}/tensorflow/core/graph/edgeset.h"
"${tensorflow_source_dir}/tensorflow/core/graph/edgeset.cc"
"${tensorflow_source_dir}/tensorflow/core/graph/graph.h"
"tf_additional_lib_deps",
"tf_additional_lib_hdrs",
"tf_additional_lib_srcs",
- "tf_additional_framework_hdrs",
- "tf_additional_framework_srcs",
"tf_additional_minimal_lib_srcs",
"tf_additional_proto_hdrs",
"tf_additional_proto_srcs",
"platform/**/device_tracer.cc",
"platform/**/logging.cc",
"platform/abi.cc",
- "platform/variant_coding.cc",
- "platform/**/variant_cord_coding.cc",
],
) + tf_additional_lib_srcs(
exclude = [
"platform/**/device_tracer.cc",
"platform/**/logging.cc",
"platform/abi.cc",
- "platform/variant_coding.cc",
- "platform/**/variant_cord_coding.cc",
] +
# Protobuf deps already included through the ":lib_proto_parsing"
# dependency.
)
FRAMEWORK_INTERNAL_PRIVATE_HEADERS = [
- "platform/variant_coding.h",
"graph/edgeset.h",
"graph/graph.h",
"graph/graph_def_builder.h",
"framework/tracking_allocator.h", # only needed for tests
"framework/unique_tensor_references.h",
"framework/variant.h",
- "platform/variant_coding.h",
"util/command_line_flags.h",
"util/env_var.h",
"util/equal_graph_def.h",
"util/presized_cuckoo_map.h",
"util/tensor_slice_set.h",
"util/tensor_slice_util.h",
-] + tf_additional_framework_hdrs()
+]
tf_cuda_library(
name = "framework_internal",
tf_cuda_library(
name = "framework_internal_impl",
- srcs = FRAMEWORK_INTERNAL_PRIVATE_HEADERS + [
- "platform/variant_coding.cc",
- ] + glob(
+ srcs = FRAMEWORK_INTERNAL_PRIVATE_HEADERS + glob(
[
"example/**/*.cc",
"framework/**/*.cc",
"util/memmapped_file_system.cc",
"util/memmapped_file_system_writer.cc",
],
- }) + tf_additional_framework_srcs(),
+ }),
hdrs = FRAMEWORK_INTERNAL_PUBLIC_HEADERS,
copts = tf_copts(),
linkopts = select({
return handle.DebugString();
}
+void EncodeResourceHandleList(const ResourceHandle* p, int64 n,
+ std::unique_ptr<port::StringListEncoder> e) {
+ ResourceHandleProto proto;
+ for (int i = 0; i < n; ++i) {
+ p[i].AsProto(&proto);
+ e->Append(proto);
+ }
+ e->Finalize();
+}
+
+bool DecodeResourceHandleList(std::unique_ptr<port::StringListDecoder> d,
+ ResourceHandle* ps, int64 n) {
+ std::vector<uint32> sizes(n);
+ if (!d->ReadSizes(&sizes)) return false;
+
+ ResourceHandleProto proto;
+ for (int i = 0; i < n; ++i) {
+ if (!proto.ParseFromArray(d->Data(sizes[i]), sizes[i])) {
+ return false;
+ }
+ ps[i].FromProto(proto);
+ }
+ return true;
+}
+
} // namespace tensorflow
#ifndef TENSORFLOW_FRAMEWORK_RESOURCE_HANDLE_H_
#define TENSORFLOW_FRAMEWORK_RESOURCE_HANDLE_H_
+#include "tensorflow/core/platform/tensor_coding.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// For backwards compatibility for when this was a proto
string ProtoDebugString(const ResourceHandle& handle);
+// Encodes a list of ResourceHandle protos in the given StringListEncoder.
+void EncodeResourceHandleList(const ResourceHandle* p, int64 n,
+ std::unique_ptr<port::StringListEncoder> e);
+
+// Decodes a list of ResourceHandle protos from the given StringListDecoder.
+bool DecodeResourceHandleList(std::unique_ptr<port::StringListDecoder> d,
+ ResourceHandle* ps, int64 n);
+
} // namespace tensorflow
#endif // TENSORFLOW_FRAMEWORK_RESOURCE_HANDLE_H_
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/tensor_coding.h"
#include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/platform/variant_coding.h"
namespace tensorflow {
// "out", which is usually the TensorProto::tensor_content.
template <typename Destination>
static void Encode(TensorBuffer* in, int64 n, Destination* out) {
- port::EncodeResourceHandleList(in->base<const ResourceHandle>(), n, out);
+ EncodeResourceHandleList(in->base<const ResourceHandle>(), n,
+ port::NewStringListEncoder(out));
}
// Decodes "n" elements of type string from "in" and constructs a
static TensorBuffer* Decode(Allocator* a, const Source& in, int64 n) {
auto* buf = new Buffer<ResourceHandle>(a, n);
ResourceHandle* ps = buf->template base<ResourceHandle>();
- if (ps == nullptr || !port::DecodeResourceHandleList(in, ps, n)) {
+ if (ps == nullptr ||
+ !DecodeResourceHandleList(port::NewStringListDecoder(in), ps, n)) {
buf->Unref();
return nullptr;
}
// "out", which is usually the TensorProto::tensor_content.
template <typename Destination>
static void Encode(TensorBuffer* in, int64 n, Destination* out) {
- port::EncodeVariantList(in->base<const Variant>(), n, out);
+ EncodeVariantList(in->base<const Variant>(), n,
+ port::NewStringListEncoder(out));
}
// Decodes "n" elements of type Variant from "in" and constructs a
static TensorBuffer* Decode(Allocator* a, const Source& in, int64 n) {
auto* buf = new Buffer<Variant>(a, n);
Variant* ps = buf->template base<Variant>();
- if (ps == nullptr || !port::DecodeVariantList(in, ps, n)) {
+ if (ps == nullptr ||
+ !DecodeVariantList(port::NewStringListDecoder(in), ps, n)) {
buf->Unref();
return nullptr;
}
#include "tensorflow/core/framework/variant.h"
#include "tensorflow/core/framework/tensor.pb.h"
#include "tensorflow/core/framework/variant_encode_decode.h"
+#include "tensorflow/core/framework/variant_op_registry.h"
#include "tensorflow/core/framework/variant_tensor_data.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/gtl/map_util.h"
return value->ParseFromString(buf);
}
+void EncodeVariantList(const Variant* variant_array, int64 n,
+ std::unique_ptr<port::StringListEncoder> e) {
+ for (int i = 0; i < n; ++i) {
+ string s;
+ variant_array[i].Encode(&s);
+ e->Append(s);
+ }
+ e->Finalize();
+}
+
+bool DecodeVariantList(std::unique_ptr<port::StringListDecoder> d,
+ Variant* variant_array, int64 n) {
+ std::vector<uint32> sizes(n);
+ if (!d->ReadSizes(&sizes)) return false;
+
+ for (int i = 0; i < n; ++i) {
+ if (variant_array[i].is_empty()) {
+ variant_array[i] = VariantTensorDataProto();
+ }
+ string str(d->Data(sizes[i]), sizes[i]);
+ if (!variant_array[i].Decode(str)) return false;
+ if (!DecodeUnaryVariant(&variant_array[i])) {
+ LOG(ERROR) << "Could not decode variant with type_name: \""
+ << variant_array[i].TypeName()
+ << "\". Perhaps you forgot to register a "
+ "decoder via REGISTER_UNARY_VARIANT_DECODE_FUNCTION?";
+ return false;
+ }
+ }
+ return true;
+}
+
} // end namespace tensorflow
template <>
bool DecodeVariant(const string& buf, VariantTensorDataProto* value);
+// Encodes an array of Variant objects in to the given StringListEncoder.
+// `variant_array` is assumed to point to an array of `n` Variant objects.
+void EncodeVariantList(const Variant* variant_array, int64 n,
+ std::unique_ptr<port::StringListEncoder> e);
+
+// Decodes an array of Variant objects from the given StringListDecoder.
+// `variant_array` is assumed to point to an array of `n` Variant objects.
+bool DecodeVariantList(std::unique_ptr<port::StringListDecoder> d,
+ Variant* variant_array, int64 n);
+
} // end namespace tensorflow
#endif // TENSORFLOW_FRAMEWORK_VARIANT_ENCODE_DECODE_H_
], exclude = exclude),
})
-# pylint: disable=unused-argument
-def tf_additional_framework_hdrs(exclude = []):
- return []
-
-def tf_additional_framework_srcs(exclude = []):
- return []
-# pylint: enable=unused-argument
-
def tf_additional_minimal_lib_srcs():
return [
"platform/default/integral_types.h",
--- /dev/null
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/platform/default/string_coding.h"
+
+namespace tensorflow {
+namespace port {
+
+std::unique_ptr<StringListEncoder> NewStringListEncoder(string* out) {
+ return std::unique_ptr<StringListEncoder>(new StringListEncoder(out));
+}
+
+std::unique_ptr<StringListDecoder> NewStringListDecoder(const string& in) {
+ return std::unique_ptr<StringListDecoder>(new StringListDecoder(in));
+}
+
+} // namespace port
+} // namespace tensorflow
--- /dev/null
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_PLATFORM_DEFAULT_STRING_CODING_H_
+#define TENSORFLOW_CORE_PLATFORM_DEFAULT_STRING_CODING_H_
+
+// IWYU pragma: private, include "third_party/tensorflow/core/platform/tensor_coding.h"
+// IWYU pragma: friend third_party/tensorflow/core/platform/tensor_coding.h
+
+#include "tensorflow/core/lib/core/coding.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace port {
+
+// Encodes sequences of strings and serialized protocol buffers into a string.
+// Normal usage consists of zero or more calls to Append() and a single call to
+// Finalize().
+class StringListEncoder {
+ public:
+ explicit StringListEncoder(string* out) : out_(out) {}
+
+ // Encodes the given protocol buffer. This may not be called after Finalize().
+ void Append(const protobuf::MessageLite& m) {
+ core::PutVarint32(out_, m.ByteSize());
+ m.AppendToString(&rest_);
+ }
+
+ // Encodes the given string. This may not be called after Finalize().
+ void Append(const string& s) {
+ core::PutVarint32(out_, s.length());
+ strings::StrAppend(&rest_, s);
+ }
+
+ // Signals end of the encoding process. No other calls are allowed after this.
+ void Finalize() { strings::StrAppend(out_, rest_); }
+
+ private:
+ string* out_;
+ string rest_;
+};
+
+// Decodes a string into sequences of strings (which may represent serialized
+// protocol buffers). Normal usage involves a single call to ReadSizes() in
+// order to retrieve the length of all the strings in the sequence. For each
+// size returned a call to Data() is expected and will return the actual
+// string.
+class StringListDecoder {
+ public:
+ explicit StringListDecoder(const string& in) : reader_(in) {}
+
+ // Populates the given vector with the lengths of each string in the sequence
+ // being decoded. Upon returning the vector is guaranteed to contain as many
+ // elements as there are strings in the sequence.
+ bool ReadSizes(std::vector<uint32>* sizes) {
+ int64 total = 0;
+ for (auto& size : *sizes) {
+ if (!core::GetVarint32(&reader_, &size)) return false;
+ total += size;
+ }
+ if (total != static_cast<int64>(reader_.size())) {
+ return false;
+ }
+ return true;
+ }
+
+ // Returns a pointer to the next string in the sequence, then prepares for the
+ // next call by advancing 'size' characters in the sequence.
+ const char* Data(uint32 size) {
+ const char* data = reader_.data();
+ reader_.remove_prefix(size);
+ return data;
+ }
+
+ private:
+ StringPiece reader_;
+};
+
+std::unique_ptr<StringListEncoder> NewStringListEncoder(string* out);
+std::unique_ptr<StringListDecoder> NewStringListDecoder(const string& in);
+
+} // namespace port
+} // namespace tensorflow
+
+#endif // TENSORFLOW_CORE_PLATFORM_DEFAULT_STRING_CODING_H_
#include "tensorflow/core/platform/tensor_coding.h"
#include <vector>
-#include "tensorflow/core/framework/resource_handle.pb.h"
+
#include "tensorflow/core/lib/core/coding.h"
#include "tensorflow/core/lib/core/stringpiece.h"
s->assign(base, bytes);
}
-void EncodeResourceHandleList(const ResourceHandle* p, int64 n, string* out) {
- out->clear();
- string rest;
- ResourceHandleProto proto;
- for (int i = 0; i < n; ++i) {
- p[i].AsProto(&proto);
- core::PutVarint32(out, proto.ByteSize());
- proto.AppendToString(&rest);
- }
- *out += rest;
-}
-
-bool DecodeResourceHandleList(const string& in, ResourceHandle* ps, int64 n) {
- std::vector<uint32> sizes(n);
- StringPiece reader(in);
- int64 total = 0;
- for (auto& size : sizes) {
- if (!core::GetVarint32(&reader, &size)) return false;
- total += size;
- }
- if (total != static_cast<int64>(reader.size())) {
- return false;
- }
- ResourceHandleProto proto;
- for (int i = 0; i < n; ++i) {
- if (!proto.ParseFromArray(reader.data(), sizes[i])) {
- return false;
- }
- ps[i].FromProto(proto);
- reader.remove_prefix(sizes[i]);
- }
- return true;
-}
-
} // namespace port
} // namespace tensorflow
#define TENSORFLOW_PLATFORM_TENSOR_CODING_H_
#include <string>
-#include "tensorflow/core/framework/resource_handle.h"
#include "tensorflow/core/lib/core/refcount.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/platform.h"
#ifdef PLATFORM_GOOGLE
#include "tensorflow/core/platform/google/cord_coding.h"
+#else
+#include "tensorflow/core/platform/default/string_coding.h"
#endif
namespace tensorflow {
// Assigns base[0..bytes-1] to *s
void CopyFromArray(string* s, const char* base, size_t bytes);
-// Encodes a list of ResourceHandle protos in the given string.
-void EncodeResourceHandleList(const ResourceHandle* handles, int64 n,
- string* out);
-
-// Decodes a list of ResourceHandle protos from the given string.
-bool DecodeResourceHandleList(const string& in, ResourceHandle* ps, int64 n);
-
} // namespace port
} // namespace tensorflow
+++ /dev/null
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/platform/variant_coding.h"
-
-#include <vector>
-#include "tensorflow/core/framework/tensor.pb.h"
-#include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/lib/core/coding.h"
-#include "tensorflow/core/lib/core/stringpiece.h"
-#include "tensorflow/core/lib/strings/strcat.h"
-
-namespace tensorflow {
-namespace port {
-
-void EncodeVariantList(const Variant* variant_array, int64 n, string* out) {
- out->clear();
- string rest;
- for (int i = 0; i < n; ++i) {
- string s;
- variant_array[i].Encode(&s);
- core::PutVarint32(out, s.length());
- strings::StrAppend(&rest, s);
- }
- strings::StrAppend(out, rest);
-}
-
-bool DecodeVariantList(const string& in, Variant* variant_array, int64 n) {
- std::vector<uint32> sizes(n);
- StringPiece reader(in);
- int64 total = 0;
- for (auto& size : sizes) {
- if (!core::GetVarint32(&reader, &size)) return false;
- total += size;
- }
- if (total != static_cast<int64>(reader.size())) {
- return false;
- }
-
- for (int i = 0; i < n; ++i) {
- if (variant_array[i].is_empty()) {
- variant_array[i] = VariantTensorDataProto();
- }
- string str(reader.data(), sizes[i]);
- if (!variant_array[i].Decode(str)) return false;
- if (!DecodeUnaryVariant(&variant_array[i])) {
- LOG(ERROR) << "Could not decode variant with type_name: \""
- << variant_array[i].TypeName()
- << "\". Perhaps you forgot to register a "
- "decoder via REGISTER_UNARY_VARIANT_DECODE_FUNCTION?";
- return false;
- }
- reader.remove_prefix(sizes[i]);
- }
- return true;
-}
-
-} // end namespace port
-} // end namespace tensorflow
+++ /dev/null
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_PLATFORM_VARIANT_CODING_H_
-#define TENSORFLOW_PLATFORM_VARIANT_CODING_H_
-
-#include "tensorflow/core/framework/variant.h"
-#include "tensorflow/core/framework/variant_encode_decode.h"
-
-#ifdef PLATFORM_GOOGLE
-#include "tensorflow/core/platform/google/variant_cord_coding.h"
-#endif
-
-namespace tensorflow {
-namespace port {
-
-// Encodes an array of Variant objects in to the given string.
-// `variant_array` is assumed to point to an array of `n` Variant objects.
-void EncodeVariantList(const Variant* variant_array, int64 n, string* out);
-
-// Decodes an array of Variant objects from the given string.
-// `variant_array` is assumed to point to an array of `n` Variant objects.
-bool DecodeVariantList(const string& in, Variant* variant_array, int64 n);
-
-} // end namespace port
-} // end namespace tensorflow
-
-#endif // TENSORFLOW_PLATFORM_VARIANT_CODING_H_
projects / platform / upstream / tensorflow.git / commitdiff