From 81780e66cbcd2c417b30b765b546e7a9510c3195 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=D0=A2=D0=B8=D0=BC=D1=83=D1=80=20=D0=9E=D1=82=D0=B5=D0=BB?= =?utf8?q?=D0=BB=D0=BE=D0=B2=D0=B8=D1=87=20=D0=90=D0=B1=D0=BB=D1=8F=D0=B7?= =?utf8?q?=D0=B8=D0=BC=D0=BE=D0=B2/AI=20Tools=20Lab=20/SRR/Staff=20Enginee?= =?utf8?q?r/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Fri, 30 Nov 2018 16:57:27 +0300 Subject: [PATCH] [nnc] Utility to generate input tensors (#2143) Implemented utility that generates input data in the hdf5 format. Signed-off-by: Timur Ablyazimov --- contrib/nnc/utils/CMakeLists.txt | 3 + contrib/nnc/utils/input_gen/CMakeLists.txt | 9 ++ contrib/nnc/utils/input_gen/tensor_gen.cpp | 189 +++++++++++++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 contrib/nnc/utils/input_gen/CMakeLists.txt create mode 100644 contrib/nnc/utils/input_gen/tensor_gen.cpp diff --git a/contrib/nnc/utils/CMakeLists.txt b/contrib/nnc/utils/CMakeLists.txt index 51399c4..3298866 100644 --- a/contrib/nnc/utils/CMakeLists.txt +++ b/contrib/nnc/utils/CMakeLists.txt @@ -8,3 +8,6 @@ add_subdirectory(tflite_model_generator) add_subdirectory(tflite_dot_dumper) add_subdirectory(caffe_dot_dumper) add_subdirectory(caffe2_dot_dumper) + +# input tensors generator +add_subdirectory(input_gen) diff --git a/contrib/nnc/utils/input_gen/CMakeLists.txt b/contrib/nnc/utils/input_gen/CMakeLists.txt new file mode 100644 index 0000000..57734fe --- /dev/null +++ b/contrib/nnc/utils/input_gen/CMakeLists.txt @@ -0,0 +1,9 @@ +find_package(HDF5 COMPONENTS CXX QUIET) + +if(NOT HDF5_FOUND) + return() +endif(NOT HDF5_FOUND) + +add_executable(tensor_gen tensor_gen.cpp) +target_include_directories(tensor_gen PRIVATE ${HDF5_INCLUDE_DIRS}) +target_link_libraries(tensor_gen ${HDF5_CXX_LIBRARIES}) diff --git a/contrib/nnc/utils/input_gen/tensor_gen.cpp b/contrib/nnc/utils/input_gen/tensor_gen.cpp new file mode 100644 index 0000000..c63d1d2 --- /dev/null +++ b/contrib/nnc/utils/input_gen/tensor_gen.cpp @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +class Tensor; + +static void iterate(Tensor& tensor, function&)> on_loop); + +class Tensor { +public: + explicit Tensor(const vector& shape) : _shape(shape), _data(0), _num_elems(1) { + _strides.resize(shape.size()); + + for (int i = _shape.size() - 1; i >= 0; --i) { + _strides[i] = _num_elems; + _num_elems *= _shape[i]; + } + + _data = new float[_num_elems]; + } + + ~Tensor() { delete[] _data; } + int rank() const { return _shape.size(); } + int dim(int d) const { return _shape[d]; } + float* data() { return _data; } + float numElems() const { return _num_elems; } + + float& at(const vector& coords) { + int offset = 0; + + for (int i = 0; i < coords.size(); ++i) + offset += coords[i] * _strides[i]; + + return _data[offset]; + } + + Tensor transpose(const vector& reshape) { + vector tr_shape(_shape.size()); + + for (int i = 0; i < _shape.size(); ++i) + tr_shape[i] = _shape[reshape[i]]; + + Tensor result(tr_shape); + auto on_loop = [this, &reshape, &result](vector& coords) { + vector tr_coords; + + for (int i = 0; i < rank(); ++i) + tr_coords[i] = coords[reshape[i]]; + + result.at(tr_coords) = at(coords); + }; + iterate(*this, on_loop); + return result; + } + +private: + vector _shape; + vector _strides; + float* _data; + hsize_t _num_elems; +}; + +static void fillTensor(Tensor& tensor) { + int v = 10; + + for (int i = 0; i < tensor.numElems(); ++i) { + tensor.data()[i] = v; + v += 10; + } +} + +static void iterate(Tensor& tensor, function&)> on_loop) { + int num_dims = tensor.rank(); + vector coords(num_dims, 0); + vector dims(num_dims); + + for (int i = 0; i < num_dims; ++i) + dims[i] = tensor.dim(i); + + for (;;) { + on_loop(coords); + + int i; + for (i = num_dims - 1; i >= 0; --i) { + if (coords[i] < dims[i] - 1) { + ++coords[i]; + break; + } + } + + if (i < 0) + break; + + fill(coords.begin() + i + 1, coords.end(), 0); + } +} + +static void dumpTensor(Tensor& tensor) { + auto on_loop = [&tensor](vector& coords) { + for (int i = 0; i < tensor.rank(); ++i) { + if (i > 0) + cout << ", "; + + cout << coords[i]; + } + + cout << " = " << tensor.at(coords) << endl; + }; + + iterate(tensor, on_loop); +} + +static vector dimensions; + +static void writeTensorToDatFile(const string& file_name, Tensor& tensor) { + ofstream of(file_name + ".dat", ios_base::binary); + + if (of.fail()) + cout << "Could not output tensor to the: " << file_name + ".dat"; + + of.write(reinterpret_cast(tensor.data()), tensor.numElems() * sizeof(float)); +} + +static void writeTensorToHDF5File(const string& tensor_name, const string& file_name, Tensor& tensor) { + H5::H5File h5File(file_name + ".hdf5", H5F_ACC_TRUNC); + H5::DataSpace dataspace(dimensions.size(), &dimensions[0]); + auto dataset = h5File.createDataSet(tensor_name, H5::PredType::IEEE_F32BE, dataspace); + dataset.write(tensor.data(), H5::PredType::NATIVE_FLOAT); +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + cout << "Usage: " << argv[0] << " dim0, dim1, dim2, ..." << endl; + cout << "Where: dim0, dim1, dim2, ... are the generated tensor shape dimensions" << endl; + return 1; + } + + for (int i = 3; i < argc; ++i) { + try { + int d = stoi(argv[i]); + + if (d <= 0) { + cout << "The dimensions must be positive values. This is not a correct dimension value: " << d << endl; + return 1; + } + + dimensions.push_back(d); + } catch(const invalid_argument&) { + cout << "The parameter does not look as an integer value: " << argv[i] << endl; + return 1; + } catch(const out_of_range&) { + cout << "The value is out of the C++ \"int\" type range: " << argv[i] << endl; + return 1; + } + } + + Tensor caffe_tensor(dimensions); + fillTensor(caffe_tensor); + dumpTensor(caffe_tensor); + writeTensorToHDF5File(argv[1], "in_" + string(argv[2]) + "_caffe", caffe_tensor); + + vector tf_reshape{0, 2, 3, 1}; + Tensor tf_tensor = caffe_tensor.transpose(tf_reshape); + writeTensorToDatFile(string(argv[2]) + "_tf", tf_tensor); + + return 0; +} -- 2.7.4