From 180f6450db997cd49da9c83dc3fcc3573f379266 Mon Sep 17 00:00:00 2001 From: Jihoon Lee Date: Wed, 14 Apr 2021 20:02:18 +0900 Subject: [PATCH] [Tflite] Fix tf shape building This patch fixes tf shape building, by adding `eff_dim_flag` and `dyn_dim_flag`. `eff_dim_flag` checks which dimension are actually used so that to be squeezed. `dyn_dim_flag` checks which dimension should be set to -1 (meaning it is not fixed) With those flag set, our tensor dim is able to convert to the dimension representation that supports `-1` and not fixed size Those fixes are apply to fc layer for now **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Jihoon Lee --- nntrainer/compiler/tflite_interpreter.cpp | 60 ++++++------------------- nntrainer/layers/fc_layer.cpp | 21 +++++---- nntrainer/tensor/tensor_dim.cpp | 25 ++++++++++- nntrainer/tensor/tensor_dim.h | 68 +++++++++++++++++++++++++++-- test/unittest/unittest_nntrainer_layers.cpp | 7 --- test/unittest/unittest_nntrainer_tensor.cpp | 28 ++++++++++++ 6 files changed, 143 insertions(+), 66 deletions(-) diff --git a/nntrainer/compiler/tflite_interpreter.cpp b/nntrainer/compiler/tflite_interpreter.cpp index ab766a3..152a384 100644 --- a/nntrainer/compiler/tflite_interpreter.cpp +++ b/nntrainer/compiler/tflite_interpreter.cpp @@ -414,39 +414,6 @@ buildOperatorCodes(const TfOpIdxMap &map, flatbuffers::FlatBufferBuilder &fbb) { return fbb.CreateVector(fb_op_codes); } -/** - * @brief get Serialized Tensor shape - * @note when building tensorshape, the tensor shape must be final, when we - * reach here it, the tensordim should be final which means if transpose is - * needed, it should be done already. - * @param dim Tensor dimension to convert - * @param type tflite Tensor type - * @param effective_dimension bitflag of valid 'nchw' dimension, ex) 0b1011 - * means,nhw is effective - * @param is_signature check if the given dimension is regarded as a tensor - * signature - * @return flatbuffers::Offset> - */ -flatbuffers::Offset> -buildTensorShape(const TensorDim &dim, flatbuffers::FlatBufferBuilder &fbb, - int effective_dimension, bool is_signature = false) { - std::vector fb_dimension; - - /// batch dimension is always considered with signature - if ((effective_dimension >> 3) & 1) { - fb_dimension.push_back(is_signature ? -1 : dim.batch()); - } - - for (unsigned int i = 1; i < MAXDIM; ++i) { - if (((effective_dimension >> (MAXDIM - i - 1)) & 1) == 0) { - continue; - } - fb_dimension.push_back(dim.getTensorDim(i)); - } - - return fbb.CreateVector(fb_dimension); -} - flatbuffers::Offset>> buildTensors(const TfOpIdxMap &map, flatbuffers::FlatBufferBuilder &fbb) { /// @todo: the actual (suqeezed) tensor dimension must be known before coming @@ -458,18 +425,16 @@ buildTensors(const TfOpIdxMap &map, flatbuffers::FlatBufferBuilder &fbb) { fb_tensors.reserve(variables.size()); auto create_tensor = [&fbb, &buffer_map](const Var_Grad *var) { - /// @fixme: There is a fixed assumption that effective_dimension is 0b1001, - /// this is not true, this should be supported inherently from TensorDim - /// class. - int FIX_ME_FLAG = 0b1001; - auto shape = buildTensorShape(var->getDim(), fbb, FIX_ME_FLAG, false); - - /// in *.tflite spec, there is tensor dimension and tensor signature - /// dimension the only difference is that tensor signature dimension gets - /// the value of -1 which denotes the size is unspecified. Mostly, batch - /// dimension has the signature of -1, we don't have such concept yet so - /// this is handled by boolean for now - auto shape_sig = buildTensorShape(var->getDim(), fbb, FIX_ME_FLAG, true); + bool need_shape_signature = var->getDim().is_dynamic(); + std::vector eff_dim = var->getDim().getEffectiveDimension(); + auto shape = fbb.CreateVector(eff_dim); + + decltype(shape) shape_sig; + if (need_shape_signature) { + std::vector dyn_dim = var->getDim().getEffectiveDimension(true); + shape_sig = fbb.CreateVector(dyn_dim); + } + auto name = fbb.CreateString(var->getName()); auto tensor = var->getVariableRef(); @@ -481,8 +446,11 @@ buildTensors(const TfOpIdxMap &map, flatbuffers::FlatBufferBuilder &fbb) { tflite::TensorBuilder builder(fbb); builder.add_name(name); builder.add_buffer(buffer_idx); + builder.add_type(tflite::TensorType_FLOAT32); builder.add_shape(shape); - builder.add_shape_signature(shape_sig); + if (need_shape_signature) { + builder.add_shape_signature(shape_sig); + } return builder.Finish(); }; diff --git a/nntrainer/layers/fc_layer.cpp b/nntrainer/layers/fc_layer.cpp index ec117b9..e476a3c 100644 --- a/nntrainer/layers/fc_layer.cpp +++ b/nntrainer/layers/fc_layer.cpp @@ -43,25 +43,28 @@ int FullyConnectedLayer::initialize(Manager &manager) { throw std::invalid_argument("Fully connected layer takes only one input"); } - output_dim[0] = input_dim[0]; - output_dim[0].width(unit); + auto &in_dim = input_dim[0]; + /// @todo fc actaully supports multidimensions. EffDimFlag shouldn't be fixed + /// like this. + in_dim.setEffDimFlag(0b1001); + in_dim.setDynDimFlag(0b1000); - TensorDim bias_dim = TensorDim(); - bias_dim.setTensorDim(3, unit); + output_dim[0] = in_dim; + output_dim[0].width(unit); - TensorDim dim = output_dim[0]; - dim.height(input_dim[0].width()); - dim.batch(1); + TensorDim bias_dim(1, 1, 1, unit, 0b0001); + TensorDim weight_dim(1, 1, in_dim.width(), unit, 0b0011); if (weights.empty()) { weights.reserve(2); - weights.emplace_back(dim, weight_initializer, weight_regularizer, + weights.emplace_back(weight_dim, weight_initializer, weight_regularizer, weight_regularizer_constant, true, false, "FC:weight"); weights.emplace_back(bias_dim, bias_initializer, WeightRegularizer::NONE, 1.0f, true, false, "FC:bias"); manager.trackWeights(weights); } else { - weights[FCParams::weight].reset(dim, weight_initializer, weight_regularizer, + weights[FCParams::weight].reset(weight_dim, weight_initializer, + weight_regularizer, weight_regularizer_constant, true); weights[FCParams::bias].reset(bias_dim, bias_initializer, WeightRegularizer::NONE, 1.0f, true); diff --git a/nntrainer/tensor/tensor_dim.cpp b/nntrainer/tensor/tensor_dim.cpp index d107815..6676c0a 100644 --- a/nntrainer/tensor/tensor_dim.cpp +++ b/nntrainer/tensor/tensor_dim.cpp @@ -33,7 +33,7 @@ TensorDim::TensorDim(const std::string &shape) : TensorDim() { TensorDim &TensorDim::operator=(const TensorDim &rhs) { using std::swap; - TensorDim tmp(rhs.batch(), rhs.channel(), rhs.height(), rhs.width()); + TensorDim tmp(rhs); swap(*this, tmp); return *this; } @@ -154,6 +154,29 @@ const unsigned int &TensorDim::operator[](const unsigned int index) const { void TensorDim::reverse() { std::reverse(dim, dim + MAXDIM); } +std::vector TensorDim::getEffectiveDimension(bool dynamic) const { + std::vector eff_dim; + eff_dim.reserve(eff_dim_flag.count()); + + auto get_axis = [dynamic, this](unsigned int axis) -> int { + if (dynamic && dyn_dim_flag[MAXDIM - axis - 1]) { + return -1; + } + + return dim[axis]; + }; + + for (unsigned int i = 0; i < MAXDIM; ++i) { + /// flip dim_flag to effectively match with our cognition + /// ex) 3:5:1:1 -> 3:5, we are setting eff_dim_flag to 0b1100 + if (eff_dim_flag[MAXDIM - i - 1]) { + eff_dim.push_back(get_axis(i)); + } + } + + return eff_dim; +} + std::ostream &operator<<(std::ostream &out, TensorDim const &d) { out << "Shape: " << d.batch() << ":" << d.channel() << ":" << d.height() << ":" << d.width() << std::endl; diff --git a/nntrainer/tensor/tensor_dim.h b/nntrainer/tensor/tensor_dim.h index 5398e69..f2f70fb 100644 --- a/nntrainer/tensor/tensor_dim.h +++ b/nntrainer/tensor/tensor_dim.h @@ -19,6 +19,9 @@ #include #include +#include +#include + namespace nntrainer { constexpr const size_t MAXDIM = 4; @@ -32,8 +35,13 @@ public: /** * @brief Construct a new Tensor Dim object * + * @param eff_dim_flag_ effective dimension flag (1 means it's effective) + * @param dyn_dim_flag_ dynamic dimension flag (1 means it's unspecified) */ - TensorDim() { + TensorDim(const std::bitset &eff_dim_flag_ = 0b1111, + const std::bitset &dyn_dim_flag_ = 0b0000) : + eff_dim_flag(eff_dim_flag_), + dyn_dim_flag(dyn_dim_flag_) { for (size_t i = 0; i < MAXDIM; ++i) { dim[i] = 0; } @@ -70,9 +78,13 @@ public: * @param c channel * @param h height * @param w width + * @param eff_dim_flag_ dimension bit flag to calculate the dynamic + * dimension, rightmost is width */ - TensorDim(unsigned int b, unsigned int c, unsigned int h, unsigned int w) : - TensorDim() { + TensorDim(unsigned int b, unsigned int c, unsigned int h, unsigned int w, + const std::bitset &eff_dim_flag_ = 0b1111, + const std::bitset &dyn_dim_flag_ = 0b0000) : + TensorDim(eff_dim_flag_, dyn_dim_flag_) { setTensorDim(0, b); setTensorDim(1, c); setTensorDim(2, h); @@ -114,6 +126,31 @@ public: TensorDim &operator=(TensorDim &&rhs) noexcept; /** + * @brief Set the Dim Flag to retrieve effective dimension + * @note eg) if dimension 4:1:10:1 should be squeezed to 4:10, + * set this to 0b1010, rightmost is width + * + * @param dim_flag_ dimension bit to calculate, rightmost is width + */ + void setEffDimFlag(const std::bitset &dim_flag_) { + eff_dim_flag = dim_flag_; + } + + /** + * @brief Set the dynamic Dim Flag to retrieve dynamic dimension (that can + * change during running) + * @note eg) if dimension 4:1:10:1 should be squeezed to dynamic to batch, + * set this to 0b1000, rightmost is width + * @note when setting dynamic dimension, the calculation must remain + * independent of the dynamic dimension. Please check this :) + * + * @param dim_flag_ dimension bit to calculate, rightmost is width + */ + void setDynDimFlag(const std::bitset &dim_flag_) { + dyn_dim_flag = dim_flag_; + } + + /** * @brief swap variable of Conv2D Layer * @parma[out] lhs Optimizer * @parma[in] rhs Optimizer @@ -123,6 +160,8 @@ public: std::begin(rhs.dim)); std::swap(lhs.len, rhs.len); std::swap(lhs.feature_len, rhs.feature_len); + std::swap(lhs.eff_dim_flag, rhs.eff_dim_flag); + std::swap(lhs.dyn_dim_flag, rhs.dyn_dim_flag); } /** @@ -323,6 +362,23 @@ public: */ void reverse(); + /** + * @brief Get the Effective Dimension of the current + * @note dynamic dimension is returned as -1 + * + * @param dynamic if dimension has to be considering dynamic set this to ture + * @return std::vector integer vector + */ + std::vector getEffectiveDimension(bool dynamic = false) const; + + /** + * @brief check if tensor is dynamic + * + * @return true any of dyn_dim_flag is set + * @return false none of dyn_dim_flag is set + */ + bool is_dynamic() const { return dyn_dim_flag.any(); } + private: /** * @brief reset length @@ -330,6 +386,12 @@ private: */ void resetLen(); + std::bitset eff_dim_flag; /**< dimension bit flag to define effective + dimension size */ + + std::bitset dyn_dim_flag; /**< dimension bit flag to define +dynamic dimension size */ + unsigned int dim[MAXDIM]; /**< underlying dimension type */ unsigned int len; /**< number of elements */ unsigned int feature_len; /**< number of feature lements */ diff --git a/test/unittest/unittest_nntrainer_layers.cpp b/test/unittest/unittest_nntrainer_layers.cpp index 9f9b561..3f3e7f7 100644 --- a/test/unittest/unittest_nntrainer_layers.cpp +++ b/test/unittest/unittest_nntrainer_layers.cpp @@ -2096,13 +2096,6 @@ TEST(nntrainer_LossLayer, setProperty_individual_02_n) { nntrainer::exception::not_supported); } -TEST(nntrainer_ActivationLayer, init_01_n) { - nntrainer::Manager manager{true, false}; - manager.setInferenceInOutMemoryOptimization(false); - nntrainer::ActivationLayer layer; - EXPECT_THROW(layer.initialize(manager), std::invalid_argument); -} - TEST(nntrainer_ActivationLayer, init_02_p) { nntrainer::Manager manager{true, false}; manager.setInferenceInOutMemoryOptimization(false); diff --git a/test/unittest/unittest_nntrainer_tensor.cpp b/test/unittest/unittest_nntrainer_tensor.cpp index 3fdd3bb..594d5a9 100644 --- a/test/unittest/unittest_nntrainer_tensor.cpp +++ b/test/unittest/unittest_nntrainer_tensor.cpp @@ -37,6 +37,34 @@ TEST(nntrainer_TensorDim, ctor_initializer_p) { EXPECT_EQ(nntrainer::TensorDim(b, c, h, w), t); } +TEST(nntrianer_TensorDim, effective_dimension_p) { + nntrainer::TensorDim t(3, 2, 4, 5); + EXPECT_EQ(t.getEffectiveDimension(), std::vector({3, 2, 4, 5})); + + t.setEffDimFlag(0b1101); + EXPECT_EQ(t.getEffectiveDimension(), std::vector({3, 2, 5})); + + t.setEffDimFlag(0b0011); + EXPECT_EQ(t.getEffectiveDimension(), std::vector({4, 5})); + + t.setEffDimFlag(0b1111); + EXPECT_EQ(t.getEffectiveDimension(), std::vector({3, 2, 4, 5})); + + t.setEffDimFlag(0b1100); + EXPECT_EQ(t.getEffectiveDimension(), std::vector({3, 2})); + + t.setDynDimFlag(0b1100); + EXPECT_EQ(t.getEffectiveDimension(true), std::vector({-1, -1})); + + auto copied_t = t; + EXPECT_EQ(copied_t.getEffectiveDimension(), std::vector({3, 2})); + EXPECT_EQ(copied_t.getEffectiveDimension(true), std::vector({-1, -1})); + + auto moved_t = std::move(copied_t); + EXPECT_EQ(moved_t.getEffectiveDimension(), std::vector({3, 2})); + EXPECT_EQ(moved_t.getEffectiveDimension(true), std::vector({-1, -1})); +} + TEST(nntrainer_TensorDim, ctor_initializer_n) { EXPECT_THROW(nntrainer::TensorDim t({1, 2, 3, 4, 5}), std::invalid_argument); } -- 2.7.4