From e41844633121b1a87530177237386e4191e3b876 Mon Sep 17 00:00:00 2001 From: Jihoon Lee Date: Tue, 21 Dec 2021 20:47:31 +0900 Subject: [PATCH] [Context] out_dim -> out spec This patch prepare migration to tensor spec v2 by substituting out dimension to out specification inside layer_context **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Jihoon Lee --- nntrainer/layers/layer_context.cpp | 84 +++++++++++++++++++++++++++++++++++- nntrainer/layers/layer_context.h | 59 ++++++++++++++----------- nntrainer/tensor/tensor_wrap_specs.h | 1 + 3 files changed, 117 insertions(+), 27 deletions(-) diff --git a/nntrainer/layers/layer_context.cpp b/nntrainer/layers/layer_context.cpp index e81d567..f9cf8ce 100644 --- a/nntrainer/layers/layer_context.cpp +++ b/nntrainer/layers/layer_context.cpp @@ -12,22 +12,102 @@ #include "nntrainer_error.h" #include +#include +#include +#include #include #include #include #include namespace nntrainer { + +/** + * @brief rename specification + * + * @param spec spec to rename + * @param fn fn to rename + */ +static void renameSpec(VarGradSpecV2 &spec, + std::function fn) { + fn(spec.variable_spec.name); + if (spec.gradient_spec) { + fn(spec.gradient_spec->name); + } +} + +InitLayerContext::InitLayerContext(const std::vector &dim, + unsigned int num_req_out, bool in_place_, + const std::string &n, + const std::string &prefix_, + const float max_norm) : + input_dim(dim), + in_place(in_place_), + clip_by_global_norm(max_norm), + output_specs(), + num_requested_out(num_req_out), + name(n), + prefix(prefix_) { + NNTR_THROW_IF(!validate(), std::invalid_argument) + << "Invalid init context name: " << name + << " num inputs: " << getNumInputs(); + if (prefix.empty()) + prefix = name; // default prefix is the name +} + void InitLayerContext::setOutputDimensions( const std::vector &out_dim) { NNTR_THROW_IF(out_dim.size() < num_requested_out, std::invalid_argument) << "number of output dimension set is smaller than the number of out " "tensor slots " "requested, num output dimensions: " - << output_dim.size() << " slots to fill: " << num_requested_out + << out_dim.size() << " slots to fill: " << num_requested_out; << " context name: " << name; - output_dim = out_dim; + NNTR_THROW_IF(output_specs.size(), std::invalid_argument) + << "output specification already set, cannot set twice. Check if output is " + "already requested elsewhere"; + output_specs.reserve(out_dim.size()); + + for (unsigned i = 0u, sz = out_dim.size(); i < sz; ++i) { + auto spec = outSpec(out_dim.at(i)); + output_specs.push_back(std::move(spec)); + } +} + +VarGradSpecV2 InitLayerContext::outSpec(const TensorDim &dim, + const std::string &name, + TensorLifespan ls, + TensorLifespan grad_ls) { + VarGradSpecV2 spec; + spec.variable_spec.dim = dim; + spec.variable_spec.name = name; + spec.variable_spec.ls = ls; + spec.gradient_spec = std::make_unique(spec.variable_spec); + spec.gradient_spec->ls = grad_ls; + + return spec; +} + +void InitLayerContext::requestOutputs(std::vector &&out_specs) { + NNTR_THROW_IF(out_specs.size() < num_requested_out, std::invalid_argument) + << "number of output dimension set is smaller than the number of out " + "tensor slots requested, num output specification: " + << out_specs.size() << " slots to fill: " << num_requested_out; + NNTR_THROW_IF(output_specs.size(), std::invalid_argument) + << "output specification already set, cannot set twice. Check if output is " + "already requested elsewhere"; + output_specs.reserve(out_specs.size()); + + for (unsigned i = 0u, sz = out_specs.size(); i < sz; ++i) { + auto &spec = out_specs.at(i); + renameSpec(spec, [i](std::string &name) { name += std::to_string(i); }); + output_specs.push_back(std::move(spec)); + } +} + +const std::vector &InitLayerContext::getOutSpecs() { + return output_specs; } RunLayerContext::RunLayerContext(const std::string &name, bool trainable, diff --git a/nntrainer/layers/layer_context.h b/nntrainer/layers/layer_context.h index a39969c..f2f8cfe 100644 --- a/nntrainer/layers/layer_context.h +++ b/nntrainer/layers/layer_context.h @@ -44,20 +44,7 @@ public: */ InitLayerContext(const std::vector &dim, unsigned int num_req_out, bool in_place_, const std::string &n = "", - const std::string &prefix_ = "", - const float max_norm = 0.0) : - input_dim(dim), - in_place(in_place_), - clip_by_global_norm(max_norm), - num_requested_out(num_req_out), - name(n), - prefix(prefix_) { - NNTR_THROW_IF(!validate(), std::invalid_argument) - << "Invalid init context name: " << name - << " num inputs: " << getNumInputs(); - if (prefix.empty()) - prefix = name; // default prefix is the name - } + const std::string &prefix_ = "", const float max_norm = 0.0); /** * @brief get name by the layer @@ -115,7 +102,11 @@ public: * * @return std::vector& Output dimensions */ - const std::vector &getOutputDimensions() const { + const std::vector getOutputDimensions() const { + std::vector output_dim; + for (auto &spec : output_specs) { + output_dim.push_back(spec.variable_spec.dim); + } return output_dim; } @@ -237,14 +228,33 @@ public: unsigned int getNumTensors() const { return tensors_spec.size(); } /** - * @brief Update the dimensions for a requested tensor + * @brief create var grad specification with output default * - * @param idx index of the tensor (identifier) - * @param batch Updated batch size + * @param dim dimension dimension + * @param name name name + * @param ls variable lifespan + * @param grad_ls gradient lifespan + * @return VarGradSpecV2 var grad specification */ - void updateTensorSpec(unsigned int idx, unsigned int batch) { - std::get<0>(tensors_spec[idx]).batch(batch); - } + static VarGradSpecV2 + outSpec(const TensorDim &dim, const std::string &name = "out", + TensorLifespan ls = TensorLifespan::FORWARD_GRAD_LIFESPAN, + TensorLifespan grad_ls = TensorLifespan::BACKWARD_FUNC_LIFESPAN); + + /** + * @brief request outputs + * + * @param out_specs pack of out specification, name will be automatically + * indexed to prevent name clash + */ + void requestOutputs(std::vector &&out_specs); + + /** + * @brief Get the Out Specs object + * + * @return std::vector out specification + */ + const std::vector &getOutSpecs(); /** * @brief Validate the context @@ -278,13 +288,12 @@ public: bool executeInPlace() const { return in_place; } private: - std::vector input_dim; /**< Input dimensions for the layer */ - std::vector output_dim; /**< Output dimensions for the layer */ + std::vector input_dim; /**< Input dimensions for the layer */ bool in_place; /**< if the layer is expected to run in-place */ float clip_by_global_norm; /**< max norm value for clip by norm */ - std::vector output_spec; - std::vector weights_spec; /**< Specification for the weights */ + std::vector output_specs; /**< Specification for the output */ + std::vector weights_spec; /**< Specification for the weights */ std::vector tensors_spec; /**< Specification for the var_grad (trainable/non-trainable variables) */ diff --git a/nntrainer/tensor/tensor_wrap_specs.h b/nntrainer/tensor/tensor_wrap_specs.h index 7343b9b..f392e81 100644 --- a/nntrainer/tensor/tensor_wrap_specs.h +++ b/nntrainer/tensor/tensor_wrap_specs.h @@ -45,6 +45,7 @@ enum class TensorLifespan { CALC_GRAD_DERIV_LIFESPAN = 0b110, /**< tensor must not be reset before during the calc_grad and clac_deriv call, eg. temporary tensors needed during backward operations */ + FORWARD_GRAD_LIFESPAN = 0b101, /**< Forward + grad lifespan */ BACKWARD_FUNC_LIFESPAN = CALC_GRAD_DERIV_LIFESPAN, /**< Alias of CALC_GRAD_DERIV_LIFESPAN */ ITERATION_LIFESPAN = 0b111, /**< tensor must not be reset until the owning -- 2.7.4