[TensorV2] Feature Scaling Functions

This pull request adds two new feature scaling functions - standardization and normalization - to the Tensor class. These functions help users preprocess input data before feeding it into models, improving model performance and accuracy.

**Changes proposed in this PR:**
* Added normalization() function to rescale values to a range between 0 and 1
* Added standardization() function to center data around the mean and scales to a standard deviation of 1

**Self-evaluation:**
1. Build test: [X]Passed [ ]Failed [ ]Skipped
2. Run test:   [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: Donghyeon Jeong <dhyeon.jeong@samsung.com>

diff --git a/nntrainer/tensor/float_tensor.cpp b/nntrainer/tensor/float_tensor.cpp
index 963a27572513204af3d56fba4b64d38f097283c3..1e7690bc145ac10f8ff4608b3e5105ca6220a482 100644 (file)
--- a/nntrainer/tensor/float_tensor.cpp
+++ b/nntrainer/tensor/float_tensor.cpp
@@ -688,6 +688,10 @@ TensorV2 &FloatTensor::sum(unsigned int axis, TensorV2 &output, float alpha,
   return output;
 }
 
+float FloatTensor::l2norm() const {
+  return snrm2(size(), (float *)getData(), 1);
+}
+
 TensorV2 &FloatTensor::pow(float exponent, TensorV2 &output) const {
   auto f = [exponent](float in) { return powf(in, exponent); };
   apply(f, output);
@@ -812,6 +816,16 @@ float FloatTensor::max_abs() const {
   return *(data + idx);
 }
 
+float FloatTensor::maxValue() const {
+  const float *data = (float *)getData();
+  return *std::max_element(data, data + size());
+}
+
+float FloatTensor::minValue() const {
+  const float *data = (float *)getData();
+  return *std::min_element(data, data + size());
+}
+
 TensorV2 &FloatTensor::transpose(const std::string &direction,
                                  TensorV2 &output) const {
   unsigned int SL, SI, SJ, SK;

diff --git a/nntrainer/tensor/float_tensor.h b/nntrainer/tensor/float_tensor.h
index cfef9a9e19fd0af6b885fd31d53eafcee23cf228..1ac800937602030eddcfbc40290ef07129324cf9 100644 (file)
--- a/nntrainer/tensor/float_tensor.h
+++ b/nntrainer/tensor/float_tensor.h
@@ -286,6 +286,11 @@ public:
   TensorV2 &sum(unsigned int axis, TensorV2 &output, float alpha,
                 float beta) const override;
 
+  /**
+   * @copydoc TensorV2::l2norm
+   */
+  float l2norm() const override;
+
   /**
    * @copydoc TensorV2::pow(float exponent, TensorV2 &output)
    */
@@ -347,6 +352,15 @@ public:
    * @copydoc TensorV2::max_abs()
    */
   float max_abs() const override;
+  /**
+   * @copydoc Tensor::maxValue()
+   */
+  float maxValue() const override;
+
+  /**
+   * @copydoc Tensor::minValue()
+   */
+  float minValue() const override;
 
   /**
    * @copydoc TensorV2::transpose(const std::string &direction, TensorV2 &out)

diff --git a/nntrainer/tensor/half_tensor.cpp b/nntrainer/tensor/half_tensor.cpp
index 5dd2379796ec93256b45972f6db6939ebf5734c8..cff06918952eebc7961ec155335bd2d0bd84b0aa 100644 (file)
--- a/nntrainer/tensor/half_tensor.cpp
+++ b/nntrainer/tensor/half_tensor.cpp
@@ -638,6 +638,10 @@ TensorV2 &HalfTensor::sum(unsigned int axis, TensorV2 &output, float alpha,
   return output;
 }
 
+float HalfTensor::l2norm() const {
+  return snrm2(size(), (_FP16 *)getData(), 1);
+}
+
 TensorV2 &HalfTensor::pow(float exponent, TensorV2 &output) const {
   auto f = [exponent](float in) {
     return static_cast<_FP16>(powf(in, exponent));
@@ -1071,6 +1075,16 @@ float HalfTensor::max_abs() const {
   return (float)(*(data + idx));
 }
 
+float HalfTensor::maxValue() const {
+  const _FP16 *data = (_FP16 *)getData();
+  return (float)*std::max_element(data, data + size());
+}
+
+float HalfTensor::minValue() const {
+  const _FP16 *data = (_FP16 *)getData();
+  return (float)*std::min_element(data, data + size());
+}
+
 TensorV2 &HalfTensor::transpose(const std::string &direction,
                                 TensorV2 &output) const {
   unsigned int SL, SI, SJ, SK;

diff --git a/nntrainer/tensor/half_tensor.h b/nntrainer/tensor/half_tensor.h
index 087bc4668be0faaa873741940a9d02e904084f1b..57451e3517c575facbda03d14ca6997bddd9962e 100644 (file)
--- a/nntrainer/tensor/half_tensor.h
+++ b/nntrainer/tensor/half_tensor.h
@@ -285,6 +285,11 @@ public:
   TensorV2 &sum(unsigned int axis, TensorV2 &output, float alpha,
                 float beta) const override;
 
+  /**
+   * @copydoc TensorV2::l2norm
+   */
+  float l2norm() const override;
+
   /**
    * @copydoc TensorV2::pow(float exponent, TensorV2 &output)
    */
@@ -347,6 +352,16 @@ public:
    */
   float max_abs() const override;
 
+  /**
+   * @copydoc Tensor::maxValue()
+   */
+  float maxValue() const override;
+
+  /**
+   * @copydoc Tensor::minValue()
+   */
+  float minValue() const override;
+
   /**
    * @copydoc TensorV2::transpose(const std::string &direction, TensorV2 &out)
    */

diff --git a/nntrainer/tensor/tensor_base.h b/nntrainer/tensor/tensor_base.h
index fab255b175e9f4f81f8d845e41c7aaf9f9c9b578..eb3d8f6affa779a33e0de72508237c6ee7473cc4 100644 (file)
--- a/nntrainer/tensor/tensor_base.h
+++ b/nntrainer/tensor/tensor_base.h
@@ -293,6 +293,11 @@ public:
   virtual TensorV2 &sum(unsigned int axis, TensorV2 &output, float alpha,
                         float beta) const = 0;
 
+  /**
+   * @copydoc TensorV2::l2norm
+   */
+  virtual float l2norm() const = 0;
+
   /**
    * @copydoc TensorV2::pow(float exponent, TensorV2 &output)
    */
@@ -384,6 +389,16 @@ public:
    */
   virtual float max_abs() const = 0;
 
+  /**
+   * @copydoc TensorV2::maxValue()
+   */
+  virtual float maxValue() const = 0;
+
+  /**
+   * @copydoc TensorV2::minValue()
+   */
+  virtual float minValue() const = 0;
+
   /**
    * @copydoc TensorV2::transpose(const std::string &direction, TensorV2 &out)
    */

diff --git a/nntrainer/tensor/tensor_v2.cpp b/nntrainer/tensor/tensor_v2.cpp
index 4c415e0813369ca7efb1dbd65eb775ab98037b19..4c6b109d700242a15b0d137a49441948ee640ae1 100644 (file)
--- a/nntrainer/tensor/tensor_v2.cpp
+++ b/nntrainer/tensor/tensor_v2.cpp
@@ -523,6 +523,57 @@ TensorV2 &TensorV2::erf(TensorV2 &output) const {
   return output;
 }
 
+float TensorV2::l2norm() const { return itensor->l2norm(); }
+
+void TensorV2::normalization_i() {
+  NNTR_THROW_IF(!getContiguous(), std::invalid_argument)
+    << getName() << " is not contiguous, cannot do normalization.";
+
+  const float min = minValue();
+  const float max = maxValue();
+
+  if (max == min) {
+    TensorV2 tmp = *this;
+    this->subtract_i(tmp);
+  } else {
+    this->subtract_i(min);
+    this->divide_i(max - min);
+  }
+}
+
+void TensorV2::standardization_i() {
+  TensorV2 mean_by_batch = this->sum_by_batch();
+  mean_by_batch.divide_i(getDim().getFeatureLen());
+
+  this->subtract_i(mean_by_batch);
+  TensorV2 std_dev_by_batch(batch(), 1, 1, 1, getFormat(), getDataType());
+  std_dev_by_batch.setZero();
+
+  /// @todo remove conditional statement
+  if (getDataType() == ml::train::TensorDim::DataType::FP32) {
+    float *std_dev = std_dev_by_batch.getData<float>();
+
+    for (unsigned int k = 0; k < batch(); ++k) {
+      TensorV2 sub_this = this->getBatchSlice(k, 1);
+      std_dev[k] = sub_this.l2norm();
+    }
+  } else if (getDataType() == ml::train::TensorDim::DataType::FP16) {
+#ifdef ENABLE_FP16
+    _FP16 *std_dev = std_dev_by_batch.getData<_FP16>();
+
+    for (unsigned int k = 0; k < batch(); ++k) {
+      TensorV2 sub_this = this->getBatchSlice(k, 1);
+      std_dev[k] = static_cast<_FP16>(sub_this.l2norm());
+    }
+#else
+    throw std::invalid_argument("Error: enable-fp16 is not enabled");
+#endif
+  }
+
+  std_dev_by_batch.divide_i(getDim().getFeatureLen());
+  this->divide_i(std_dev_by_batch);
+}
+
 TensorV2 TensorV2::dot(TensorV2 const &input, bool trans, bool trans_in) const {
   TensorV2 output("", this->getFormat(), this->getDataType());
   dot(input, output, trans, trans_in);
@@ -852,6 +903,10 @@ float TensorV2::max_abs() const {
   return itensor->max_abs();
 }
 
+float TensorV2::maxValue() const { return itensor->maxValue(); }
+
+float TensorV2::minValue() const { return itensor->minValue(); }
+
 TensorV2 TensorV2::transpose(const std::string &direction) const {
   TensorV2 output(getDim());
   transpose(direction, output);

diff --git a/nntrainer/tensor/tensor_v2.h b/nntrainer/tensor/tensor_v2.h
index c7355e93a547c43b9065ecd84af8d4e58b1c3aa0..6e99228d234f4e173d42e44d5ca7878a66afe1b2 100644 (file)
--- a/nntrainer/tensor/tensor_v2.h
+++ b/nntrainer/tensor/tensor_v2.h
@@ -918,6 +918,36 @@ public:
    */
   TensorV2 &erf(TensorV2 &output) const;
 
+  /**
+   * @brief     l2norm the Tensor elements
+   * @retval    Calculated l2norm
+   */
+  float l2norm() const;
+
+  /**
+   * @brief     Normalize the Tensor elements
+   * @retval    Calculated Tensor
+   */
+  TensorV2 &normalization(TensorV2 &output) const;
+
+  /**
+   * @brief     Standardize the Tensor elements
+   * @retval    Calculated Tensor
+   */
+  TensorV2 &standardization(TensorV2 &output) const;
+
+  /**
+   * @brief     Normalize the Tensor elements in-place
+   * @retval    Calculated Tensor
+   */
+  void normalization_i();
+
+  /**
+   * @brief     Standardize the Tensor elements in-place
+   * @retval    Calculated Tensor
+   */
+  void standardization_i();
+
   /**
    * @brief     Dot Product of Tensor ( equal MxM )
    * @details   This applies dot of the last dimension of this and second-last
@@ -1155,6 +1185,18 @@ public:
    */
   float max_abs() const;
 
+  /**
+   * @brief  return maximum value
+   * @retval Maximum value of the tensor data
+   */
+  float maxValue() const;
+
+  /**
+   * @brief  return minimum value
+   * @retval Minimum value of the tensor data
+   */
+  float minValue() const;
+
   /**
    * @brief  Transpose Tensor
    * @param  direction to transpose ex) 0:2:1