[layer/test] Bug fix for dropout + unittest

This patch adds bug fix for dropout for training as well as inference
mode. Futher, added unittest:
- when dropout_rate is 0 or 100%, all the values are checked
- when dropout_rate between 0 and 100, weak check ensures that either
values are equal or one of the values (golden vs output) is 0
- when dropout_rate r between 0 and 100, strong check ensures that
100 - 2*r percentage of values must always match

All the checks for performed in the unittests.

Signed-off-by: Parichay Kapoor <pk.kapoor@samsung.com>

diff --git a/nntrainer/layers/dropout.cpp b/nntrainer/layers/dropout.cpp
index bc270e9..d9ad07b 100644 (file)
--- a/nntrainer/layers/dropout.cpp
+++ b/nntrainer/layers/dropout.cpp
@@ -39,16 +39,16 @@ void DropOutLayer::forwarding(RunLayerContext &context, bool training) {
   // buffer. So if the training is false, the output is the same with input. In
   // other words, there is nothing happen during inference.
 
-  if (training && rate_ > epsilon) {
-    for (unsigned int i = 0; i < context.getNumInputs(); ++i) {
-      Tensor &input_ = context.getInput(i);
-      Tensor &output_ = context.getOutput(i);
-      Tensor &mask_ = context.getTensor(mask_idx[i]);
+  for (unsigned int i = 0; i < context.getNumInputs(); ++i) {
+    Tensor &input_ = context.getInput(i);
+    Tensor &output_ = context.getOutput(i);
 
+    /** @todo make this in-place */
+    if (training && rate_ > epsilon) {
+      Tensor &mask_ = context.getTensor(mask_idx[i]);
       mask_ = input_.dropout_mask(rate_);
-      input_.multiply_i(mask_);
-
-      /** @todo: remove below once in_place support is ready from manager */
+      input_.multiply(mask_, output_);
+    } else {
       output_.fill(input_);
     }
   }
@@ -57,15 +57,16 @@ void DropOutLayer::forwarding(RunLayerContext &context, bool training) {
 void DropOutLayer::calcDerivative(RunLayerContext &context) {
   // Assume it is in-place calculation
   auto &rate_ = std::get<props::DropOutRate>(dropout_rate).get();
-  if (rate_ > epsilon) {
-    for (unsigned int i = 0; i < context.getNumInputs(); ++i) {
-      Tensor &derivative_ = context.getIncomingDerivative(i);
-      Tensor &ret_ = context.getOutgoingDerivative(SINGLE_INOUT_IDX);
-      Tensor &mask_ = context.getTensor(mask_idx[i]);
 
-      derivative_.multiply_i(mask_);
+  for (unsigned int i = 0; i < context.getNumInputs(); ++i) {
+    Tensor &derivative_ = context.getIncomingDerivative(i);
+    Tensor &ret_ = context.getOutgoingDerivative(SINGLE_INOUT_IDX);
 
-      /** @todo: remove below once in_place support is ready from manager */
+    /** @todo make this in-place */
+    if (rate_ > epsilon) {
+      Tensor &mask_ = context.getTensor(mask_idx[i]);
+      derivative_.multiply(mask_, ret_);
+    } else {
       ret_.fill(derivative_);
     }
   }

diff --git a/packaging/unittest_layers_v2.tar.gz b/packaging/unittest_layers_v2.tar.gz
index bf7df52..3abae0e 100644 (file)
Binary files a/packaging/unittest_layers_v2.tar.gz and b/packaging/unittest_layers_v2.tar.gz differ

diff --git a/test/input_gen/genLayerTests.py b/test/input_gen/genLayerTests.py
index dcb80d9..f1291b6 100644 (file)
--- a/test/input_gen/genLayerTests.py
+++ b/test/input_gen/genLayerTests.py
@@ -136,5 +136,15 @@ if __name__ == "__main__":
                          return_state=False)
     record_single(gru, (3, 4, 7), "gru_multi_step_seq_act", input_type='float')
 
-inspect_file("gru_single_step_seq.nnlayergolden")
+    dropout = K.layers.Dropout(rate=0.2)
+    record_single(dropout, (2, 3, 2, 3), "dropout_20_training", {"training": True})
+    record_single(dropout, (2, 3, 2, 3), "dropout_20_inference", {"training": False})
+
+    dropout = K.layers.Dropout(rate=0.0)
+    record_single(dropout, (2, 3, 2, 3), "dropout_0_training", {"training": True})
+
+    dropout = K.layers.Dropout(rate=0.9999)
+    record_single(dropout, (2, 3, 2, 3), "dropout_100_training", {"training": True})
+
+inspect_file("dropout_20_training.nnlayergolden")
 

diff --git a/test/unittest/layers/layers_common_tests.h b/test/unittest/layers/layers_common_tests.h
index b4bdbce..9397554 100644 (file)
--- a/test/unittest/layers/layers_common_tests.h
+++ b/test/unittest/layers/layers_common_tests.h
@@ -90,6 +90,8 @@ typedef enum {
   FORWARD_MODE_INFERENCE =
     1 << 2, /**< set if layer should be forwarded with inference mode */
 
+  DROPOUT_MATCH_60_PERCENT = 1 << 3, /**< set if only 60 percentage output
+                               match is sufficient for dropout */
   DEFAULT =
     0, /**< default set up, compare forward, backward in training mode */
 } LayerGoldenTestParamOptions;
@@ -134,6 +136,14 @@ public:
   bool shouldForwardWithInferenceMode();
 
   /**
+   * @brief check if given test suite must compare results using with a percent
+   * match for the tensors enabled
+   *
+   * @return bool layer should be match approximately
+   */
+  bool shouldMatchDropout60Percent();
+
+  /**
    * @brief check if given test suite should skip calculating derivative
    *
    * @return bool true if should skip calculating derivative

diff --git a/test/unittest/layers/layers_golden_tests.cpp b/test/unittest/layers/layers_golden_tests.cpp
index a69c2d0..829e62a 100644 (file)
--- a/test/unittest/layers/layers_golden_tests.cpp
+++ b/test/unittest/layers/layers_golden_tests.cpp
@@ -147,10 +147,38 @@ static RunLayerContext prepareRunContext(const TensorPacks &packs) {
 }
 
 static void compareRunContext(RunLayerContext &rc, std::ifstream &file,
-                              bool skip_grad, bool skip_deriv) {
+                              bool skip_grad, bool skip_deriv,
+                              bool dropout_match) {
   file.seekg(0, std::ios::beg);
-  auto compare_tensors = [&file](unsigned length, auto tensor_getter, auto pred,
-                                 bool skip_compare, const std::string &name) {
+  auto compare_percentage_tensors = [](const Tensor &t1, const Tensor &t2,
+                                       unsigned int match_percentage) -> bool {
+    if (match_percentage == 100)
+      return t1 == t2;
+
+    if (t1.getDim() != t2.getDim())
+      return false;
+
+    unsigned int total = t1.size();
+    unsigned int weak_match = 0;
+    unsigned int strong_match = 0;
+
+    for (unsigned int idx = 0; idx < total; idx++) {
+      auto d1 = t1.getValue(idx);
+      auto d2 = t2.getValue(idx);
+      /** either both the values must be equal or 1 must be zero */
+      weak_match +=
+        std::min((d1 == d2) + (d1 == 0 && d2 != 0) + (d1 != 0 && d2 == 0), 1);
+      strong_match += (d1 == d2);
+    }
+
+    return (weak_match == total) &
+           (strong_match >= (total * match_percentage) / 100);
+  };
+
+  auto compare_tensors = [&file, compare_percentage_tensors](
+                           unsigned length, auto tensor_getter, auto pred,
+                           bool skip_compare, const std::string &name,
+                           unsigned int match_percentage = 100) {
     for (unsigned i = 0; i < length; ++i) {
       if (!pred(i)) {
         continue;
@@ -162,7 +190,8 @@ static void compareRunContext(RunLayerContext &rc, std::ifstream &file,
       if (skip_compare) {
         continue;
       }
-      EXPECT_EQ(tensor, answer) << name << " at " << std::to_string(i);
+      EXPECT_TRUE(compare_percentage_tensors(tensor, answer, match_percentage))
+        << name << " at " << std::to_string(i);
     }
   };
 
@@ -171,6 +200,10 @@ static void compareRunContext(RunLayerContext &rc, std::ifstream &file,
     return rc.weightHasGradient(idx);
   };
 
+  int match_percentage = 100;
+  if (dropout_match)
+    match_percentage = 60;
+
   constexpr bool skip_compare = true;
 
   compare_tensors(rc.getNumWeights(),
@@ -181,7 +214,7 @@ static void compareRunContext(RunLayerContext &rc, std::ifstream &file,
                   !skip_compare, "inputs");
   compare_tensors(rc.getNumOutputs(),
                   [&rc](unsigned idx) { return rc.getOutput(idx); },
-                  always_read, !skip_compare, "outputs");
+                  always_read, !skip_compare, "outputs", match_percentage);
   compare_tensors(rc.getNumWeights(),
                   [&rc](unsigned idx) { return rc.getWeightGrad(idx); },
                   only_read_trainable, skip_grad, "gradients");
@@ -190,7 +223,7 @@ static void compareRunContext(RunLayerContext &rc, std::ifstream &file,
                   always_read, !skip_compare, "weights");
   compare_tensors(rc.getNumInputs(),
                   [&rc](unsigned idx) { return rc.getOutgoingDerivative(idx); },
-                  always_read, skip_deriv, "derivatives");
+                  always_read, skip_deriv, "derivatives", match_percentage);
 }
 
 LayerGoldenTest::~LayerGoldenTest() {}
@@ -199,6 +232,11 @@ void LayerGoldenTest::SetUp() {}
 
 void LayerGoldenTest::TearDown() {}
 
+bool LayerGoldenTest::shouldMatchDropout60Percent() {
+  return std::get<int>(GetParam()) &
+         LayerGoldenTestParamOptions::DROPOUT_MATCH_60_PERCENT;
+}
+
 bool LayerGoldenTest::shouldForwardWithInferenceMode() {
   return std::get<int>(GetParam()) &
          LayerGoldenTestParamOptions::FORWARD_MODE_INFERENCE;
@@ -227,6 +265,7 @@ TEST_P(LayerGoldenTest, run) {
 
   bool skip_calc_grad = shouldSkipCalcGrad();
   bool skip_calc_deriv = shouldSkipCalcDeriv();
+  bool dropout_compare_60_percent = shouldMatchDropout60Percent();
 
   for (int i = 0; i < 4; ++i) {
     /// warm layer multiple times
@@ -241,7 +280,8 @@ TEST_P(LayerGoldenTest, run) {
     layer->calcDerivative(rc);
   }
 
-  compareRunContext(rc, golden_file, skip_calc_grad, skip_calc_deriv);
+  compareRunContext(rc, golden_file, skip_calc_grad, skip_calc_deriv,
+                    dropout_compare_60_percent);
 
   EXPECT_TRUE(true); // stub test for tcm
 }

diff --git a/test/unittest/layers/meson.build b/test/unittest/layers/meson.build
index 80ce748..820b703 100644 (file)
--- a/test/unittest/layers/meson.build
+++ b/test/unittest/layers/meson.build
@@ -50,6 +50,7 @@ test_target = [
   'unittest_layers_concat.cpp',
   'unittest_layers_permute.cpp',
   'unittest_layers_attention.cpp',
+  'unittest_layers_dropout.cpp',
 ]
 
 if get_option('enable-tflite-backbone')

diff --git a/test/unittest/layers/unittest_layers_dropout.cpp b/test/unittest/layers/unittest_layers_dropout.cpp
new file mode 100644 (file)
index 0000000..3a38801
--- /dev/null
+++ b/test/unittest/layers/unittest_layers_dropout.cpp
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2021 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file unittest_layers_dropout.cpp
+ * @date 15 October 2021
+ * @brief Dropout Layer Test
+ * @see        https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ */
+#include <tuple>
+
+#include <gtest/gtest.h>
+
+#include <dropout.h>
+#include <layers_common_tests.h>
+
+auto semantic_dropout =
+  LayerSemanticsParamType(nntrainer::createLayer<nntrainer::DropOutLayer>,
+                          nntrainer::DropOutLayer::type, {}, 0, false, 1);
+
+INSTANTIATE_TEST_CASE_P(Dropout, LayerSemantics,
+                        ::testing::Values(semantic_dropout));
+
+auto dropout_inference_option =
+  LayerGoldenTestParamOptions::SKIP_CALC_GRAD |
+  LayerGoldenTestParamOptions::SKIP_CALC_DERIV |
+  LayerGoldenTestParamOptions::FORWARD_MODE_INFERENCE;
+
+auto dropout_20_training = LayerGoldenTestParamType(
+  nntrainer::createLayer<nntrainer::DropOutLayer>, {"dropout_rate=0.2"},
+  "2:3:2:3", "dropout_20_training.nnlayergolden",
+  LayerGoldenTestParamOptions::DEFAULT |
+    LayerGoldenTestParamOptions::DROPOUT_MATCH_60_PERCENT);
+
+auto dropout_20_inference = LayerGoldenTestParamType(
+  nntrainer::createLayer<nntrainer::DropOutLayer>, {"dropout_rate=0.2"},
+  "2:3:2:3", "dropout_20_inference.nnlayergolden", dropout_inference_option);
+
+auto dropout_0_training = LayerGoldenTestParamType(
+  nntrainer::createLayer<nntrainer::DropOutLayer>, {"dropout_rate=0.0"},
+  "2:3:2:3", "dropout_0_training.nnlayergolden",
+  LayerGoldenTestParamOptions::DEFAULT);
+
+auto dropout_100_training = LayerGoldenTestParamType(
+  nntrainer::createLayer<nntrainer::DropOutLayer>, {"dropout_rate=1.0"},
+  "2:3:2:3", "dropout_100_training.nnlayergolden",
+  LayerGoldenTestParamOptions::DEFAULT);
+
+INSTANTIATE_TEST_CASE_P(Dropout, LayerGoldenTest,
+                        ::testing::Values(dropout_20_training,
+                                          dropout_0_training,
+                                          dropout_100_training,
+                                          dropout_20_inference));