[init] Making random deterministic

Adding determinism to the random number generators in the library
DataBuffer has multiple threads but single thread of train/valid/test
which run in sequence in my understanding

Resolves #167

Signed-off-by: Parichay Kapoor <pk.kapoor@samsung.com>

diff --git a/nntrainer/include/databuffer.h b/nntrainer/include/databuffer.h
index 2398bc0b8e2eafceb54d65ffd18cc387338755b7..2a3806e8db2c3a0fe70f4bf9e6a54c7c4d3293b5 100644 (file)
--- a/nntrainer/include/databuffer.h
+++ b/nntrainer/include/databuffer.h
@@ -30,8 +30,10 @@
 #include <functional>
 #include <iostream>
 #include <memory>
+#include <random>
 #include <tensor_dim.h>
 #include <thread>
+#include <util_func.h>
 #include <vector>
 
 /*
@@ -135,6 +137,7 @@ public:
     train_running = false;
     val_running = false;
     test_running = false;
+    rng.seed(getSeed());
   };
 
   /**
@@ -346,6 +349,8 @@ protected:
    * @retval    int return value
    */
   int rangeRandom(int min, int max);
+
+  std::mt19937 rng;
 };
 
 } // namespace nntrainer

diff --git a/nntrainer/include/tensor.h b/nntrainer/include/tensor.h
index efb2d106ea9809fcfed9857fe62bb5b1c3e9ee83..fee33e1ba5b74d31cc17cc8316194f724dd45cb4 100644 (file)
--- a/nntrainer/include/tensor.h
+++ b/nntrainer/include/tensor.h
@@ -351,7 +351,7 @@ public:
   int getBatch() const { return dim.batch(); };
 
   /**
-   * @brief     Set the elelemnt value
+   * @brief     Set the element value
    * @param[in] batch batch location
    * @param[in] c channel location
    * @param[in] i height location
@@ -361,6 +361,26 @@ public:
   void setValue(unsigned int batch, unsigned int c, unsigned int i,
                 unsigned int j, float value);
 
+  /**
+   * @brief     Fill the Tensor elements with value
+   * @param[in] value value to be stored
+   */
+  void setValue(float value);
+
+  /**
+   * @brief     Set the tensor with random normal distribution
+   * @param[in] mean mean of the distribution
+   * @param[in] std standard deviation of the distribution
+   */
+  void setRandNormal(float mean = 0.0, float std = 0.05);
+
+  /**
+   * @brief     Set the tensor with random uniform distribution
+   * @param[in] min minimum value for the distribution
+   * @param[in] max maximum value for the distribution
+   */
+  void setRandUniform(float min = -0.05, float max = 0.05);
+
   /**
    * @brief     Copy the Tensor
    * @param[in] from Tensor to be Copyed
@@ -398,7 +418,6 @@ public:
    */
   float *getData() { return data.data(); }
 
-
   const float *getData() const { return data.data(); }
 
   /**
@@ -407,7 +426,6 @@ public:
    */
   float *getAddress(unsigned int i);
 
-
 private:
   /**< handle the data as a std::vector type */
   std::vector<float> data;
@@ -415,6 +433,8 @@ private:
 
   static constexpr float min_limits = std::numeric_limits<float>::min();
   static constexpr float max_limits = std::numeric_limits<float>::max();
+  template<typename T>
+    void setDist(T dist);
 };
 
 /**

diff --git a/nntrainer/include/util_func.h b/nntrainer/include/util_func.h
index cf51f13529df67f6e20064ce452e6f92a313118a..ef9db26ad4a7f836cb3c79d643e3ce30860a3c9e 100644 (file)
--- a/nntrainer/include/util_func.h
+++ b/nntrainer/include/util_func.h
@@ -28,6 +28,12 @@
 
 namespace nntrainer {
 
+/**
+ * @brief     get the seed
+ * @retVal    seed
+ */
+unsigned int getSeed();
+
 /**
  * @brief     derivative softmax function for Tensor Type
  * @param[in] x Tensor
@@ -44,9 +50,8 @@ Tensor softmax(Tensor t);
 
 /**
  * @brief     random function
- * @param[in] x float
  */
-float random(float x);
+float random();
 
 /**
  * @brief     sqrt function for float type

diff --git a/nntrainer/src/conv2d_layer.cpp b/nntrainer/src/conv2d_layer.cpp
index 06182180bc819cfa728f2385de8c8cd7b7de3b7f..0ae169958ae35f6e1d661749536ba1fa2d0c47fd 100644 (file)
--- a/nntrainer/src/conv2d_layer.cpp
+++ b/nntrainer/src/conv2d_layer.cpp
@@ -17,7 +17,6 @@
 #include <nntrainer_error.h>
 #include <nntrainer_log.h>
 #include <parse_util.h>
-#include <random>
 #include <util_func.h>
 
 namespace nntrainer {
@@ -50,7 +49,7 @@ int Conv2DLayer::initialize(bool last) {
     if (init_zero) {
       B = 0.0;
     } else {
-      B = random(B);
+      B = random();
     }
     bias.push_back(B);
   }

diff --git a/nntrainer/src/databuffer.cpp b/nntrainer/src/databuffer.cpp
index e7b5253c8d3e9537d5b08dc03f4e20a38c3e8b09..d6ebeda999eef6a6ebbd47d198a3cd5810176841 100644 (file)
--- a/nntrainer/src/databuffer.cpp
+++ b/nntrainer/src/databuffer.cpp
@@ -52,13 +52,8 @@ std::condition_variable cv_val;
 std::condition_variable cv_test;
 
 int DataBuffer::rangeRandom(int min, int max) {
-  int n = max - min + 1;
-  int remainder = RAND_MAX % n;
-  int x;
-  do {
-    x = rand();
-  } while (x >= RAND_MAX - remainder);
-  return min + x % n;
+  std::uniform_int_distribution<int> dist(min, max);
+  return dist(rng);
 }
 
 int DataBuffer::run(BufferType type) {

diff --git a/nntrainer/src/fc_layer.cpp b/nntrainer/src/fc_layer.cpp
index 60d4c65dfad5f0cfb94163f94f2037d7538983d5..9faa29b4cea59a3ce2a74848e749bd0ca7867647 100644 (file)
--- a/nntrainer/src/fc_layer.cpp
+++ b/nntrainer/src/fc_layer.cpp
@@ -54,7 +54,7 @@ int FullyConnectedLayer::initialize(bool last) {
   if (init_zero) {
     bias.setZero();
   } else {
-    bias = bias.apply(random);
+    bias.setRandUniform(-0.5, 0.5);
   }
   return status;
 }

diff --git a/nntrainer/src/layer.cpp b/nntrainer/src/layer.cpp
index 940658c20b491cc5b4e83e04d5acdcb0b04cd46d..2f560eea6f4a97b8950f76bf391151b36e4c4f6c 100644 (file)
--- a/nntrainer/src/layer.cpp
+++ b/nntrainer/src/layer.cpp
@@ -25,50 +25,10 @@
 #include <nntrainer_error.h>
 #include <nntrainer_log.h>
 #include <parse_util.h>
-#include <random>
 #include <util_func.h>
 
 namespace nntrainer {
 
-static auto rng = [] {
-  std::mt19937 rng;
-  rng.seed(std::random_device()());
-  return rng;
-}();
-
-template <typename... Args>
-static void RandNormal(unsigned int b_n, Tensor &w, Args &&... args) {
-  std::normal_distribution<float> dist(std::forward<Args>(args)...);
-  unsigned int channel = w.getChannel();
-  unsigned int width = w.getWidth();
-  unsigned int height = w.getHeight();
-
-  for (unsigned int k = 0; k < channel; ++k) {
-    for (unsigned int i = 0; i < width; ++i) {
-      for (unsigned int j = 0; j < height; ++j) {
-        w.setValue(b_n, k, j, i, dist(rng));
-      }
-    }
-  }
-}
-
-template <typename... Args>
-static void RandUniform(unsigned int b_n, Tensor &w, Args &&... args) {
-  std::uniform_real_distribution<float> dist(std::forward<Args>(args)...);
-
-  unsigned int channel = w.getChannel();
-  unsigned int width = w.getWidth();
-  unsigned int height = w.getHeight();
-
-  for (unsigned int k = 0; k < channel; ++k) {
-    for (unsigned int i = 0; i < width; ++i) {
-      for (unsigned int j = 0; j < height; ++j) {
-        w.setValue(b_n, k, j, i, dist(rng));
-      }
-    }
-  }
-}
-
 int Layer::setActivation(ActiType acti) {
   int status = ML_ERROR_NONE;
   if (acti == ACT_UNKNOWN) {
@@ -132,32 +92,30 @@ Tensor Layer::initializeWeight(TensorDim w_dim, WeightIniType init_type,
     init_type = WEIGHT_XAVIER_NORMAL;
   }
 
-  for (unsigned int i = 0; i < w_dim.batch(); ++i) {
-    switch (init_type) {
-    case WEIGHT_LECUN_NORMAL:
-      RandNormal(i, w, 0, sqrt(1.0 / dim.height()));
-      break;
-    case WEIGHT_XAVIER_NORMAL:
-      RandNormal(i, w, 0, sqrt(2.0 / (dim.width() + dim.height())));
-      break;
-    case WEIGHT_HE_NORMAL:
-      RandNormal(i, w, 0, sqrt(2.0 / (dim.height())));
-      break;
-    case WEIGHT_LECUN_UNIFORM:
-      RandUniform(i, w, -1.0 * sqrt(1.0 / dim.height()),
-                  sqrt(1.0 / dim.height()));
-      break;
-    case WEIGHT_XAVIER_UNIFORM:
-      RandUniform(i, w, -1.0 * sqrt(6.0 / (dim.height() + dim.width())),
-                  sqrt(6.0 / (dim.height() + dim.width())));
-      break;
-    case WEIGHT_HE_UNIFORM:
-      RandUniform(i, w, -1.0 * sqrt(6.0 / (dim.height())),
-                  sqrt(6.0 / (dim.height())));
-      break;
-    default:
-      break;
-    }
+  switch (init_type) {
+  case WEIGHT_LECUN_NORMAL:
+    w.setRandNormal(0, sqrt(1.0 / dim.height()));
+    break;
+  case WEIGHT_XAVIER_NORMAL:
+    w.setRandNormal(0, sqrt(2.0 / (dim.width() + dim.height())));
+    break;
+  case WEIGHT_HE_NORMAL:
+    w.setRandNormal(0, sqrt(2.0 / (dim.height())));
+    break;
+  case WEIGHT_LECUN_UNIFORM:
+    w.setRandUniform(-1.0 * sqrt(1.0 / dim.height()),
+        sqrt(1.0 / dim.height()));
+    break;
+  case WEIGHT_XAVIER_UNIFORM:
+    w.setRandUniform(-1.0 * sqrt(6.0 / (dim.height() + dim.width())),
+        sqrt(6.0 / (dim.height() + dim.width())));
+    break;
+  case WEIGHT_HE_UNIFORM:
+    w.setRandUniform(-1.0 * sqrt(6.0 / (dim.height())),
+        sqrt(6.0 / (dim.height())));
+    break;
+  default:
+    break;
   }
   return w;
 }

diff --git a/nntrainer/src/tensor.cpp b/nntrainer/src/tensor.cpp
index 238c0180e953c011c64c72e1eb9dd6a2df0f810d..9e2c36f954fbf935c97ef43b89e6cb65b57a0563 100644 (file)
--- a/nntrainer/src/tensor.cpp
+++ b/nntrainer/src/tensor.cpp
@@ -26,9 +26,11 @@
 #include <nntrainer_error.h>
 #include <nntrainer_log.h>
 #include <parse_util.h>
+#include <random>
 #include <sstream>
 #include <stdio.h>
 #include <tensor.h>
+#include <util_func.h>
 
 #include <lazy_tensor.h>
 
@@ -53,6 +55,12 @@
 
 namespace nntrainer {
 
+static auto rng = [] {
+  std::mt19937 rng;
+  rng.seed(getSeed());
+  return rng;
+}();
+
 Tensor::Tensor(const TensorDim d) {
   dim = d;
   this->data = std::vector<float>(dim.getDataLen());
@@ -95,6 +103,23 @@ void Tensor::setValue(unsigned int batch, unsigned int c, unsigned int h,
              c * dim.height() * dim.width() + h * dim.width() + w] = value;
 }
 
+template<typename T>
+void Tensor::setDist(T dist) {
+  for (unsigned int i = 0; i < dim.getDataLen(); ++i) {
+    data[i] = dist(rng);
+  }
+}
+
+void Tensor::setRandNormal(float mean, float std) {
+  setDist<std::normal_distribution<float>>(
+      std::normal_distribution<float> (mean, std));
+}
+
+void Tensor::setRandUniform(float min, float max) {
+  setDist<std::uniform_real_distribution<float>>(
+      std::uniform_real_distribution<float> (min, max));
+}
+
 Tensor::Tensor(std::vector<std::vector<float>> const &d) {
 
   dim.height(d.size());
@@ -814,8 +839,12 @@ Tensor Tensor::average(int axis) const {
   return result;
 }
 
+void Tensor::setValue(float val) {
+  memset(this->data.data(), val, sizeof(float) * dim.getDataLen());
+}
+
 void Tensor::setZero() {
-  memset(this->data.data(), 0, sizeof(float) * dim.getDataLen());
+  setValue(0);
 }
 
 int Tensor::argmax() {

diff --git a/nntrainer/src/util_func.cpp b/nntrainer/src/util_func.cpp
index 01bbea12c6b17f498706ea7cdcf91ec3827dd3ac..35490a3e2754eee572e5a47d094635e1fd129cc5 100644 (file)
--- a/nntrainer/src/util_func.cpp
+++ b/nntrainer/src/util_func.cpp
@@ -22,11 +22,21 @@
 
 #include <assert.h>
 #include <math.h>
+#include <random>
 #include <tensor.h>
 #include <util_func.h>
 
 namespace nntrainer {
 
+static auto rng = [] {
+  std::mt19937 rng;
+  rng.seed(getSeed());
+  return rng;
+}();
+static std::uniform_real_distribution<float> dist(-0.5, 0.5);
+
+unsigned int getSeed() { return 0; }
+
 Tensor softmaxPrime(Tensor x) {
   int batch = x.getBatch();
   int channel = x.getChannel();
@@ -113,7 +123,7 @@ Tensor softmax(Tensor t) {
   return result;
 }
 
-float random(float x) { return (float)(rand() % 10000 + 1) / 10000 - 0.5; }
+float random() { return dist(rng); }
 
 float sqrtFloat(float x) { return (float)(sqrt(x)); };
 

diff --git a/test/nntrainer_test_util.cpp b/test/nntrainer_test_util.cpp
index d8195cf7582f2718fefd3033c613bbb287f35f0e..1ee66f8ea30ce184a58a03efde0e5fbffcd2b486 100644 (file)
--- a/test/nntrainer_test_util.cpp
+++ b/test/nntrainer_test_util.cpp
@@ -24,6 +24,7 @@
 #include "nntrainer_test_util.h"
 #include <climits>
 #include <iostream>
+#include <random>
 
 #define num_class 10
 #define mini_batch 16
@@ -33,6 +34,7 @@ static bool *duplicate;
 static bool *valduplicate;
 static bool alloc_train = false;
 static bool alloc_val = false;
+static std::mt19937 rng(0);
 
 /**
  * @brief replace string and save it in file
@@ -66,13 +68,8 @@ void replaceString(const std::string &from, const std::string &to,
  * @retval    min < random value < max
  */
 static int rangeRandom(int min, int max) {
-  int n = max - min + 1;
-  int remainder = RAND_MAX % n;
-  int x;
-  do {
-    x = rand();
-  } while (x >= RAND_MAX - remainder);
-  return min + x % n;
+  std::uniform_int_distribution<int> dist(min, max);
+  return dist(rng);
 }
 
 /**

diff --git a/test/unittest/unittest_nntrainer_tensor.cpp b/test/unittest/unittest_nntrainer_tensor.cpp
index 79c5ef026830a0106aa19029aff2f145c65131df..318c5cc5d688ecacb31cfe039358b5f4d3ac6feb 100644 (file)
--- a/test/unittest/unittest_nntrainer_tensor.cpp
+++ b/test/unittest/unittest_nntrainer_tensor.cpp
@@ -799,6 +799,17 @@ end_transpose_01_p:
   EXPECT_EQ(status, ML_ERROR_NONE);
 }
 
+TEST(nntrainer_Tensor, set_01_p) {
+  nntrainer::Tensor tensor = nntrainer::Tensor(1, 1, 1, 1);
+
+  tensor.setZero();
+  EXPECT_EQ(tensor.getValue(0, 0, 0, 0), 0.0);
+
+  tensor.setRandUniform(-0.5, 0);
+  float val = tensor.getValue(0, 0, 0, 0);
+  EXPECT_TRUE(val >= -0.5 && val < 0);
+}
+
 /**
  * @brief Main gtest
  */

diff --git a/test/unittest/unittest_util_func.cpp b/test/unittest/unittest_util_func.cpp
index 355ee96a0c0ab7562482ea2714554432ec0d6327..10ce8f97f676aa0d021f11250b728f884471ae01 100644 (file)
--- a/test/unittest/unittest_util_func.cpp
+++ b/test/unittest/unittest_util_func.cpp
@@ -95,15 +95,6 @@ TEST(nntrainer_util_func, softmax_prime_01_p) {
   }
 }
 
-TEST(nntrainer_util_func, random_01_p) {
-  int status = ML_ERROR_INVALID_PARAMETER;
-  srand(time(NULL));
-  float x = nntrainer::random(0.0);
-  if (-1.0 < x && x < 1.0)
-    status = ML_ERROR_NONE;
-  EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
 TEST(nntrainer_util_func, sqrtFloat_01_p) {
   int status = ML_ERROR_INVALID_PARAMETER;