call caffe_set_rng at the end of each vRng function to maintain state

diff --git a/include/caffe/common.hpp b/include/caffe/common.hpp
index c1bc395..908deee 100644 (file)
--- a/include/caffe/common.hpp
+++ b/include/caffe/common.hpp
@@ -85,19 +85,18 @@ class Caffe {
    public:
     RNG();
     explicit RNG(unsigned int seed);
-    ~RNG();
-    RNG(const RNG&);
+    explicit RNG(const RNG&);
     RNG& operator=(const RNG&);
     const void* generator() const;
-    void* generator();
+    void set_generator(const void* other_rng);
    private:
     class Generator;
-    Generator* generator_;
+    shared_ptr<Generator> generator_;
   };
 
   // Getters for boost rng, curand, and cublas handles
-  inline static RNG &rng_stream() {
-    return Get().random_generator_;
+  inline static const RNG& rng_stream() {
+    return *(Get().random_generator_);
   }
   inline static cublasHandle_t cublas_handle() { return Get().cublas_handle_; }
   inline static curandGenerator_t curand_generator() {
@@ -118,6 +117,8 @@ class Caffe {
   inline static void set_phase(Phase phase) { Get().phase_ = phase; }
   // Sets the random seed of both boost and curand
   static void set_random_seed(const unsigned int seed);
+  // Sets the boost RNG engine from another RNG engine
+  static void set_generator(const void* other_rng);
   // Sets the device. Since we have cublas and curand stuff, set device also
   // requires us to reset those values.
   static void SetDevice(const int device_id);
@@ -127,7 +128,7 @@ class Caffe {
  protected:
   cublasHandle_t cublas_handle_;
   curandGenerator_t curand_generator_;
-  RNG random_generator_;
+  shared_ptr<RNG> random_generator_;
 
   Brew mode_;
   Phase phase_;

diff --git a/include/caffe/util/math_functions.hpp b/include/caffe/util/math_functions.hpp
index c501f23..77e3234 100644 (file)
--- a/include/caffe/util/math_functions.hpp
+++ b/include/caffe/util/math_functions.hpp
@@ -116,7 +116,7 @@ void caffe_vRngGaussian(const int n, Dtype* r, const Dtype a,
     const Dtype sigma);
 
 template <typename Dtype>
-void caffe_vRngBernoulli(const int n, Dtype* r, const double p);
+void caffe_vRngBernoulli(const int n, int* r, const Dtype p);
 
 template <typename Dtype>
 void caffe_exp(const int n, const Dtype* a, Dtype* y);

diff --git a/include/caffe/util/rng.hpp b/include/caffe/util/rng.hpp
index 8151a9a..e684b35 100644 (file)
--- a/include/caffe/util/rng.hpp
+++ b/include/caffe/util/rng.hpp
@@ -9,9 +9,13 @@
 namespace caffe {
 
   typedef boost::mt19937 rng_t;
-  inline rng_t& caffe_rng() {
-    Caffe::RNG &generator = Caffe::rng_stream();
-    return *(caffe::rng_t*) generator.generator();
+
+  inline const rng_t& caffe_rng() {
+    return *static_cast<const caffe::rng_t*>(Caffe::rng_stream().generator());
+  }
+
+  inline void caffe_set_rng(const caffe::rng_t& other) {
+    Caffe::set_generator(static_cast<const void*>(&other));
   }
 
 }  // namespace caffe

diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp
index feda44d..0bfc677 100644 (file)
--- a/include/caffe/vision_layers.hpp
+++ b/include/caffe/vision_layers.hpp
@@ -70,8 +70,8 @@ class DropoutLayer : public NeuronLayer<Dtype> {
       const bool propagate_down, vector<Blob<Dtype>*>* bottom);
 
   shared_ptr<SyncedMemory> rand_vec_;
-  float threshold_;
-  float scale_;
+  Dtype threshold_;
+  Dtype scale_;
   unsigned int uint_thres_;
 };
 

diff --git a/src/caffe/common.cpp b/src/caffe/common.cpp
index 17e4ca5..65c6800 100644 (file)
--- a/src/caffe/common.cpp
+++ b/src/caffe/common.cpp
@@ -60,7 +60,11 @@ void Caffe::set_random_seed(const unsigned int seed) {
     LOG(ERROR) << "Curand not available. Skipping setting the curand seed.";
   }
   // RNG seed
-  Get().random_generator_ = RNG(seed);
+  Get().random_generator_.reset(new RNG(seed));
+}
+
+void Caffe::set_generator(const void* other_rng) {
+  Get().random_generator_->set_generator(other_rng);
 }
 
 void Caffe::SetDevice(const int device_id) {
@@ -117,36 +121,32 @@ void Caffe::DeviceQuery() {
 
 class Caffe::RNG::Generator {
  public:
-  caffe::rng_t rng;
+  Generator() : rng_(new caffe::rng_t(cluster_seedgen())) {}
+  explicit Generator(unsigned int seed) : rng_(new caffe::rng_t(seed)) {}
+  explicit Generator(const caffe::rng_t& other) :
+      rng_(new caffe::rng_t(other)) {}
+  shared_ptr<caffe::rng_t> rng_;
 };
 
-Caffe::RNG::RNG()
-: generator_(new Generator) {
-  generator_->rng = caffe::rng_t(cluster_seedgen());
-}
-
-Caffe::RNG::RNG(unsigned int seed)
-: generator_(new Generator) {
-  generator_->rng = caffe::rng_t(seed);
-}
+Caffe::RNG::RNG() : generator_(new Generator) { }
 
-Caffe::RNG::~RNG() { delete generator_; }
+Caffe::RNG::RNG(unsigned int seed) : generator_(new Generator(seed)) { }
 
-Caffe::RNG::RNG(const RNG& other) : generator_(new Generator) {
-  *generator_ = *other.generator_;
-}
+Caffe::RNG::RNG(const RNG& other) : generator_(new Generator(*other.generator_))
+    { }
 
 Caffe::RNG& Caffe::RNG::operator=(const RNG& other) {
-  *generator_ = *other.generator_;
+  generator_.reset(other.generator_.get());
   return *this;
 }
 
-void* Caffe::RNG::generator() {
-  return &generator_->rng;
+const void* Caffe::RNG::generator() const {
+  return static_cast<const void*>(generator_->rng_.get());
 }
 
-const void* Caffe::RNG::generator() const {
-  return &generator_->rng;
+void Caffe::RNG::set_generator(const void* other_rng) {
+  const caffe::rng_t& rng = *static_cast<const caffe::rng_t*>(other_rng);
+  return generator_.reset(new Generator(rng));
 }
 
 const char* cublasGetErrorString(cublasStatus_t error) {

diff --git a/src/caffe/layers/dropout_layer.cpp b/src/caffe/layers/dropout_layer.cpp
index 5dbba5d..a57999c 100644 (file)
--- a/src/caffe/layers/dropout_layer.cpp
+++ b/src/caffe/layers/dropout_layer.cpp
@@ -32,7 +32,7 @@ Dtype DropoutLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
   const int count = bottom[0]->count();
   if (Caffe::phase() == Caffe::TRAIN) {
     // Create random numbers
-    caffe_vRngBernoulli<int>(count, mask, 1. - threshold_);
+    caffe_vRngBernoulli(count, mask, 1. - threshold_);
     for (int i = 0; i < count; ++i) {
       top_data[i] = bottom_data[i] * mask[i] * scale_;
     }

diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp
index 69a19aa..25335c0 100644 (file)
--- a/src/caffe/test/test_random_number_generator.cpp
+++ b/src/caffe/test/test_random_number_generator.cpp
@@ -50,12 +50,12 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussian) {
   TypeParam mu = 0;
   TypeParam sigma = 1;
   caffe_vRngGaussian(sample_size,
-      reinterpret_cast<TypeParam*>(data_a.mutable_cpu_data()), mu, sigma);
+      static_cast<TypeParam*>(data_a.mutable_cpu_data()), mu, sigma);
   TypeParam true_mean = mu;
   TypeParam true_std = sigma;
   TypeParam bound = this->mean_bound(true_std, sample_size);
   TypeParam empirical_mean =
-      this->sample_mean(reinterpret_cast<const TypeParam*>(data_a.cpu_data()),
+      this->sample_mean(static_cast<const TypeParam*>(data_a.cpu_data()),
           sample_size);
   EXPECT_NEAR(empirical_mean, true_mean, bound);
 }
@@ -68,12 +68,12 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniform) {
   TypeParam lower = 0;
   TypeParam upper = 1;
   caffe_vRngUniform(sample_size,
-      reinterpret_cast<TypeParam*>(data_a.mutable_cpu_data()), lower, upper);
+      static_cast<TypeParam*>(data_a.mutable_cpu_data()), lower, upper);
   TypeParam true_mean = (lower + upper) / 2;
   TypeParam true_std = (upper - lower) / sqrt(12);
   TypeParam bound = this->mean_bound(true_std, sample_size);
   TypeParam empirical_mean =
-      this->sample_mean(reinterpret_cast<const TypeParam*>(data_a.cpu_data()),
+      this->sample_mean(static_cast<const TypeParam*>(data_a.cpu_data()),
           sample_size);
   EXPECT_NEAR(empirical_mean, true_mean, bound);
 }
@@ -103,13 +103,13 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) {
   // Sample from 0 mean Gaussian
   TypeParam mu = 0;
   TypeParam sigma = 1;
-  caffe_vRngGaussian(sample_size, reinterpret_cast<TypeParam*>(
+  caffe_vRngGaussian(sample_size, static_cast<TypeParam*>(
       gaussian_data.mutable_cpu_data()), mu, sigma);
   TypeParam true_mean = mu;
   TypeParam true_std = sigma;
   TypeParam bound = this->mean_bound(true_std, sample_size);
   TypeParam empirical_mean = this->sample_mean(
-      reinterpret_cast<const TypeParam*>(gaussian_data.cpu_data()),
+      static_cast<const TypeParam*>(gaussian_data.cpu_data()),
       sample_size);
   EXPECT_NEAR(empirical_mean, true_mean, bound);
   int num_pos = 0;
@@ -144,7 +144,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) {
   // Sample from Bernoulli with p = 0.3
   p = 0.3;
   caffe_vRngBernoulli(sample_size,
-      reinterpret_cast<int*>(bernoulli_data.mutable_cpu_data()), p);
+      static_cast<int*>(bernoulli_data.mutable_cpu_data()), p);
   true_mean = p;
   true_std = sqrt(p * (1 - p));
   bound = this->mean_bound(true_std, sample_size);
@@ -157,7 +157,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) {
   int num_ones = 0;
   int num_other = 0;
   const int* bernoulli_samples =
-      reinterpret_cast<const int*>(bernoulli_data.cpu_data());
+      static_cast<const int*>(bernoulli_data.cpu_data());
   for (int i = 0; i < sample_size; ++i) {
     if (bernoulli_samples[i] == 0) {
       ++bernoulli_num_zeros;
@@ -170,8 +170,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) {
   LOG(INFO) << "Bernoulli: zeros: "  << bernoulli_num_zeros
             << "; ones: " << num_ones << "; other: " << num_other;
   EXPECT_EQ(0, num_other);
-  EXPECT_EQ(sample_size * empirical_mean, num_ones);
-  EXPECT_EQ(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros);
+  TypeParam epsilon = 1e-4;
+  EXPECT_NEAR(sample_size * empirical_mean, num_ones, epsilon);
+  EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros,
+              epsilon);
   // Multiply Gaussian by Bernoulli
   for (int i = 0; i < sample_size; ++i) {
     samples[i] *= bernoulli_samples[i];
@@ -214,13 +216,13 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) {
   // Sample from Uniform on [-1, 1]
   TypeParam a = -1;
   TypeParam b = 1;
-  caffe_vRngUniform(sample_size, reinterpret_cast<TypeParam*>(
+  caffe_vRngUniform(sample_size, static_cast<TypeParam*>(
       uniform_data.mutable_cpu_data()), a, b);
   TypeParam true_mean = (a + b) / 2;
   TypeParam true_std = (b - a) / sqrt(12);
   TypeParam bound = this->mean_bound(true_std, sample_size);
   TypeParam empirical_mean = this->sample_mean(
-      reinterpret_cast<const TypeParam*>(uniform_data.cpu_data()),
+      static_cast<const TypeParam*>(uniform_data.cpu_data()),
       sample_size);
   EXPECT_NEAR(empirical_mean, true_mean, bound);
   int num_pos = 0;
@@ -241,7 +243,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) {
   // improbable), and roughly half positives and half negatives (with bound
   // computed from a Bernoulli with p = 0.5).
   EXPECT_EQ(0, num_zeros);
-  double p = 0.5;
+  TypeParam p = 0.5;
   true_mean = p;
   true_std = sqrt(p * (1 - p));
   bound = this->mean_bound(true_std, sample_size);
@@ -255,7 +257,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) {
   // Sample from Bernoulli with p = 0.3
   p = 0.3;
   caffe_vRngBernoulli(sample_size,
-      reinterpret_cast<int*>(bernoulli_data.mutable_cpu_data()), p);
+      static_cast<int*>(bernoulli_data.mutable_cpu_data()), p);
   true_mean = p;
   true_std = sqrt(p * (1 - p));
   bound = this->mean_bound(true_std, sample_size);
@@ -268,7 +270,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) {
   int num_ones = 0;
   int num_other = 0;
   const int* bernoulli_samples =
-      reinterpret_cast<const int*>(bernoulli_data.cpu_data());
+      static_cast<const int*>(bernoulli_data.cpu_data());
   for (int i = 0; i < sample_size; ++i) {
     if (bernoulli_samples[i] == 0) {
       ++bernoulli_num_zeros;
@@ -281,8 +283,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) {
   LOG(INFO) << "Bernoulli: zeros: "  << bernoulli_num_zeros
             << "; ones: " << num_ones << "; other: " << num_other;
   EXPECT_EQ(0, num_other);
-  EXPECT_EQ(sample_size * empirical_mean, num_ones);
-  EXPECT_EQ(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros);
+  TypeParam epsilon = 1e-4;
+  EXPECT_NEAR(sample_size * empirical_mean, num_ones, epsilon);
+  EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros,
+              epsilon);
   // Multiply Uniform by Bernoulli
   for (int i = 0; i < sample_size; ++i) {
     samples[i] *= bernoulli_samples[i];
@@ -322,17 +326,17 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) {
   SyncedMemory bernoulli1_data(sample_size * sizeof(int));
   SyncedMemory bernoulli2_data(sample_size * sizeof(int));
   Caffe::set_random_seed(1701);
-  double p1 = 0.5;
-  caffe_vRngBernoulli(sample_size, reinterpret_cast<int*>(
+  TypeParam p1 = 0.5;
+  caffe_vRngBernoulli(sample_size, static_cast<int*>(
       bernoulli1_data.mutable_cpu_data()), p1);
   TypeParam empirical_mean = this->sample_mean(
-      reinterpret_cast<const int*>(bernoulli1_data.cpu_data()),
+      static_cast<const int*>(bernoulli1_data.cpu_data()),
       sample_size);
   int bernoulli1_num_zeros = 0;
   int num_ones = 0;
   int num_other = 0;
   int* bernoulli_samples =
-      reinterpret_cast<int*>(bernoulli1_data.mutable_cpu_data());
+      static_cast<int*>(bernoulli1_data.mutable_cpu_data());
   for (int i = 0; i < sample_size; ++i) {
     if (bernoulli_samples[i] == 0) {
       ++bernoulli1_num_zeros;
@@ -351,19 +355,19 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) {
             << "; sample mean = " << empirical_mean;
   LOG(INFO) << "Bernoulli1: zeros: "  << bernoulli1_num_zeros
             << "; ones: " << num_ones << "; other: " << num_other;
-  empirical_mean =
-      this->sample_mean((const int *)bernoulli2_data.cpu_data(), sample_size);
+  empirical_mean = this->sample_mean(
+      static_cast<const int*>(bernoulli1_data.cpu_data()), sample_size);
   EXPECT_NEAR(empirical_mean, true_mean, bound);
   EXPECT_EQ(num_other, 0);
   // Sample from Bernoulli with p = 0.3
-  double p = 0.3;
-  caffe_vRngBernoulli(sample_size,
-      reinterpret_cast<int*>(bernoulli2_data.mutable_cpu_data()), p);
+  TypeParam p = 0.3;
+  caffe_vRngBernoulli(sample_size, static_cast<int*>(
+      bernoulli2_data.mutable_cpu_data()), p);
   true_mean = p;
   true_std = sqrt(p * (1 - p));
   bound = this->mean_bound(true_std, sample_size);
-  empirical_mean =
-      this->sample_mean((const int *)bernoulli2_data.cpu_data(), sample_size);
+  empirical_mean = this->sample_mean(
+      static_cast<const int*>(bernoulli2_data.cpu_data()), sample_size);
   LOG(INFO) << "Bernoulli2: Expected mean = " << true_mean
             << "; sample mean = " << empirical_mean;
   EXPECT_NEAR(empirical_mean, true_mean, bound);
@@ -371,7 +375,7 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) {
   num_ones = 0;
   num_other = 0;
   const int* bernoulli2_samples =
-      reinterpret_cast<const int*>(bernoulli2_data.cpu_data());
+      static_cast<const int*>(bernoulli2_data.cpu_data());
   for (int i = 0; i < sample_size; ++i) {
     if (bernoulli2_samples[i] == 0) {
       ++bernoulli2_num_zeros;
@@ -384,8 +388,10 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) {
   LOG(INFO) << "Bernoulli2: zeros: "  << bernoulli2_num_zeros
             << "; ones: " << num_ones << "; other: " << num_other;
   EXPECT_EQ(0, num_other);
-  EXPECT_EQ(sample_size * empirical_mean, num_ones);
-  EXPECT_EQ(sample_size * (1.0 - empirical_mean), bernoulli2_num_zeros);
+  TypeParam epsilon = 1e-4;
+  EXPECT_NEAR(sample_size * empirical_mean, num_ones, epsilon);
+  EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli2_num_zeros,
+              epsilon);
   // Multiply Bernoulli1 by Bernoulli2
   for (int i = 0; i < sample_size; ++i) {
     bernoulli_samples[i] *= bernoulli2_samples[i];
@@ -407,8 +413,8 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulliTimesBernoulli) {
   p *= p1;
   true_mean = p;
   true_std = sqrt(p * (1 - p));
-  empirical_mean =
-      this->sample_mean((const int *)bernoulli2_data.cpu_data(), sample_size);
+  empirical_mean = this->sample_mean(
+      static_cast<const int *>(bernoulli2_data.cpu_data()), sample_size);
   bound = this->mean_bound(true_std, sample_size);
   LOG(INFO) << "Bernoulli1*Bernoulli2: Expected mean = " << true_mean
             << "; sample mean = " << empirical_mean;

diff --git a/src/caffe/util/math_functions.cpp b/src/caffe/util/math_functions.cpp
index 3062dbf..0791a86 100644 (file)
--- a/src/caffe/util/math_functions.cpp
+++ b/src/caffe/util/math_functions.cpp
@@ -320,16 +320,13 @@ void caffe_vRngUniform(const int n, Dtype* r,
   CHECK_GE(n, 0);
   CHECK(r);
   CHECK_LE(a, b);
-
-  boost::uniform_real<Dtype> random_distribution(
-      a, caffe_nextafter<Dtype>(b));
-  boost::variate_generator<caffe::rng_t,
-      boost::uniform_real<Dtype> > variate_generator(
-      caffe_rng(), random_distribution);
-
+  boost::uniform_real<Dtype> random_distribution(a, caffe_nextafter<Dtype>(b));
+  boost::variate_generator<caffe::rng_t, boost::uniform_real<Dtype> >
+      variate_generator(caffe_rng(), random_distribution);
   for (int i = 0; i < n; ++i) {
     r[i] = variate_generator();
   }
+  caffe_set_rng(variate_generator.engine());
 }
 
 template
@@ -346,13 +343,12 @@ void caffe_vRngGaussian(const int n, Dtype* r, const Dtype a,
   CHECK(r);
   CHECK_GT(sigma, 0);
   boost::normal_distribution<Dtype> random_distribution(a, sigma);
-  boost::variate_generator<caffe::rng_t,
-      boost::normal_distribution<Dtype> > variate_generator(
-      caffe_rng(), random_distribution);
-
+  boost::variate_generator<caffe::rng_t, boost::normal_distribution<Dtype> >
+      variate_generator(caffe_rng(), random_distribution);
   for (int i = 0; i < n; ++i) {
     r[i] = variate_generator();
   }
+  caffe_set_rng(variate_generator.engine());
 }
 
 template
@@ -364,23 +360,25 @@ void caffe_vRngGaussian<double>(const int n, double* r, const double a,
     const double sigma);
 
 template <typename Dtype>
-void caffe_vRngBernoulli(const int n, Dtype* r, const double p) {
+void caffe_vRngBernoulli(const int n, int* r, const Dtype p) {
   CHECK_GE(n, 0);
   CHECK(r);
   CHECK_GE(p, 0);
   CHECK_LE(p, 1);
-  boost::bernoulli_distribution<double> random_distribution(p);
-  boost::variate_generator<caffe::rng_t,
-      boost::bernoulli_distribution<double> > variate_generator(
-      caffe_rng(), random_distribution);
-
+  boost::bernoulli_distribution<Dtype> random_distribution(p);
+  boost::variate_generator<caffe::rng_t, boost::bernoulli_distribution<Dtype> >
+      variate_generator(caffe_rng(), random_distribution);
   for (int i = 0; i < n; ++i) {
     r[i] = variate_generator();
   }
+  caffe_set_rng(variate_generator.engine());
 }
 
 template
-void caffe_vRngBernoulli<int>(const int n, int* r, const double p);
+void caffe_vRngBernoulli<double>(const int n, int* r, const double p);
+
+template
+void caffe_vRngBernoulli<float>(const int n, int* r, const float p);
 
 template <>
 float caffe_cpu_dot<float>(const int n, const float* x, const float* y) {