From 27a0f9edb0255945fb339e5ec1b7a6b9776ddaf1 Mon Sep 17 00:00:00 2001 From: Yangqing Jia Date: Tue, 17 Sep 2013 16:14:10 -0700 Subject: [PATCH] working gradient check --- src/caffeine/common.cpp | 9 ++++++ src/caffeine/common.hpp | 1 + src/caffeine/test/test_gradient_check_util.cpp | 25 ++++++++++----- src/caffeine/test/test_gradient_check_util.hpp | 6 ++-- src/caffeine/test/test_neuron_layer.cpp | 42 ++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/src/caffeine/common.cpp b/src/caffeine/common.cpp index d5a7b22..7ab33ed 100644 --- a/src/caffeine/common.cpp +++ b/src/caffeine/common.cpp @@ -55,5 +55,14 @@ void Caffeine::set_phase(Caffeine::Phase phase) { Get().phase_ = phase; } +void Caffeine::set_random_seed(const unsigned int seed) { + // Curand seed + CURAND_CHECK(curandSetPseudoRandomGeneratorSeed(Get().curand_generator_, + seed)); + // VSL seed + VSL_CHECK(vslDeleteStream(&(Get().vsl_stream_))); + VSL_CHECK(vslNewStream(&(Get().vsl_stream_), VSL_BRNG_MT19937, seed)); +} + } // namespace caffeine diff --git a/src/caffeine/common.hpp b/src/caffeine/common.hpp index cc10434..1bcd785 100644 --- a/src/caffeine/common.hpp +++ b/src/caffeine/common.hpp @@ -50,6 +50,7 @@ class Caffeine { // The setters for the variables static void set_mode(Brew mode); static void set_phase(Phase phase); + static void set_random_seed(const unsigned int seed); private: Caffeine(); static shared_ptr singleton_; diff --git a/src/caffeine/test/test_gradient_check_util.cpp b/src/caffeine/test/test_gradient_check_util.cpp index 024e566..4b5c17d 100644 --- a/src/caffeine/test/test_gradient_check_util.cpp +++ b/src/caffeine/test/test_gradient_check_util.cpp @@ -1,4 +1,5 @@ #include +#include #include #include "caffeine/test/test_gradient_check_util.hpp" @@ -23,32 +24,42 @@ void GradientChecker::CheckGradient(Layer& layer, blobs_to_check.push_back(bottom[check_bottom]); } // go through the blobs - for (int i = 0; i < blobs_to_check.size(); ++i) { - Blob* current_blob = blobs_to_check[i]; + LOG(ERROR) << "Checking " << blobs_to_check.size() << " blobs."; + for (int blobid = 0; blobid < blobs_to_check.size(); ++blobid) { + Blob* current_blob = blobs_to_check[blobid]; + LOG(ERROR) << "Blob " << blobid << ": checking " << current_blob->count() + << " parameters."; // go through the values - for (int j = 0; j < current_blob->count(); ++j) { + for (int feat_id = 0; feat_id < current_blob->count(); ++feat_id) { // First, obtain the original data + Caffeine::set_random_seed(seed_); layer.Forward(bottom, &top); Dtype computed_objective = GetObjAndGradient(top); // Get any additional loss from the layer computed_objective += layer.Backward(top, true, &bottom); - Dtype computed_gradient = current_blob->cpu_diff()[i]; + Dtype computed_gradient = current_blob->cpu_diff()[feat_id]; // compute score by adding stepsize - current_blob->mutable_cpu_data()[i] += stepsize_; + current_blob->mutable_cpu_data()[feat_id] += stepsize_; + Caffeine::set_random_seed(seed_); layer.Forward(bottom, &top); Dtype positive_objective = GetObjAndGradient(top); positive_objective += layer.Backward(top, true, &bottom); // compute score by subtracting stepsize - current_blob->mutable_cpu_data()[i] -= stepsize_ * 2; + current_blob->mutable_cpu_data()[feat_id] -= stepsize_ * 2; + Caffeine::set_random_seed(seed_); layer.Forward(bottom, &top); Dtype negative_objective = GetObjAndGradient(top); negative_objective += layer.Backward(top, true, &bottom); // Recover stepsize - current_blob->mutable_cpu_data()[i] += stepsize_; + current_blob->mutable_cpu_data()[feat_id] += stepsize_; Dtype estimated_gradient = (positive_objective - negative_objective) / stepsize_ / 2.; + Dtype feature = current_blob->cpu_data()[feat_id]; EXPECT_GT(computed_gradient, estimated_gradient - threshold_); EXPECT_LT(computed_gradient, estimated_gradient + threshold_); + //LOG(ERROR) << "Feature: " << current_blob->cpu_data()[feat_id]; + //LOG(ERROR) << "computed gradient: " << computed_gradient + // << " estimated_gradient: " << estimated_gradient; } } } diff --git a/src/caffeine/test/test_gradient_check_util.hpp b/src/caffeine/test/test_gradient_check_util.hpp index 1c00c10..8ae5ed5 100644 --- a/src/caffeine/test/test_gradient_check_util.hpp +++ b/src/caffeine/test/test_gradient_check_util.hpp @@ -10,8 +10,9 @@ namespace caffeine { template class GradientChecker { public: - GradientChecker(const Dtype stepsize, const Dtype threshold) - : stepsize_(stepsize), threshold_(threshold) {}; + GradientChecker(const Dtype stepsize, const Dtype threshold, + const unsigned int seed = 1701) + : stepsize_(stepsize), threshold_(threshold), seed_(seed) {}; // Checks the gradient of a layer, with provided bottom layers and top // layers. The gradient checker will check the gradient with respect to // the parameters of the layer, as well as the input blobs if check_through @@ -24,6 +25,7 @@ class GradientChecker { Dtype GetObjAndGradient(vector*>& top); Dtype stepsize_; Dtype threshold_; + unsigned int seed_; }; } // namespace caffeine diff --git a/src/caffeine/test/test_neuron_layer.cpp b/src/caffeine/test/test_neuron_layer.cpp index d64a014..3eff333 100644 --- a/src/caffeine/test/test_neuron_layer.cpp +++ b/src/caffeine/test/test_neuron_layer.cpp @@ -6,6 +6,7 @@ #include "caffeine/common.hpp" #include "caffeine/filler.hpp" #include "caffeine/vision_layers.hpp" +#include "caffeine/test/test_gradient_check_util.hpp" namespace caffeine { @@ -47,6 +48,16 @@ TYPED_TEST(NeuronLayerTest, TestReLUCPU) { } } + +TYPED_TEST(NeuronLayerTest, TestReLUGradientCPU) { + LayerParameter layer_param; + Caffeine::set_mode(Caffeine::CPU); + ReLULayer layer(layer_param); + GradientChecker checker(1e-3, 1e-3); + checker.CheckGradient(layer, this->blob_bottom_vec_, this->blob_top_vec_); +} + + TYPED_TEST(NeuronLayerTest, TestReLUGPU) { LayerParameter layer_param; Caffeine::set_mode(Caffeine::GPU); @@ -62,6 +73,16 @@ TYPED_TEST(NeuronLayerTest, TestReLUGPU) { } } + +TYPED_TEST(NeuronLayerTest, TestReLUGradientGPU) { + LayerParameter layer_param; + Caffeine::set_mode(Caffeine::GPU); + ReLULayer layer(layer_param); + GradientChecker checker(1e-3, 1e-3); + checker.CheckGradient(layer, this->blob_bottom_vec_, this->blob_top_vec_); +} + + TYPED_TEST(NeuronLayerTest, TestDropoutCPU) { LayerParameter layer_param; Caffeine::set_mode(Caffeine::CPU); @@ -80,6 +101,16 @@ TYPED_TEST(NeuronLayerTest, TestDropoutCPU) { } } + +TYPED_TEST(NeuronLayerTest, TestDropoutGradientCPU) { + LayerParameter layer_param; + Caffeine::set_mode(Caffeine::CPU); + DropoutLayer layer(layer_param); + GradientChecker checker(1e-2, 1e-3); + checker.CheckGradient(layer, this->blob_bottom_vec_, this->blob_top_vec_); +} + + TYPED_TEST(NeuronLayerTest, TestDropoutCPUTestPhase) { LayerParameter layer_param; Caffeine::set_mode(Caffeine::CPU); @@ -98,6 +129,7 @@ TYPED_TEST(NeuronLayerTest, TestDropoutCPUTestPhase) { } } + TYPED_TEST(NeuronLayerTest, TestDropoutGPU) { LayerParameter layer_param; Caffeine::set_mode(Caffeine::GPU); @@ -116,6 +148,16 @@ TYPED_TEST(NeuronLayerTest, TestDropoutGPU) { } } + +TYPED_TEST(NeuronLayerTest, TestDropoutGradientGPU) { + LayerParameter layer_param; + Caffeine::set_mode(Caffeine::GPU); + DropoutLayer layer(layer_param); + GradientChecker checker(1e-2, 1e-3); + checker.CheckGradient(layer, this->blob_bottom_vec_, this->blob_top_vec_); +} + + TYPED_TEST(NeuronLayerTest, TestDropoutGPUTestPhase) { LayerParameter layer_param; Caffeine::set_mode(Caffeine::GPU); -- 2.7.4