numerical stability improvement

diff --git a/src/caffe/layers/relu_layer.cpp b/src/caffe/layers/relu_layer.cpp
index 3a3e8a2..b94086d 100644 (file)
--- a/src/caffe/layers/relu_layer.cpp
+++ b/src/caffe/layers/relu_layer.cpp
@@ -33,8 +33,8 @@ void ReLULayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
     const int count = (*bottom)[0]->count();
     Dtype negative_slope = this->layer_param_.relu_param().negative_slope();
     for (int i = 0; i < count; ++i) {
-      bottom_diff[i] = top_diff[i] * ((bottom_data[i] >= 0)
-          + negative_slope * (bottom_data[i] < 0));
+      bottom_diff[i] = top_diff[i] * ((bottom_data[i] > 0)
+          + negative_slope * (bottom_data[i] <= 0));
     }
   }
 }

diff --git a/src/caffe/layers/relu_layer.cu b/src/caffe/layers/relu_layer.cu
index 503147c..e0d6aa1 100644 (file)
--- a/src/caffe/layers/relu_layer.cu
+++ b/src/caffe/layers/relu_layer.cu
@@ -39,8 +39,8 @@ template <typename Dtype>
 __global__ void ReLUBackward(const int n, const Dtype* in_diff,
     const Dtype* in_data, Dtype* out_diff, Dtype negative_slope) {
   CUDA_KERNEL_LOOP(index, n) {
-    out_diff[index] = in_diff[index] * ((in_data[index] >= 0)
-        + (in_data[index] < 0) * negative_slope);
+    out_diff[index] = in_diff[index] * ((in_data[index] > 0)
+        + (in_data[index] <= 0) * negative_slope);
   }
 }