From ee5191b3e41fddae73653e0d61172360b89526ca Mon Sep 17 00:00:00 2001 From: Kang Kim Date: Thu, 8 Oct 2015 01:26:25 +0900 Subject: [PATCH] Improve numerical stability of variance computation in MVNLayer --- src/caffe/layers/mvn_layer.cpp | 42 ++++++++++++----------------------------- src/caffe/layers/mvn_layer.cu | 43 +++++++++++++----------------------------- 2 files changed, 25 insertions(+), 60 deletions(-) diff --git a/src/caffe/layers/mvn_layer.cpp b/src/caffe/layers/mvn_layer.cpp index 325691b..61c2141 100644 --- a/src/caffe/layers/mvn_layer.cpp +++ b/src/caffe/layers/mvn_layer.cpp @@ -42,29 +42,21 @@ void MVNLayer::Forward_cpu(const vector*>& bottom, int dim = bottom[0]->count() / num; - if (this->layer_param_.mvn_param().normalize_variance()) { - // put the squares of bottom into temp_ - caffe_powx(bottom[0]->count(), bottom_data, Dtype(2), - temp_.mutable_cpu_data()); + // subtract mean + caffe_cpu_gemv(CblasNoTrans, num, dim, 1. / dim, bottom_data, + sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data()); // EX + caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, -1., + mean_.cpu_data(), sum_multiplier_.cpu_data(), 0., + temp_.mutable_cpu_data()); + caffe_add(temp_.count(), bottom_data, temp_.cpu_data(), top_data); // X-EX - // computes variance using var(X) = E(X^2) - (EX)^2 - caffe_cpu_gemv(CblasNoTrans, num, dim, 1. / dim, bottom_data, - sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data()); // EX + if (this->layer_param_.mvn_param().normalize_variance()) { + // compute variance using var(X) = E((X-EX)^2) + caffe_powx(bottom[0]->count(), top_data, Dtype(2), + temp_.mutable_cpu_data()); // (X-EX)^2 caffe_cpu_gemv(CblasNoTrans, num, dim, 1. / dim, temp_.cpu_data(), sum_multiplier_.cpu_data(), 0., - variance_.mutable_cpu_data()); // E(X^2) - caffe_powx(mean_.count(), mean_.cpu_data(), Dtype(2), - temp_.mutable_cpu_data()); // (EX)^2 - caffe_sub(mean_.count(), variance_.cpu_data(), temp_.cpu_data(), - variance_.mutable_cpu_data()); // variance - - // do mean and variance normalization - // subtract mean - caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, -1., - mean_.cpu_data(), sum_multiplier_.cpu_data(), 0., - temp_.mutable_cpu_data()); - - caffe_add(temp_.count(), bottom_data, temp_.cpu_data(), top_data); + variance_.mutable_cpu_data()); // E((X-EX)^2) // normalize variance caffe_powx(variance_.count(), variance_.cpu_data(), Dtype(0.5), @@ -77,16 +69,6 @@ void MVNLayer::Forward_cpu(const vector*>& bottom, temp_.mutable_cpu_data()); caffe_div(temp_.count(), top_data, temp_.cpu_data(), top_data); - } else { - caffe_cpu_gemv(CblasNoTrans, num, dim, 1. / dim, bottom_data, - sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data()); // EX - - // subtract mean - caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, -1., - mean_.cpu_data(), sum_multiplier_.cpu_data(), 0., - temp_.mutable_cpu_data()); - - caffe_add(temp_.count(), bottom_data, temp_.cpu_data(), top_data); } } diff --git a/src/caffe/layers/mvn_layer.cu b/src/caffe/layers/mvn_layer.cu index d86a2e7..5cbb112 100644 --- a/src/caffe/layers/mvn_layer.cu +++ b/src/caffe/layers/mvn_layer.cu @@ -20,29 +20,22 @@ void MVNLayer::Forward_gpu(const vector*>& bottom, int dim = bottom[0]->count() / num; - if (this->layer_param_.mvn_param().normalize_variance()) { - // put the squares of bottom into temp_ - caffe_gpu_powx(bottom[0]->count(), bottom_data, Dtype(2), - temp_.mutable_gpu_data()); + // subtract mean + caffe_gpu_gemv(CblasNoTrans, num, dim, 1. / dim, bottom_data, + sum_multiplier_.gpu_data(), 0., mean_.mutable_gpu_data()); // EX + caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, -1., + mean_.gpu_data(), sum_multiplier_.gpu_data(), 0., + temp_.mutable_gpu_data()); + caffe_gpu_add(temp_.count(), bottom_data, temp_.gpu_data(), + top_data); // X-EX - // computes variance using var(X) = E(X^2) - (EX)^2 - caffe_gpu_gemv(CblasNoTrans, num, dim, 1. / dim, bottom_data, - sum_multiplier_.gpu_data(), 0., mean_.mutable_gpu_data()); // EX + if (this->layer_param_.mvn_param().normalize_variance()) { + // compute variance using var(X) = E((X-EX)^2) + caffe_gpu_powx(bottom[0]->count(), top_data, Dtype(2), + temp_.mutable_gpu_data()); // (X-EX)^2 caffe_gpu_gemv(CblasNoTrans, num, dim, 1. / dim, temp_.gpu_data(), sum_multiplier_.gpu_data(), 0., - variance_.mutable_gpu_data()); // E(X^2) - caffe_gpu_powx(mean_.count(), mean_.gpu_data(), Dtype(2), - temp_.mutable_gpu_data()); // (EX)^2 - caffe_gpu_sub(mean_.count(), variance_.gpu_data(), temp_.gpu_data(), - variance_.mutable_gpu_data()); // variance - - // do mean and variance normalization - // subtract mean - caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, -1., - mean_.gpu_data(), sum_multiplier_.gpu_data(), 0., - temp_.mutable_gpu_data()); - - caffe_gpu_add(temp_.count(), bottom_data, temp_.gpu_data(), top_data); + variance_.mutable_gpu_data()); // E((X-EX)^2) // normalize variance caffe_gpu_powx(variance_.count(), variance_.gpu_data(), Dtype(0.5), @@ -55,16 +48,6 @@ void MVNLayer::Forward_gpu(const vector*>& bottom, temp_.mutable_gpu_data()); caffe_gpu_div(temp_.count(), top_data, temp_.gpu_data(), top_data); - } else { - caffe_gpu_gemv(CblasNoTrans, num, dim, 1. / dim, bottom_data, - sum_multiplier_.gpu_data(), 0., mean_.mutable_gpu_data()); // EX - - // subtract mean - caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, -1., - mean_.gpu_data(), sum_multiplier_.gpu_data(), 0., - temp_.mutable_gpu_data()); - - caffe_gpu_add(temp_.count(), bottom_data, temp_.gpu_data(), top_data); } } -- 2.7.4