From 374fb8c79c3f23ee36c46d0bcaeb2176037aa4b8 Mon Sep 17 00:00:00 2001 From: Ran Date: Sat, 15 Aug 2015 20:09:43 +0300 Subject: [PATCH] Output accuracies per class. Fixed case where number of samples in class can be zero. - Fixed ignore_label case, also added a test. - Two other fixes. Fixed lint errors. Small fix. --- include/caffe/loss_layers.hpp | 8 ++- src/caffe/layers/accuracy_layer.cpp | 20 ++++++ src/caffe/test/test_accuracy_layer.cpp | 107 +++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 1 deletion(-) diff --git a/include/caffe/loss_layers.hpp b/include/caffe/loss_layers.hpp index 5282663..02687a9 100644 --- a/include/caffe/loss_layers.hpp +++ b/include/caffe/loss_layers.hpp @@ -39,7 +39,11 @@ class AccuracyLayer : public Layer { virtual inline const char* type() const { return "Accuracy"; } virtual inline int ExactNumBottomBlobs() const { return 2; } - virtual inline int ExactNumTopBlobs() const { return 1; } + + // If there are two top blobs, then the second blob will contain + // accuracies per class. + virtual inline int MinTopBlobs() const { return 1; } + virtual inline int MaxTopBlos() const { return 2; } protected: /** @@ -86,6 +90,8 @@ class AccuracyLayer : public Layer { bool has_ignore_label_; /// The label indicating that an instance should be ignored. int ignore_label_; + /// Keeps counts of the number of samples per class. + Blob nums_buffer_; }; /** diff --git a/src/caffe/layers/accuracy_layer.cpp b/src/caffe/layers/accuracy_layer.cpp index 90aad67..e2d8d9f 100644 --- a/src/caffe/layers/accuracy_layer.cpp +++ b/src/caffe/layers/accuracy_layer.cpp @@ -38,6 +38,13 @@ void AccuracyLayer::Reshape( << "with integer values in {0, 1, ..., C-1}."; vector top_shape(0); // Accuracy is a scalar; 0 axes. top[0]->Reshape(top_shape); + if (top.size() > 1) { + // Per-class accuracy is a vector; 1 axes. + vector top_shape_per_class(1); + top_shape_per_class[0] = bottom[0]->shape(label_axis_); + top[1]->Reshape(top_shape_per_class); + nums_buffer_.Reshape(top_shape_per_class); + } } template @@ -50,6 +57,10 @@ void AccuracyLayer::Forward_cpu(const vector*>& bottom, const int num_labels = bottom[0]->shape(label_axis_); vector maxval(top_k_+1); vector max_id(top_k_+1); + if (top.size() > 1) { + caffe_set(nums_buffer_.count(), Dtype(0), nums_buffer_.mutable_cpu_data()); + caffe_set(top[1]->count(), Dtype(0), top[1]->mutable_cpu_data()); + } int count = 0; for (int i = 0; i < outer_num_; ++i) { for (int j = 0; j < inner_num_; ++j) { @@ -58,6 +69,7 @@ void AccuracyLayer::Forward_cpu(const vector*>& bottom, if (has_ignore_label_ && label_value == ignore_label_) { continue; } + if (top.size() > 1) ++nums_buffer_.mutable_cpu_data()[label_value]; DCHECK_GE(label_value, 0); DCHECK_LT(label_value, num_labels); // Top-k accuracy @@ -73,6 +85,7 @@ void AccuracyLayer::Forward_cpu(const vector*>& bottom, for (int k = 0; k < top_k_; k++) { if (bottom_data_vector[k].second == label_value) { ++accuracy; + if (top.size() > 1) ++top[1]->mutable_cpu_data()[label_value]; break; } } @@ -82,6 +95,13 @@ void AccuracyLayer::Forward_cpu(const vector*>& bottom, // LOG(INFO) << "Accuracy: " << accuracy; top[0]->mutable_cpu_data()[0] = accuracy / count; + if (top.size() > 1) { + for (int i = 0; i < top[1]->count(); ++i) { + top[1]->mutable_cpu_data()[i] = + nums_buffer_.cpu_data()[i] == 0 ? 0 + : top[1]->cpu_data()[i] / nums_buffer_.cpu_data()[i]; + } + } // Accuracy layer should not be used as a loss function. } diff --git a/src/caffe/test/test_accuracy_layer.cpp b/src/caffe/test/test_accuracy_layer.cpp index c14b67c..94e529b 100644 --- a/src/caffe/test/test_accuracy_layer.cpp +++ b/src/caffe/test/test_accuracy_layer.cpp @@ -22,6 +22,7 @@ class AccuracyLayerTest : public CPUDeviceTest { : blob_bottom_data_(new Blob()), blob_bottom_label_(new Blob()), blob_top_(new Blob()), + blob_top_per_class_(new Blob()), top_k_(3) { vector shape(2); shape[0] = 100; @@ -34,6 +35,8 @@ class AccuracyLayerTest : public CPUDeviceTest { blob_bottom_vec_.push_back(blob_bottom_data_); blob_bottom_vec_.push_back(blob_bottom_label_); blob_top_vec_.push_back(blob_top_); + blob_top_per_class_vec_.push_back(blob_top_); + blob_top_per_class_vec_.push_back(blob_top_per_class_); } virtual void FillBottoms() { @@ -56,12 +59,15 @@ class AccuracyLayerTest : public CPUDeviceTest { delete blob_bottom_data_; delete blob_bottom_label_; delete blob_top_; + delete blob_top_per_class_; } Blob* const blob_bottom_data_; Blob* const blob_bottom_label_; Blob* const blob_top_; + Blob* const blob_top_per_class_; vector*> blob_bottom_vec_; vector*> blob_top_vec_; + vector*> blob_top_per_class_vec_; int top_k_; }; @@ -90,6 +96,20 @@ TYPED_TEST(AccuracyLayerTest, TestSetupTopK) { EXPECT_EQ(this->blob_top_->width(), 1); } +TYPED_TEST(AccuracyLayerTest, TestSetupOutputPerClass) { + LayerParameter layer_param; + AccuracyLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, this->blob_top_per_class_vec_); + EXPECT_EQ(this->blob_top_->num(), 1); + EXPECT_EQ(this->blob_top_->channels(), 1); + EXPECT_EQ(this->blob_top_->height(), 1); + EXPECT_EQ(this->blob_top_->width(), 1); + EXPECT_EQ(this->blob_top_per_class_->num(), 10); + EXPECT_EQ(this->blob_top_per_class_->channels(), 1); + EXPECT_EQ(this->blob_top_per_class_->height(), 1); + EXPECT_EQ(this->blob_top_per_class_->width(), 1); +} + TYPED_TEST(AccuracyLayerTest, TestForwardCPU) { LayerParameter layer_param; AccuracyLayer layer(layer_param); @@ -228,4 +248,91 @@ TYPED_TEST(AccuracyLayerTest, TestForwardCPUTopK) { num_correct_labels / 100.0, 1e-4); } +TYPED_TEST(AccuracyLayerTest, TestForwardCPUPerClass) { + LayerParameter layer_param; + Caffe::set_mode(Caffe::CPU); + AccuracyLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, this->blob_top_per_class_vec_); + layer.Forward(this->blob_bottom_vec_, this->blob_top_per_class_vec_); + + TypeParam max_value; + int max_id; + int num_correct_labels = 0; + const int num_class = this->blob_top_per_class_->num(); + vector correct_per_class(num_class, 0); + vector num_per_class(num_class, 0); + for (int i = 0; i < 100; ++i) { + max_value = -FLT_MAX; + max_id = 0; + for (int j = 0; j < 10; ++j) { + if (this->blob_bottom_data_->data_at(i, j, 0, 0) > max_value) { + max_value = this->blob_bottom_data_->data_at(i, j, 0, 0); + max_id = j; + } + } + ++num_per_class[this->blob_bottom_label_->data_at(i, 0, 0, 0)]; + if (max_id == this->blob_bottom_label_->data_at(i, 0, 0, 0)) { + ++num_correct_labels; + ++correct_per_class[max_id]; + } + } + EXPECT_NEAR(this->blob_top_->data_at(0, 0, 0, 0), + num_correct_labels / 100.0, 1e-4); + for (int i = 0; i < num_class; ++i) { + EXPECT_NEAR(this->blob_top_per_class_->data_at(i, 0, 0, 0), + static_cast(correct_per_class[i]) / num_per_class[i], + 1e-4); + } +} + + +TYPED_TEST(AccuracyLayerTest, TestForwardCPUPerClassWithIgnoreLabel) { + LayerParameter layer_param; + Caffe::set_mode(Caffe::CPU); + const TypeParam kIgnoreLabelValue = -1; + layer_param.mutable_accuracy_param()->set_ignore_label(kIgnoreLabelValue); + AccuracyLayer layer(layer_param); + // Manually set some labels to the ignore label value (-1). + this->blob_bottom_label_->mutable_cpu_data()[2] = kIgnoreLabelValue; + this->blob_bottom_label_->mutable_cpu_data()[5] = kIgnoreLabelValue; + this->blob_bottom_label_->mutable_cpu_data()[32] = kIgnoreLabelValue; + layer.SetUp(this->blob_bottom_vec_, this->blob_top_per_class_vec_); + layer.Forward(this->blob_bottom_vec_, this->blob_top_per_class_vec_); + + TypeParam max_value; + int max_id; + int num_correct_labels = 0; + const int num_class = this->blob_top_per_class_->num(); + vector correct_per_class(num_class, 0); + vector num_per_class(num_class, 0); + int count = 0; + for (int i = 0; i < 100; ++i) { + if (kIgnoreLabelValue == this->blob_bottom_label_->data_at(i, 0, 0, 0)) { + continue; + } + ++count; + max_value = -FLT_MAX; + max_id = 0; + for (int j = 0; j < 10; ++j) { + if (this->blob_bottom_data_->data_at(i, j, 0, 0) > max_value) { + max_value = this->blob_bottom_data_->data_at(i, j, 0, 0); + max_id = j; + } + } + ++num_per_class[this->blob_bottom_label_->data_at(i, 0, 0, 0)]; + if (max_id == this->blob_bottom_label_->data_at(i, 0, 0, 0)) { + ++num_correct_labels; + ++correct_per_class[max_id]; + } + } + EXPECT_EQ(count, 97); + EXPECT_NEAR(this->blob_top_->data_at(0, 0, 0, 0), + num_correct_labels / TypeParam(count), 1e-4); + for (int i = 0; i < 10; ++i) { + EXPECT_NEAR(this->blob_top_per_class_->data_at(i, 0, 0, 0), + TypeParam(correct_per_class[i]) / num_per_class[i], + 1e-4); + } +} + } // namespace caffe -- 2.7.4