From c6518fb5752344e1922eaa1b1eb686bae5cc3964 Mon Sep 17 00:00:00 2001 From: Eli Osherovich Date: Thu, 11 Feb 2016 09:07:43 +0200 Subject: [PATCH 1/2] First implementation of the triplet loss. Signed-off-by: Eli Osherovich --- include/caffe/layers/triplet_loss_layer.hpp | 48 +++++++++ src/caffe/layers/triplet_loss_layer.cpp | 93 +++++++++++++++++ src/caffe/test/test_triplet_loss_layer.cpp | 108 ++++++++++++++++++++ 3 files changed, 249 insertions(+) create mode 100644 include/caffe/layers/triplet_loss_layer.hpp create mode 100644 src/caffe/layers/triplet_loss_layer.cpp create mode 100644 src/caffe/test/test_triplet_loss_layer.cpp diff --git a/include/caffe/layers/triplet_loss_layer.hpp b/include/caffe/layers/triplet_loss_layer.hpp new file mode 100644 index 00000000000..b96a4371f50 --- /dev/null +++ b/include/caffe/layers/triplet_loss_layer.hpp @@ -0,0 +1,48 @@ +#ifndef CAFFE_TRIPLET_LOSS_LAYER_HPP_ +#define CAFFE_TRIPLET_LOSS_LAYER_HPP_ + +#include + +#include "caffe/blob.hpp" +#include "caffe/layer.hpp" +#include "caffe/proto/caffe.pb.h" + +#include "caffe/layers/loss_layer.hpp" + +namespace caffe { + +template +class TripletLossLayer : public LossLayer { + public: + explicit TripletLossLayer(const LayerParameter& param) + : LossLayer(param), diff_same_class_(), diff_diff_class_() {} + + void Reshape(const vector*>& bottom, + const vector*>& top); + + inline const char* type() const { return "TripletLoss"; } + inline int ExactNumBottomBlobs() const { return 3; } + inline bool AllowForceBackward(const int bottom_index) const { return true; } + + virtual void LayerSetUp(const vector*>& bottom, + const vector*>& top); + + protected: + /// @copydoc TripletLossLayer + virtual void Forward_cpu(const vector*>& bottom, + const vector*>& top); + virtual void Backward_cpu(const vector*>& top, + const vector& propagate_down, + const vector*>& bottom); + + Blob diff_same_class_; + Blob diff_diff_class_; + Dtype alpha_; + vector vec_loss_; + int batch_size_; + int vec_dimension_; +}; + +} // namespace caffe + +#endif // CAFFE_TRIPLET_LOSS_LAYER_HPP_ diff --git a/src/caffe/layers/triplet_loss_layer.cpp b/src/caffe/layers/triplet_loss_layer.cpp new file mode 100644 index 00000000000..34a61449e3f --- /dev/null +++ b/src/caffe/layers/triplet_loss_layer.cpp @@ -0,0 +1,93 @@ +#include + +#include "caffe/layers/triplet_loss_layer.hpp" +#include "caffe/util/math_functions.hpp" + +namespace caffe { + +template +void TripletLossLayer::Reshape(const vector*>& bottom, + const vector*>& top) { + CHECK(bottom[0]->shape() == bottom[1]->shape()) + << "Inputs must have the same dimension."; + CHECK(bottom[0]->shape() == bottom[2]->shape()) + << "Inputs must have the same dimension."; + diff_same_class_.ReshapeLike(*bottom[0]); + diff_diff_class_.ReshapeLike(*bottom[0]); + + vector loss_shape(0); // Loss layers output a scalar; 0 axes. + top[0]->Reshape(loss_shape); + batch_size_ = bottom[0]->shape(0); + vec_dimension_ = bottom[0]->count()/batch_size_; + vec_loss_.resize(batch_size_); +} + +template +void TripletLossLayer::LayerSetUp(const vector*>& bottom, + const vector*>& top) { + LossLayer::LayerSetUp(bottom, top); + alpha_ = this->layer_param_.threshold_param().threshold(); +} + +template +void TripletLossLayer::Forward_cpu(const vector*>& bottom, + const vector*>& top) { + int count = bottom[0]->count(); + + caffe_sub(count, bottom[0]->cpu_data(), bottom[1]->cpu_data(), + diff_same_class_.mutable_cpu_data()); + caffe_sub(count, bottom[0]->cpu_data(), bottom[2]->cpu_data(), + diff_diff_class_.mutable_cpu_data()); + + Dtype loss = 0; + for (int v = 0; v < batch_size_; ++v) { + vec_loss_[v] = + alpha_ + + caffe_cpu_dot(vec_dimension_, + diff_same_class_.cpu_data() + v * vec_dimension_, + diff_same_class_.cpu_data() + v * vec_dimension_) - + caffe_cpu_dot(vec_dimension_, + diff_diff_class_.cpu_data() + v * vec_dimension_, + diff_diff_class_.cpu_data() + v * vec_dimension_); + vec_loss_[v] = std::max(Dtype(0), vec_loss_[v]); + loss += vec_loss_[v]; + } + + loss /= batch_size_ * Dtype(2); + top[0]->mutable_cpu_data()[0] = loss; +} + +template +void TripletLossLayer::Backward_cpu(const vector*>& top, + const vector& propagate_down, + const vector*>& bottom) { + const Dtype scale = top[0]->cpu_diff()[0] / bottom[0]->num(); + const int n = bottom[0]->count(); + + caffe_sub(n, diff_same_class_.cpu_data(), diff_diff_class_.cpu_data(), + bottom[0]->mutable_cpu_diff()); + caffe_scal(n, scale, bottom[0]->mutable_cpu_diff()); + + caffe_cpu_scale(n, -scale, diff_same_class_.cpu_data(), + bottom[1]->mutable_cpu_diff()); + + caffe_cpu_scale(n, scale, diff_diff_class_.cpu_data(), + bottom[2]->mutable_cpu_diff()); + + for (int v = 0; v < batch_size_; ++v) { + if (vec_loss_[v] == 0) { + caffe_set(vec_dimension_, Dtype(0), bottom[0]->mutable_cpu_diff() + v*vec_dimension_); + caffe_set(vec_dimension_, Dtype(0), bottom[1]->mutable_cpu_diff() + v*vec_dimension_); + caffe_set(vec_dimension_, Dtype(0), bottom[2]->mutable_cpu_diff() + v*vec_dimension_); + } + } +} + +#ifdef CPU_ONLY + //STUB_GPU(TripletLossLayer); +#endif + +INSTANTIATE_CLASS(TripletLossLayer); +REGISTER_LAYER_CLASS(TripletLoss); + +} // namespace caffe diff --git a/src/caffe/test/test_triplet_loss_layer.cpp b/src/caffe/test/test_triplet_loss_layer.cpp new file mode 100644 index 00000000000..a55ea14cd30 --- /dev/null +++ b/src/caffe/test/test_triplet_loss_layer.cpp @@ -0,0 +1,108 @@ +#include +#include + +#include "gtest/gtest.h" + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/layers/triplet_loss_layer.hpp" + +#include "caffe/test/test_caffe_main.hpp" +#include "caffe/test/test_gradient_check_util.hpp" + +namespace caffe { + +template +class TripletLossLayerTest : public MultiDeviceTest { + typedef typename TypeParam::Dtype Dtype; + + protected: + TripletLossLayerTest() + : blob_bottom_anchor_(new Blob(10, 4, 5, 2)), + blob_bottom_same_(new Blob(10, 4, 5, 2)), + blob_bottom_diff_(new Blob(10, 4, 5, 2)), + blob_top_loss_(new Blob()) { + // fill the values + FillerParameter filler_param; + GaussianFiller filler(filler_param); + filler.Fill(this->blob_bottom_anchor_); + blob_bottom_vec_.push_back(blob_bottom_anchor_); + + filler.Fill(this->blob_bottom_same_); + blob_bottom_vec_.push_back(blob_bottom_same_); + + filler.Fill(this->blob_bottom_diff_); + blob_bottom_vec_.push_back(blob_bottom_diff_); + + blob_top_vec_.push_back(blob_top_loss_); + } + virtual ~TripletLossLayerTest() { + delete blob_bottom_anchor_; + delete blob_bottom_same_; + delete blob_bottom_diff_; + delete blob_top_loss_; + } + + void TestForward() { + // Get the loss without a specified objective weight -- should be + // equivalent to explicitly specifiying a weight of 1. + LayerParameter layer_param; + TripletLossLayer layer_weight_1(layer_param); + layer_weight_1.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + const Dtype loss_weight_1 = + layer_weight_1.Forward(this->blob_bottom_vec_, this->blob_top_vec_); + + // Make sure the loss is non-trivial. + const Dtype kNonTrivialAbsThresh = 1e-1; + EXPECT_GE(fabs(loss_weight_1), kNonTrivialAbsThresh); + + // Get the loss again with a different objective weight; check that it is + // scaled appropriately. + const Dtype kLossWeight = 3.7; + layer_param.add_loss_weight(kLossWeight); + TripletLossLayer layer_weight_2(layer_param); + layer_weight_2.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + const Dtype loss_weight_2 = + layer_weight_2.Forward(this->blob_bottom_vec_, this->blob_top_vec_); + const Dtype kErrorMargin = 1e-5; + EXPECT_NEAR(loss_weight_1 * kLossWeight, loss_weight_2, kErrorMargin); + + // Get the loss again with a different alpha; check that it is changed + // appropriately. + const Dtype kAlpha = 0.314; + layer_param.mutable_threshold_param()->set_threshold(kAlpha); + TripletLossLayer layer_weight_2_alpha(layer_param); + layer_weight_2_alpha.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + const Dtype loss_weight_2_alpha = + layer_weight_2_alpha.Forward(this->blob_bottom_vec_, this->blob_top_vec_); + EXPECT_GE(loss_weight_2_alpha, loss_weight_2); + } + + Blob* const blob_bottom_anchor_; + Blob* const blob_bottom_same_; + Blob* const blob_bottom_diff_; + Blob* const blob_top_loss_; + vector*> blob_bottom_vec_; + vector*> blob_top_vec_; +}; + +TYPED_TEST_CASE(TripletLossLayerTest, TestDtypesAndDevices); + +TYPED_TEST(TripletLossLayerTest, TestForward) { + this->TestForward(); +} + +TYPED_TEST(TripletLossLayerTest, TestGradient) { + typedef typename TypeParam::Dtype Dtype; + LayerParameter layer_param; + const Dtype kLossWeight = 3.7; + layer_param.add_loss_weight(kLossWeight); + TripletLossLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + GradientChecker checker(1e-2, 1e-2, 1701); + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_); +} + +} // namespace caffe From 0a9a81d20471176b796cb953df854cc0e3c5dcd1 Mon Sep 17 00:00:00 2001 From: Eli Osherovich Date: Thu, 11 Feb 2016 09:20:52 +0200 Subject: [PATCH 2/2] Code linting and style adjustments. Signed-off-by: Eli Osherovich --- include/caffe/layers/triplet_loss_layer.hpp | 4 ++-- src/caffe/layers/triplet_loss_layer.cpp | 24 ++++++++++++--------- src/caffe/test/test_triplet_loss_layer.cpp | 10 ++++----- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/include/caffe/layers/triplet_loss_layer.hpp b/include/caffe/layers/triplet_loss_layer.hpp index b96a4371f50..a052380e33e 100644 --- a/include/caffe/layers/triplet_loss_layer.hpp +++ b/include/caffe/layers/triplet_loss_layer.hpp @@ -26,7 +26,7 @@ class TripletLossLayer : public LossLayer { virtual void LayerSetUp(const vector*>& bottom, const vector*>& top); - + protected: /// @copydoc TripletLossLayer virtual void Forward_cpu(const vector*>& bottom, @@ -34,7 +34,7 @@ class TripletLossLayer : public LossLayer { virtual void Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom); - + Blob diff_same_class_; Blob diff_diff_class_; Dtype alpha_; diff --git a/src/caffe/layers/triplet_loss_layer.cpp b/src/caffe/layers/triplet_loss_layer.cpp index 34a61449e3f..01f6bcdb690 100644 --- a/src/caffe/layers/triplet_loss_layer.cpp +++ b/src/caffe/layers/triplet_loss_layer.cpp @@ -1,3 +1,4 @@ +#include #include #include "caffe/layers/triplet_loss_layer.hpp" @@ -7,7 +8,7 @@ namespace caffe { template void TripletLossLayer::Reshape(const vector*>& bottom, - const vector*>& top) { + const vector*>& top) { CHECK(bottom[0]->shape() == bottom[1]->shape()) << "Inputs must have the same dimension."; CHECK(bottom[0]->shape() == bottom[2]->shape()) @@ -18,13 +19,13 @@ void TripletLossLayer::Reshape(const vector*>& bottom, vector loss_shape(0); // Loss layers output a scalar; 0 axes. top[0]->Reshape(loss_shape); batch_size_ = bottom[0]->shape(0); - vec_dimension_ = bottom[0]->count()/batch_size_; + vec_dimension_ = bottom[0]->count() / batch_size_; vec_loss_.resize(batch_size_); } template void TripletLossLayer::LayerSetUp(const vector*>& bottom, - const vector*>& top) { + const vector*>& top) { LossLayer::LayerSetUp(bottom, top); alpha_ = this->layer_param_.threshold_param().threshold(); } @@ -33,7 +34,7 @@ template void TripletLossLayer::Forward_cpu(const vector*>& bottom, const vector*>& top) { int count = bottom[0]->count(); - + caffe_sub(count, bottom[0]->cpu_data(), bottom[1]->cpu_data(), diff_same_class_.mutable_cpu_data()); caffe_sub(count, bottom[0]->cpu_data(), bottom[2]->cpu_data(), @@ -59,8 +60,8 @@ void TripletLossLayer::Forward_cpu(const vector*>& bottom, template void TripletLossLayer::Backward_cpu(const vector*>& top, - const vector& propagate_down, - const vector*>& bottom) { + const vector& propagate_down, + const vector*>& bottom) { const Dtype scale = top[0]->cpu_diff()[0] / bottom[0]->num(); const int n = bottom[0]->count(); @@ -76,15 +77,18 @@ void TripletLossLayer::Backward_cpu(const vector*>& top, for (int v = 0; v < batch_size_; ++v) { if (vec_loss_[v] == 0) { - caffe_set(vec_dimension_, Dtype(0), bottom[0]->mutable_cpu_diff() + v*vec_dimension_); - caffe_set(vec_dimension_, Dtype(0), bottom[1]->mutable_cpu_diff() + v*vec_dimension_); - caffe_set(vec_dimension_, Dtype(0), bottom[2]->mutable_cpu_diff() + v*vec_dimension_); + caffe_set(vec_dimension_, Dtype(0), + bottom[0]->mutable_cpu_diff() + v * vec_dimension_); + caffe_set(vec_dimension_, Dtype(0), + bottom[1]->mutable_cpu_diff() + v * vec_dimension_); + caffe_set(vec_dimension_, Dtype(0), + bottom[2]->mutable_cpu_diff() + v * vec_dimension_); } } } #ifdef CPU_ONLY - //STUB_GPU(TripletLossLayer); +// STUB_GPU(TripletLossLayer); #endif INSTANTIATE_CLASS(TripletLossLayer); diff --git a/src/caffe/test/test_triplet_loss_layer.cpp b/src/caffe/test/test_triplet_loss_layer.cpp index a55ea14cd30..a262e0cea36 100644 --- a/src/caffe/test/test_triplet_loss_layer.cpp +++ b/src/caffe/test/test_triplet_loss_layer.cpp @@ -74,8 +74,8 @@ class TripletLossLayerTest : public MultiDeviceTest { layer_param.mutable_threshold_param()->set_threshold(kAlpha); TripletLossLayer layer_weight_2_alpha(layer_param); layer_weight_2_alpha.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); - const Dtype loss_weight_2_alpha = - layer_weight_2_alpha.Forward(this->blob_bottom_vec_, this->blob_top_vec_); + const Dtype loss_weight_2_alpha = layer_weight_2_alpha.Forward( + this->blob_bottom_vec_, this->blob_top_vec_); EXPECT_GE(loss_weight_2_alpha, loss_weight_2); } @@ -89,9 +89,7 @@ class TripletLossLayerTest : public MultiDeviceTest { TYPED_TEST_CASE(TripletLossLayerTest, TestDtypesAndDevices); -TYPED_TEST(TripletLossLayerTest, TestForward) { - this->TestForward(); -} +TYPED_TEST(TripletLossLayerTest, TestForward) { this->TestForward(); } TYPED_TEST(TripletLossLayerTest, TestGradient) { typedef typename TypeParam::Dtype Dtype; @@ -102,7 +100,7 @@ TYPED_TEST(TripletLossLayerTest, TestGradient) { layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); GradientChecker checker(1e-2, 1e-2, 1701); checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, - this->blob_top_vec_); + this->blob_top_vec_); } } // namespace caffe