diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp index 5af7b28df3c..fb0da9809a6 100644 --- a/include/caffe/vision_layers.hpp +++ b/include/caffe/vision_layers.hpp @@ -686,6 +686,39 @@ class SplitLayer : public Layer { int count_; }; +/* + * Weighted Approximate-Rank Pairwise loss for image retrieval/annotation etc. + * References: + * [1] Yunchao Gong, Yangqing Jia, Sergey Ioffe, Alexander Toshev, Thomas + * Leung. Deep Convolutional Ranking for Multilabel Image Annotation. + * arXiv:1312.4894 [cs.CV] + * [2] Jason Weston, Samy Bengio, and Nicolas Usunier. Wsabie: Scaling up + * to large vocabulary image annotation. In IJCAI, 2011. + */ +template +class WARPLossLayer : public Layer { + public: + explicit WARPLossLayer(const LayerParameter& param) + : Layer(param) {} + virtual void SetUp(const vector*>& bottom, + vector*>* top); + + protected: + virtual Dtype Forward_cpu(const vector*>& bottom, + vector*>* top); + virtual Dtype Forward_gpu(const vector*>& bottom, + vector*>* top); + virtual void Backward_cpu(const vector*>& top, + const bool propagate_down, vector*>* bottom) { return; } + virtual void Backward_gpu(const vector*>& top, + const bool propagate_down, vector*>* bottom) { return; } + // In image retrieval or annotations, results or labels form ranked list + // based on their scores. + // Weighted Approximate-Rank Pairwise loss uses a function to + // set weight for different ranks, e.g. 1 / j for the j-th rank. + Blob rank_weights_; +}; + // This function is used to create a pthread that prefetches the window data. template void* WindowDataLayerPrefetch(void* layer_pointer); diff --git a/src/caffe/layers/warp_loss_layer.cpp b/src/caffe/layers/warp_loss_layer.cpp new file mode 100644 index 00000000000..df54996a103 --- /dev/null +++ b/src/caffe/layers/warp_loss_layer.cpp @@ -0,0 +1,105 @@ +// Copyright 2014 kloudkl@github + +#include + +#include "caffe/blob.hpp" +#include "caffe/vision_layers.hpp" +#include "caffe/util/math_functions.hpp" + +namespace caffe { + +template +void WARPLossLayer::SetUp(const vector*>& bottom, + vector*>* top) { + CHECK_EQ(bottom.size(), 2)<< + "WARPLossLayer takes two blobs as input."; + CHECK_EQ(top->size(), 0) << "WARPLossLayer takes no output."; + CHECK_EQ(bottom[0]->num(), bottom[1]->num()) << + "The two input blobs should have the same number."; + int dim = bottom[0]->count() / bottom[0]->num(); + rank_weights_.Reshape(1, dim, 1, 1); + Dtype* rank_weights_data = rank_weights_.mutable_cpu_data(); + rank_weights_data[0] = 1. / (1 + 0); + for (int i = 1; i < dim; ++i) { + rank_weights_data[i] = rank_weights_data[i - 1] + 1. / (1 + i); + } +}; + +template +Dtype WARPLossLayer::Forward_cpu(const vector*>& bottom, + vector*>* top) { + const Dtype* bottom_data = bottom[0]->cpu_data(); + const Dtype* bottom_labels = bottom[1]->cpu_data(); + const Dtype* rank_weights_data = rank_weights_.cpu_data(); + const int num_data = bottom[0]->num(); + const int dim = bottom[0]->count() / bottom[0]->num(); + const int num_bottom_labels = dim; + const int max_num_trials = num_bottom_labels - 1; + Dtype* bottom_diff = bottom[0]->mutable_cpu_diff(); + memset(bottom_diff, 0, sizeof(Dtype) * bottom[0]->count()); + Dtype loss = 0; + Dtype score_margin; + int random_label; + int estimated_rank; + int num_trials; + int offset; + Dtype rank_weight; + for (int i = 0; i < num_data; ++i) { +// printf("i %d\n", i); + offset = i * dim; + for (int j = 0; j < num_bottom_labels; ++j) { +// printf("\t j %d\n", j); + if (bottom_labels[offset + j] == 1) { + // Since the real rank is too costly to compute when the number of + // labels is large, bottom_labels j's rank is estimated based on + // the score margin violation which is defined as + // 1 - bottom_data[j] + bottom_data[a_negative_label] > 0. + num_trials = 0; + do { + do { // sample with replacement, TODO: without replacement + // TODO: more precise uniform random int generator + random_label = rand() + % (num_bottom_labels - 1/* num bottom_labels except j */); + if (random_label >= j) { + ++random_label; // shift one to skip j + } +// printf("\t random_label %d\n", random_label); + // sample until a negative bottom_labels is found + } while (bottom_labels[offset + random_label] == 1); + ++num_trials; +// printf("\t num_trials %d\n", num_trials); + score_margin = 1 - bottom_data[offset + j] + + bottom_data[offset + random_label]; + } while (score_margin <= 0 & num_trials < max_num_trials); + estimated_rank = floor(max_num_trials / num_trials); + rank_weight = rank_weights_data[estimated_rank]; +// LOG(ERROR)<< "rank_weight " << rank_weight; + for (int k = 0; k < num_bottom_labels; ++k) { +// printf("\t\t k %d\n", k); + if (bottom_labels[offset + k] == 0) { + score_margin = 1 - bottom_data[offset + j] + bottom_data[offset + k]; +// LOG(ERROR)<< "score margin " << score_margin << +// ", loss " << (loss + rank_weight * score_margin); + if (score_margin > 0) { + loss += rank_weight * score_margin; + bottom_diff[offset + j] -= rank_weight; + bottom_diff[offset + k] += rank_weight; + } + } + } // for (int k = 0; k < num_bottom_labels; ++k) { + } // if (bottom_labels[j] == 1) { + } // for (int j = 0; j < num_bottom_labels; ++j) { + } // for (int i = 0; i < num_data; ++i) { + caffe_scal(bottom[0]->count(), Dtype(1) / num_data, bottom_diff); + return loss / num_data; +} + +template +Dtype WARPLossLayer::Forward_gpu(const vector*>& bottom, + vector*>* top) { + return Forward_cpu(bottom, top); +} + +INSTANTIATE_CLASS(WARPLossLayer); + +} // namespace caffe diff --git a/src/caffe/test/test_warp_loss_layer.cpp b/src/caffe/test/test_warp_loss_layer.cpp new file mode 100644 index 00000000000..5d9e36e988a --- /dev/null +++ b/src/caffe/test/test_warp_loss_layer.cpp @@ -0,0 +1,83 @@ +// Copyright 2014 kloudkl@github + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/vision_layers.hpp" +#include "caffe/test/test_gradient_check_util.hpp" + +#include "caffe/test/test_caffe_main.hpp" + +namespace caffe { + +extern cudaDeviceProp CAFFE_TEST_CUDA_PROP; + +template +class WARPLossLayerTest : public ::testing::Test { + protected: + WARPLossLayerTest() + : blob_bottom_data_(new Blob(10, 10, 1, 1)), + blob_bottom_label_(new Blob(10, 10, 1, 1)) { + // fill the values + FillerParameter filler_param; + filler_param.set_std(10); + GaussianFiller filler(filler_param); + filler.Fill(this->blob_bottom_data_); + blob_bottom_vec_.push_back(blob_bottom_data_); + int dim = blob_bottom_label_->count() / blob_bottom_label_->num(); + Dtype* label_ptr = blob_bottom_label_->mutable_cpu_data(); + memset(label_ptr, 0, sizeof(Dtype) * blob_bottom_label_->count()); + int offset; + int num_trials = std::max(std::min(dim, 3), 1); + for (int i = 0; i < blob_bottom_label_->num(); ++i) { + offset = i * dim; + for (int j = 0; j < num_trials; ++j) { + label_ptr[offset + rand() % dim] = 1; + } + } + blob_bottom_vec_.push_back(blob_bottom_label_); + } + + virtual ~WARPLossLayerTest() { + delete blob_bottom_data_; + delete blob_bottom_label_; + } + + Blob* const blob_bottom_data_; + Blob* const blob_bottom_label_; + vector*> blob_bottom_vec_; + vector*> blob_top_vec_; +}; + +typedef ::testing::Types Dtypes; +TYPED_TEST_CASE(WARPLossLayerTest, Dtypes); + + +TYPED_TEST(WARPLossLayerTest, TestGradientCPU) { + LayerParameter layer_param; + Caffe::set_mode(Caffe::CPU); + WARPLossLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + GradientChecker checker(1e-2, 1e-2, 1701); + checker.CheckGradientSingle(&layer, &(this->blob_bottom_vec_), + &(this->blob_top_vec_), 0, -1, -1); +} + +TYPED_TEST(WARPLossLayerTest, TestGradientGPU) { + LayerParameter layer_param; + Caffe::set_mode(Caffe::GPU); + WARPLossLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + GradientChecker checker(1e-2, 1e-2, 1701); + checker.CheckGradientSingle(&layer, &(this->blob_bottom_vec_), + &(this->blob_top_vec_), 0, -1, -1); +} + +}