Implement WARPLossLayer

include/caffe/vision_layers.hpp

@@ -686,6 +686,39 @@ class SplitLayer : public Layer<Dtype> {
   int count_;
 };
 
+/*
+ * Weighted Approximate-Rank Pairwise loss for image retrieval/annotation etc.
+ * References:
+ * [1] Yunchao Gong, Yangqing Jia, Sergey Ioffe, Alexander Toshev, Thomas
+ *     Leung. Deep Convolutional Ranking for Multilabel Image Annotation.
+ *     arXiv:1312.4894 [cs.CV]
+ * [2] Jason Weston, Samy Bengio, and Nicolas Usunier. Wsabie: Scaling up
+ *     to large vocabulary image annotation. In IJCAI, 2011.
+ */
+template <typename Dtype>
+class WARPLossLayer : public Layer<Dtype> {
+ public:
+  explicit WARPLossLayer(const LayerParameter& param)
+      : Layer<Dtype>(param) {}
+  virtual void SetUp(const vector<Blob<Dtype>*>& bottom,
+      vector<Blob<Dtype>*>* top);
+
+ protected:
+  virtual Dtype Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+      vector<Blob<Dtype>*>* top);
+  virtual Dtype Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+      vector<Blob<Dtype>*>* top);
+  virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+      const bool propagate_down, vector<Blob<Dtype>*>* bottom) { return; }
+  virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+      const bool propagate_down, vector<Blob<Dtype>*>* bottom) { return; }
+  // In image retrieval or annotations, results or labels form ranked list
+  //   based on their scores.
+  // Weighted Approximate-Rank Pairwise loss uses a function to
+  //   set weight for different ranks, e.g. 1 / j for the j-th rank.
+  Blob<Dtype> rank_weights_;
+};
+
 // This function is used to create a pthread that prefetches the window data.
 template <typename Dtype>
 void* WindowDataLayerPrefetch(void* layer_pointer);

src/caffe/layers/warp_loss_layer.cpp

@@ -0,0 +1,105 @@
+// Copyright 2014 kloudkl@github
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/vision_layers.hpp"
+#include "caffe/util/math_functions.hpp"
+
+namespace caffe {
+
+template<typename Dtype>
+void WARPLossLayer<Dtype>::SetUp(const vector<Blob<Dtype>*>& bottom,
+                                 vector<Blob<Dtype>*>* top) {
+  CHECK_EQ(bottom.size(), 2)<<
+  "WARPLossLayer takes two blobs as input.";
+  CHECK_EQ(top->size(), 0) << "WARPLossLayer takes no output.";
+  CHECK_EQ(bottom[0]->num(), bottom[1]->num()) <<
+      "The two input blobs should have the same number.";
+  int dim = bottom[0]->count() / bottom[0]->num();
+  rank_weights_.Reshape(1, dim, 1, 1);
+  Dtype* rank_weights_data = rank_weights_.mutable_cpu_data();
+  rank_weights_data[0] = 1. / (1 + 0);
+  for (int i = 1; i < dim; ++i) {
+    rank_weights_data[i] = rank_weights_data[i - 1] + 1. / (1 + i);
+  }
+};
+
+template<typename Dtype>
+Dtype WARPLossLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+                                       vector<Blob<Dtype>*>* top) {
+  const Dtype* bottom_data = bottom[0]->cpu_data();
+  const Dtype* bottom_labels = bottom[1]->cpu_data();
+  const Dtype* rank_weights_data = rank_weights_.cpu_data();
+  const int num_data = bottom[0]->num();
+  const int dim = bottom[0]->count() / bottom[0]->num();
+  const int num_bottom_labels = dim;
+  const int max_num_trials = num_bottom_labels - 1;
+  Dtype* bottom_diff = bottom[0]->mutable_cpu_diff();
+  memset(bottom_diff, 0, sizeof(Dtype) * bottom[0]->count());
+  Dtype loss = 0;
+  Dtype score_margin;
+  int random_label;
+  int estimated_rank;
+  int num_trials;
+  int offset;
+  Dtype rank_weight;
+  for (int i = 0; i < num_data; ++i) {
+//    printf("i %d\n", i);
+    offset = i * dim;
+    for (int j = 0; j < num_bottom_labels; ++j) {
+//      printf("\t j %d\n", j);
+      if (bottom_labels[offset + j] == 1) {
+        // Since the real rank is too costly to compute when the number of
+        //   labels is large, bottom_labels j's rank is estimated based on
+        //   the score margin violation which is defined as
+        //   1 - bottom_data[j] + bottom_data[a_negative_label] > 0.
+        num_trials = 0;
+        do {
+          do { // sample with replacement, TODO: without replacement
+            // TODO: more precise uniform random int generator
+            random_label = rand()
+                % (num_bottom_labels - 1/* num bottom_labels except j */);
+            if (random_label >= j) {
+              ++random_label;  // shift one to skip j
+            }
+//            printf("\t random_label %d\n", random_label);
+            // sample until a negative bottom_labels is found
+          } while (bottom_labels[offset + random_label] == 1);
+          ++num_trials;
+//          printf("\t num_trials %d\n", num_trials);
+          score_margin = 1 - bottom_data[offset + j] +
+              bottom_data[offset + random_label];
+        } while (score_margin <= 0 & num_trials < max_num_trials);
+        estimated_rank = floor(max_num_trials / num_trials);
+        rank_weight = rank_weights_data[estimated_rank];
+//        LOG(ERROR)<< "rank_weight " << rank_weight;
+        for (int k = 0; k < num_bottom_labels; ++k) {
+//          printf("\t\t k %d\n", k);
+          if (bottom_labels[offset + k] == 0) {
+            score_margin = 1 - bottom_data[offset + j] + bottom_data[offset + k];
+//            LOG(ERROR)<< "score margin " << score_margin <<
+//                ", loss " << (loss + rank_weight * score_margin);
+            if (score_margin > 0) {
+              loss += rank_weight * score_margin;
+              bottom_diff[offset + j] -= rank_weight;
+              bottom_diff[offset + k] += rank_weight;
+            }
+          }
+        }  // for (int k = 0; k < num_bottom_labels; ++k) {
+      }  // if (bottom_labels[j] == 1) {
+    }  // for (int j = 0; j < num_bottom_labels; ++j) {
+  }  //  for (int i = 0; i < num_data; ++i) {
+  caffe_scal(bottom[0]->count(), Dtype(1) / num_data, bottom_diff);
+  return loss / num_data;
+}
+
+template<typename Dtype>
+Dtype WARPLossLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+                                       vector<Blob<Dtype>*>* top) {
+  return Forward_cpu(bottom, top);
+}
+
+INSTANTIATE_CLASS(WARPLossLayer);
+
+}  // namespace caffe

src/caffe/test/test_warp_loss_layer.cpp

@@ -0,0 +1,83 @@
+// Copyright 2014 kloudkl@github
+
+#include <algorithm>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include <cuda_runtime.h>
+
+#include "gtest/gtest.h"
+#include "caffe/blob.hpp"
+#include "caffe/common.hpp"
+#include "caffe/filler.hpp"
+#include "caffe/vision_layers.hpp"
+#include "caffe/test/test_gradient_check_util.hpp"
+
+#include "caffe/test/test_caffe_main.hpp"
+
+namespace caffe {
+
+extern cudaDeviceProp CAFFE_TEST_CUDA_PROP;
+
+template <typename Dtype>
+class WARPLossLayerTest : public ::testing::Test {
+ protected:
+  WARPLossLayerTest()
+      : blob_bottom_data_(new Blob<Dtype>(10, 10, 1, 1)),
+        blob_bottom_label_(new Blob<Dtype>(10, 10, 1, 1)) {
+    // fill the values
+    FillerParameter filler_param;
+    filler_param.set_std(10);
+    GaussianFiller<Dtype> filler(filler_param);
+    filler.Fill(this->blob_bottom_data_);
+    blob_bottom_vec_.push_back(blob_bottom_data_);
+    int dim = blob_bottom_label_->count() / blob_bottom_label_->num();
+    Dtype* label_ptr = blob_bottom_label_->mutable_cpu_data();
+    memset(label_ptr, 0, sizeof(Dtype) * blob_bottom_label_->count());
+    int offset;
+    int num_trials = std::max(std::min(dim, 3), 1);
+    for (int i = 0; i < blob_bottom_label_->num(); ++i) {
+      offset = i * dim;
+      for (int j = 0; j < num_trials; ++j) {
+        label_ptr[offset + rand() % dim] = 1;
+      }
+    }
+    blob_bottom_vec_.push_back(blob_bottom_label_);
+  }
+
+  virtual ~WARPLossLayerTest() {
+    delete blob_bottom_data_;
+    delete blob_bottom_label_;
+  }
+
+  Blob<Dtype>* const blob_bottom_data_;
+  Blob<Dtype>* const blob_bottom_label_;
+  vector<Blob<Dtype>*> blob_bottom_vec_;
+  vector<Blob<Dtype>*> blob_top_vec_;
+};
+
+typedef ::testing::Types<float, double> Dtypes;
+TYPED_TEST_CASE(WARPLossLayerTest, Dtypes);
+
+
+TYPED_TEST(WARPLossLayerTest, TestGradientCPU) {
+  LayerParameter layer_param;
+  Caffe::set_mode(Caffe::CPU);
+  WARPLossLayer<TypeParam> layer(layer_param);
+  layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_);
+  GradientChecker<TypeParam> checker(1e-2, 1e-2, 1701);
+  checker.CheckGradientSingle(&layer, &(this->blob_bottom_vec_),
+      &(this->blob_top_vec_), 0, -1, -1);
+}
+
+TYPED_TEST(WARPLossLayerTest, TestGradientGPU) {
+  LayerParameter layer_param;
+  Caffe::set_mode(Caffe::GPU);
+  WARPLossLayer<TypeParam> layer(layer_param);
+  layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_);
+  GradientChecker<TypeParam> checker(1e-2, 1e-2, 1701);
+  checker.CheckGradientSingle(&layer, &(this->blob_bottom_vec_),
+      &(this->blob_top_vec_), 0, -1, -1);
+}
+
+}