diff --git a/docs/mnist.md b/docs/mnist.md index c97f3cfe9e1..1e0f49f33f8 100644 --- a/docs/mnist.md +++ b/docs/mnist.md @@ -15,7 +15,7 @@ You will first need to download and convert the data format from the MNIST websi cd $CAFFE_ROOT/data/mnist ./get_mnist.sh - cd $CAFFE_ROOT/examples/lenet + cd $CAFFE_ROOT/examples/mnist ./create_mnist.sh If it complains that `wget` or `gunzip` are not installed, you need to install them respectively. After running the script there should be two datasets, `mnist-train-leveldb`, and `mnist-test-leveldb`. @@ -33,7 +33,7 @@ Training and Testing the Model Training the model is simple after you have written the network definition protobuf and solver protobuf files. Simply run `train_mnist.sh`, or the following command directly: - cd $CAFFE_ROOT/examples/lenet + cd $CAFFE_ROOT/examples/mnist ./train_lenet.sh `train_lenet.sh` is a simple script, but here are a few explanations: `GLOG_logtostderr=1` is the google logging flag that prints all the logging messages directly to stderr. The main tool for training is `train_net.bin`, with the solver protobuf text file as its argument. diff --git a/examples/lenet/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp similarity index 100% rename from examples/lenet/convert_mnist_data.cpp rename to examples/mnist/convert_mnist_data.cpp diff --git a/examples/lenet/create_mnist.sh b/examples/mnist/create_mnist.sh similarity index 100% rename from examples/lenet/create_mnist.sh rename to examples/mnist/create_mnist.sh diff --git a/examples/lenet/lenet.prototxt b/examples/mnist/lenet.prototxt similarity index 100% rename from examples/lenet/lenet.prototxt rename to examples/mnist/lenet.prototxt diff --git a/examples/lenet/lenet_solver.prototxt b/examples/mnist/lenet_solver.prototxt similarity index 100% rename from examples/lenet/lenet_solver.prototxt rename to examples/mnist/lenet_solver.prototxt diff --git a/examples/lenet/lenet_test.prototxt b/examples/mnist/lenet_test.prototxt similarity index 100% rename from examples/lenet/lenet_test.prototxt rename to examples/mnist/lenet_test.prototxt diff --git a/examples/lenet/lenet_train.prototxt b/examples/mnist/lenet_train.prototxt similarity index 100% rename from examples/lenet/lenet_train.prototxt rename to examples/mnist/lenet_train.prototxt diff --git a/examples/mnist/mnist_autoencoder_solver.prototxt b/examples/mnist/mnist_autoencoder_solver.prototxt new file mode 100644 index 00000000000..9b48c30eca6 --- /dev/null +++ b/examples/mnist/mnist_autoencoder_solver.prototxt @@ -0,0 +1,14 @@ +train_net: "mnist_autoencoder_train.prototxt" +test_net: "mnist_autoencoder_test.prototxt" +test_iter: 50 +test_interval: 100 +test_compute_loss: true +base_lr: 0.0001 +lr_policy: "fixed" +display: 20 +max_iter: 4000000 +weight_decay: 0.0005 +snapshot: 10000 +snapshot_prefix: "mnist_autoencoder_train" +momentum: 0.9 +solver_mode: 1 diff --git a/examples/mnist/mnist_autoencoder_test.prototxt b/examples/mnist/mnist_autoencoder_test.prototxt new file mode 100644 index 00000000000..5090e82fe0a --- /dev/null +++ b/examples/mnist/mnist_autoencoder_test.prototxt @@ -0,0 +1,145 @@ +name: "MNISTAutoencoder" +layers { + top: "data" + name: "data" + type: DATA + data_param { + source: "mnist-test-leveldb" + scale: 0.0039215684 + batch_size: 100 + } +} +layers { + bottom: "data" + top: "flatdata" + name: "flatdata" + type: FLATTEN +} +layers { + bottom: "data" + top: "encode1" + name: "encode1" + type: INNER_PRODUCT + inner_product_param { + num_output: 1000 + } +} +layers { + bottom: "encode1" + top: "encode1neuron" + name: "encode1neuron" + type: SIGMOID +} +layers { + bottom: "encode1neuron" + top: "encode2" + name: "encode2" + type: INNER_PRODUCT + inner_product_param { + num_output: 500 + } +} +layers { + bottom: "encode2" + top: "encode2neuron" + name: "encode2neuron" + type: SIGMOID +} +layers { + bottom: "encode2neuron" + top: "encode3" + name: "encode3" + type: INNER_PRODUCT + inner_product_param { + num_output: 250 + } +} +layers { + bottom: "encode3" + top: "encode3neuron" + name: "encode3neuron" + type: SIGMOID +} +layers { + bottom: "encode3neuron" + top: "encode4" + name: "encode4" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 30 + } +} +layers { + bottom: "encode4" + top: "decode4" + name: "decode4" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 250 + } +} +layers { + bottom: "decode4" + top: "decode4neuron" + name: "decode4neuron" + type: SIGMOID +} +layers { + bottom: "decode4neuron" + top: "decode3" + name: "decode3" + type: INNER_PRODUCT + inner_product_param { + num_output: 500 + } +} +layers { + bottom: "decode3" + top: "decode3neuron" + name: "decode3neuron" + type: SIGMOID +} +layers { + bottom: "decode3neuron" + top: "decode2" + name: "decode2" + type: INNER_PRODUCT + inner_product_param { + num_output: 1000 + } +} +layers { + bottom: "decode2" + top: "decode2neuron" + name: "decode2neuron" + type: SIGMOID +} +layers { + bottom: "decode2neuron" + top: "decode1" + name: "decode1" + type: INNER_PRODUCT + inner_product_param { + num_output: 784 + } +} +layers { + bottom: "decode1" + top: "decode1neuron" + name: "decode1neuron" + type: SIGMOID +} +layers { + bottom: "decode1neuron" + bottom: "flatdata" + name: "loss" + type: EUCLIDEAN_LOSS +} diff --git a/examples/mnist/mnist_autoencoder_train.prototxt b/examples/mnist/mnist_autoencoder_train.prototxt new file mode 100644 index 00000000000..90d2cff99b8 --- /dev/null +++ b/examples/mnist/mnist_autoencoder_train.prototxt @@ -0,0 +1,235 @@ +name: "MNISTAutoencoder" +layers { + top: "data" + name: "data" + type: DATA + data_param { + source: "mnist-train-leveldb" + scale: 0.0039215684 + batch_size: 100 + } +} +layers { + bottom: "data" + top: "flatdata" + name: "flatdata" + type: FLATTEN +} +layers { + bottom: "data" + top: "encode1" + name: "encode1" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 1000 + weight_filler { + type: "gaussian" + std: 1 + sparse: 15 + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layers { + bottom: "encode1" + top: "encode1neuron" + name: "encode1neuron" + type: SIGMOID +} +layers { + bottom: "encode1neuron" + top: "encode2" + name: "encode2" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 500 + weight_filler { + type: "gaussian" + std: 1 + sparse: 15 + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layers { + bottom: "encode2" + top: "encode2neuron" + name: "encode2neuron" + type: SIGMOID +} +layers { + bottom: "encode2neuron" + top: "encode3" + name: "encode3" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 250 + weight_filler { + type: "gaussian" + std: 1 + sparse: 15 + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layers { + bottom: "encode3" + top: "encode3neuron" + name: "encode3neuron" + type: SIGMOID +} +layers { + bottom: "encode3neuron" + top: "encode4" + name: "encode4" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 30 + weight_filler { + type: "gaussian" + std: 1 + sparse: 15 + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layers { + bottom: "encode4" + top: "decode4" + name: "decode4" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 250 + weight_filler { + type: "gaussian" + std: 1 + sparse: 15 + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layers { + bottom: "decode4" + top: "decode4neuron" + name: "decode4neuron" + type: SIGMOID +} +layers { + bottom: "decode4neuron" + top: "decode3" + name: "decode3" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 500 + weight_filler { + type: "gaussian" + std: 1 + sparse: 15 + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layers { + bottom: "decode3" + top: "decode3neuron" + name: "decode3neuron" + type: SIGMOID +} +layers { + bottom: "decode3neuron" + top: "decode2" + name: "decode2" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 1000 + weight_filler { + type: "gaussian" + std: 1 + sparse: 15 + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layers { + bottom: "decode2" + top: "decode2neuron" + name: "decode2neuron" + type: SIGMOID +} +layers { + bottom: "decode2neuron" + top: "decode1" + name: "decode1" + type: INNER_PRODUCT + blobs_lr: 1 + blobs_lr: 1 + weight_decay: 1 + weight_decay: 0 + inner_product_param { + num_output: 784 + weight_filler { + type: "gaussian" + std: 1 + sparse: 15 + } + bias_filler { + type: "constant" + value: 0 + } + } +} +layers { + bottom: "decode1" + bottom: "flatdata" + name: "loss" + type: SIGMOID_CROSS_ENTROPY_LOSS +} diff --git a/examples/lenet/train_lenet.sh b/examples/mnist/train_lenet.sh similarity index 100% rename from examples/lenet/train_lenet.sh rename to examples/mnist/train_lenet.sh diff --git a/examples/mnist/train_mnist_autoencoder.sh b/examples/mnist/train_mnist_autoencoder.sh new file mode 100755 index 00000000000..af2245e07f0 --- /dev/null +++ b/examples/mnist/train_mnist_autoencoder.sh @@ -0,0 +1,4 @@ +#!/bin/bash +TOOLS=../../build/tools + +GLOG_logtostderr=1 $TOOLS/train_net.bin mnist_autoencoder_solver.prototxt diff --git a/include/caffe/filler.hpp b/include/caffe/filler.hpp index 50a397e1ee1..242f11a3513 100644 --- a/include/caffe/filler.hpp +++ b/include/caffe/filler.hpp @@ -41,6 +41,8 @@ class ConstantFiller : public Filler { for (int i = 0; i < count; ++i) { data[i] = value; } + CHECK_EQ(this->filler_param_.sparse(), -1) + << "Sparsity not supported by this Filler."; } }; @@ -53,6 +55,8 @@ class UniformFiller : public Filler { CHECK(blob->count()); caffe_rng_uniform(blob->count(), Dtype(this->filler_param_.min()), Dtype(this->filler_param_.max()), blob->mutable_cpu_data()); + CHECK_EQ(this->filler_param_.sparse(), -1) + << "Sparsity not supported by this Filler."; } }; @@ -66,7 +70,28 @@ class GaussianFiller : public Filler { CHECK(blob->count()); caffe_rng_gaussian(blob->count(), Dtype(this->filler_param_.mean()), Dtype(this->filler_param_.std()), blob->mutable_cpu_data()); + int sparse = this->filler_param_.sparse(); + CHECK_GE(sparse, -1); + if (sparse >= 0) { + // Sparse initialization is implemented for "weight" blobs; i.e. matrices. + // These have num == channels == 1; height is number of inputs; width is + // number of outputs. The 'sparse' variable specifies the mean number + // of non-zero input weights for a given output. + CHECK_EQ(blob->num(), 1); + CHECK_EQ(blob->channels(), 1); + int num_inputs = blob->height(); + Dtype non_zero_probability = Dtype(sparse) / Dtype(num_inputs); + rand_vec_.reset(new SyncedMemory(blob->count() * sizeof(int))); + int* mask = reinterpret_cast(rand_vec_->mutable_cpu_data()); + caffe_rng_bernoulli(blob->count(), non_zero_probability, mask); + for (int i = 0; i < blob->count(); ++i) { + data[i] *= mask[i]; + } + } } + + protected: + shared_ptr rand_vec_; }; template @@ -91,6 +116,8 @@ class PositiveUnitballFiller : public Filler { data[i * dim + j] /= sum; } } + CHECK_EQ(this->filler_param_.sparse(), -1) + << "Sparsity not supported by this Filler."; } }; @@ -113,6 +140,8 @@ class XavierFiller : public Filler { Dtype scale = sqrt(Dtype(3) / fan_in); caffe_rng_uniform(blob->count(), -scale, scale, blob->mutable_cpu_data()); + CHECK_EQ(this->filler_param_.sparse(), -1) + << "Sparsity not supported by this Filler."; } }; diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp index 5af7b28df3c..b40f4c2958b 100644 --- a/include/caffe/vision_layers.hpp +++ b/include/caffe/vision_layers.hpp @@ -134,6 +134,34 @@ class SigmoidLayer : public NeuronLayer { const bool propagate_down, vector*>* bottom); }; +template +class SigmoidCrossEntropyLossLayer : public Layer { + public: + explicit SigmoidCrossEntropyLossLayer(const LayerParameter& param) + : Layer(param), + sigmoid_layer_(new SigmoidLayer(param)), + sigmoid_output_(new Blob()) {} + virtual void SetUp(const vector*>& bottom, + vector*>* top); + + protected: + virtual Dtype Forward_cpu(const vector*>& bottom, + vector*>* top); + virtual Dtype Forward_gpu(const vector*>& bottom, + vector*>* top); + virtual void Backward_cpu(const vector*>& top, + const bool propagate_down, vector*>* bottom); + virtual void Backward_gpu(const vector*>& top, + const bool propagate_down, vector*>* bottom); + + shared_ptr > sigmoid_layer_; + // sigmoid_output stores the output of the sigmoid layer. + shared_ptr > sigmoid_output_; + // Vector holders to call the underlying sigmoid layer forward and backward. + vector*> sigmoid_bottom_vec_; + vector*> sigmoid_top_vec_; +}; + template class TanHLayer : public NeuronLayer { public: @@ -268,6 +296,7 @@ class DataLayer : public Layer { shared_ptr > prefetch_data_; shared_ptr > prefetch_label_; Blob data_mean_; + bool output_labels_; }; template diff --git a/src/caffe/layer_factory.cpp b/src/caffe/layer_factory.cpp index d30ffeeb88f..cb457511191 100644 --- a/src/caffe/layer_factory.cpp +++ b/src/caffe/layer_factory.cpp @@ -64,6 +64,8 @@ Layer* GetLayer(const LayerParameter& param) { return new ReLULayer(param); case LayerParameter_LayerType_SIGMOID: return new SigmoidLayer(param); + case LayerParameter_LayerType_SIGMOID_CROSS_ENTROPY_LOSS: + return new SigmoidCrossEntropyLossLayer(param); case LayerParameter_LayerType_SOFTMAX: return new SoftmaxLayer(param); case LayerParameter_LayerType_SOFTMAX_LOSS: diff --git a/src/caffe/layers/data_layer.cpp b/src/caffe/layers/data_layer.cpp index 399f771fd68..8340259535a 100644 --- a/src/caffe/layers/data_layer.cpp +++ b/src/caffe/layers/data_layer.cpp @@ -9,6 +9,7 @@ #include "caffe/layer.hpp" #include "caffe/util/io.hpp" +#include "caffe/util/math_functions.hpp" #include "caffe/vision_layers.hpp" using std::string; @@ -23,7 +24,10 @@ void* DataLayerPrefetch(void* layer_pointer) { Datum datum; CHECK(layer->prefetch_data_); Dtype* top_data = layer->prefetch_data_->mutable_cpu_data(); - Dtype* top_label = layer->prefetch_label_->mutable_cpu_data(); + Dtype* top_label; + if (layer->output_labels_) { + top_label = layer->prefetch_label_->mutable_cpu_data(); + } const Dtype scale = layer->layer_param_.data_param().scale(); const int batch_size = layer->layer_param_.data_param().batch_size(); const int crop_size = layer->layer_param_.data_param().crop_size(); @@ -105,7 +109,9 @@ void* DataLayerPrefetch(void* layer_pointer) { } } - top_label[item_id] = datum.label(); + if (layer->output_labels_) { + top_label[item_id] = datum.label(); + } // go to the next iter layer->iter_->Next(); if (!layer->iter_->Valid()) { @@ -128,7 +134,13 @@ template void DataLayer::SetUp(const vector*>& bottom, vector*>* top) { CHECK_EQ(bottom.size(), 0) << "Data Layer takes no input blobs."; - CHECK_EQ(top->size(), 2) << "Data Layer takes two blobs as output."; + CHECK_GE(top->size(), 1) << "Data Layer takes at least one blob as output."; + CHECK_LE(top->size(), 2) << "Data Layer takes at most two blobs as output."; + if (top->size() == 1) { + output_labels_ = false; + } else { + output_labels_ = true; + } // Initialize the leveldb leveldb::DB* db_temp; leveldb::Options options; @@ -178,9 +190,11 @@ void DataLayer::SetUp(const vector*>& bottom, << (*top)[0]->channels() << "," << (*top)[0]->height() << "," << (*top)[0]->width(); // label - (*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1); - prefetch_label_.reset( - new Blob(this->layer_param_.data_param().batch_size(), 1, 1, 1)); + if (output_labels_) { + (*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1); + prefetch_label_.reset( + new Blob(this->layer_param_.data_param().batch_size(), 1, 1, 1)); + } // datum size datum_channels_ = datum.channels(); datum_height_ = datum.height(); @@ -208,7 +222,9 @@ void DataLayer::SetUp(const vector*>& bottom, // simultaneous cudaMalloc calls when the main thread is running. In some // GPUs this seems to cause failures if we do not so. prefetch_data_->mutable_cpu_data(); - prefetch_label_->mutable_cpu_data(); + if (output_labels_) { + prefetch_label_->mutable_cpu_data(); + } data_mean_.cpu_data(); DLOG(INFO) << "Initializing prefetch"; CHECK(!pthread_create(&thread_, NULL, DataLayerPrefetch, @@ -222,10 +238,12 @@ Dtype DataLayer::Forward_cpu(const vector*>& bottom, // First, join the thread CHECK(!pthread_join(thread_, NULL)) << "Pthread joining failed."; // Copy the data - memcpy((*top)[0]->mutable_cpu_data(), prefetch_data_->cpu_data(), - sizeof(Dtype) * prefetch_data_->count()); - memcpy((*top)[1]->mutable_cpu_data(), prefetch_label_->cpu_data(), - sizeof(Dtype) * prefetch_label_->count()); + caffe_copy(prefetch_data_->count(), prefetch_data_->cpu_data(), + (*top)[0]->mutable_cpu_data()); + if (output_labels_) { + caffe_copy(prefetch_label_->count(), prefetch_label_->cpu_data(), + (*top)[1]->mutable_cpu_data()); + } // Start a new prefetch thread CHECK(!pthread_create(&thread_, NULL, DataLayerPrefetch, reinterpret_cast(this))) << "Pthread execution failed."; diff --git a/src/caffe/layers/data_layer.cu b/src/caffe/layers/data_layer.cu index 86f4757885e..15ef016670b 100644 --- a/src/caffe/layers/data_layer.cu +++ b/src/caffe/layers/data_layer.cu @@ -24,9 +24,11 @@ Dtype DataLayer::Forward_gpu(const vector*>& bottom, CUDA_CHECK(cudaMemcpy((*top)[0]->mutable_gpu_data(), prefetch_data_->cpu_data(), sizeof(Dtype) * prefetch_data_->count(), cudaMemcpyHostToDevice)); - CUDA_CHECK(cudaMemcpy((*top)[1]->mutable_gpu_data(), - prefetch_label_->cpu_data(), sizeof(Dtype) * prefetch_label_->count(), - cudaMemcpyHostToDevice)); + if (output_labels_) { + CUDA_CHECK(cudaMemcpy((*top)[1]->mutable_gpu_data(), + prefetch_label_->cpu_data(), sizeof(Dtype) * prefetch_label_->count(), + cudaMemcpyHostToDevice)); + } // Start a new prefetch thread CHECK(!pthread_create(&thread_, NULL, DataLayerPrefetch, reinterpret_cast(this))) << "Pthread execution failed."; diff --git a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp new file mode 100644 index 00000000000..f2186d65c16 --- /dev/null +++ b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp @@ -0,0 +1,67 @@ +// Copyright 2014 BVLC and contributors. + +#include +#include +#include + +#include "caffe/layer.hpp" +#include "caffe/vision_layers.hpp" +#include "caffe/util/math_functions.hpp" + +using std::max; + +namespace caffe { + +template +void SigmoidCrossEntropyLossLayer::SetUp( + const vector*>& bottom, vector*>* top) { + CHECK_EQ(bottom.size(), 2) << + "SigmoidCrossEntropyLoss Layer takes two blobs as input."; + CHECK_EQ(top->size(), 0) << + "SigmoidCrossEntropyLoss Layer takes no blob as output."; + sigmoid_bottom_vec_.clear(); + sigmoid_bottom_vec_.push_back(bottom[0]); + sigmoid_top_vec_.clear(); + sigmoid_top_vec_.push_back(sigmoid_output_.get()); + sigmoid_layer_->SetUp(sigmoid_bottom_vec_, &sigmoid_top_vec_); +} + +template +Dtype SigmoidCrossEntropyLossLayer::Forward_cpu( + const vector*>& bottom, vector*>* top) { + // The forward pass computes the sigmoid outputs. + sigmoid_bottom_vec_[0] = bottom[0]; + sigmoid_layer_->Forward(sigmoid_bottom_vec_, &sigmoid_top_vec_); + // Compute the loss (negative log likelihood) + int count = bottom[0]->count(); + int num = bottom[0]->num(); + // Stable version of loss computation from input data + const Dtype* input_data = bottom[0]->cpu_data(); + const Dtype* ground_truth = bottom[1]->cpu_data(); + Dtype loss = 0; + for (int i = 0; i < count; ++i) { + loss -= input_data[i] * (ground_truth[i] - (input_data[i] >= 0)) - + log(1 + exp(input_data[i] - 2 * input_data[i] * (input_data[i] >= 0))); + } + return loss / num; +} + +template +void SigmoidCrossEntropyLossLayer::Backward_cpu( + const vector*>& top, const bool propagate_down, + vector*>* bottom) { + // First, compute the diff + int count = (*bottom)[0]->count(); + int num = (*bottom)[0]->num(); + const Dtype* sigmoid_output_data = sigmoid_output_->cpu_data(); + const Dtype* ground_truth = (*bottom)[1]->cpu_data(); + Dtype* bottom_diff = (*bottom)[0]->mutable_cpu_diff(); + caffe_sub(count, sigmoid_output_data, ground_truth, bottom_diff); + // Scale down gradient + caffe_scal(count, Dtype(1) / num, bottom_diff); +} + +INSTANTIATE_CLASS(SigmoidCrossEntropyLossLayer); + + +} // namespace caffe diff --git a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu new file mode 100644 index 00000000000..64bc476b00f --- /dev/null +++ b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu @@ -0,0 +1,54 @@ +// Copyright 2014 BVLC and contributors. + +#include +#include +#include + +#include "caffe/layer.hpp" +#include "caffe/vision_layers.hpp" +#include "caffe/util/math_functions.hpp" + +using std::max; + +namespace caffe { + +template +Dtype SigmoidCrossEntropyLossLayer::Forward_gpu( + const vector*>& bottom, vector*>* top) { + // The forward pass computes the sigmoid outputs. + sigmoid_bottom_vec_[0] = bottom[0]; + sigmoid_layer_->Forward(sigmoid_bottom_vec_, &sigmoid_top_vec_); + // Compute the loss (negative log likelihood) + int count = bottom[0]->count(); + int num = bottom[0]->num(); + // Stable version of loss computation from input data + const Dtype* input_data = bottom[0]->cpu_data(); + const Dtype* ground_truth = bottom[1]->cpu_data(); + Dtype loss = 0; + for (int i = 0; i < count; ++i) { + loss -= input_data[i] * (ground_truth[i] - (input_data[i] >= 0)) - + log(1 + exp(input_data[i] - 2 * input_data[i] * (input_data[i] >= 0))); + } + return loss / num; +} + +template +void SigmoidCrossEntropyLossLayer::Backward_gpu( + const vector*>& top, const bool propagate_down, + vector*>* bottom) { + // First, compute the diff + int count = (*bottom)[0]->count(); + int num = (*bottom)[0]->num(); + const Dtype* sigmoid_output_data = sigmoid_output_->gpu_data(); + const Dtype* ground_truth = (*bottom)[1]->gpu_data(); + Dtype* bottom_diff = (*bottom)[0]->mutable_gpu_diff(); + caffe_gpu_copy(count, sigmoid_output_data, bottom_diff); + caffe_gpu_axpy(count, Dtype(-1), ground_truth, bottom_diff); + // Scale down gradient + caffe_gpu_scal(count, Dtype(1) / num, bottom_diff); +} + +INSTANTIATE_CLASS(SigmoidCrossEntropyLossLayer); + + +} // namespace caffe diff --git a/src/caffe/proto/caffe.proto b/src/caffe/proto/caffe.proto index da7824c8e27..284b1e29f58 100644 --- a/src/caffe/proto/caffe.proto +++ b/src/caffe/proto/caffe.proto @@ -34,8 +34,11 @@ message FillerParameter { optional float value = 2 [default = 0]; // the value in constant filler optional float min = 3 [default = 0]; // the min value in uniform filler optional float max = 4 [default = 1]; // the max value in uniform filler - optional float mean = 5 [default = 0]; // the mean value in gaussian filler - optional float std = 6 [default = 1]; // the std value in gaussian filler + optional float mean = 5 [default = 0]; // the mean value in Gaussian filler + optional float std = 6 [default = 1]; // the std value in Gaussian filler + // The expected number of non-zero input weights for a given output in + // Gaussian filler -- the default -1 means don't perform sparsification. + optional int32 sparse = 7 [default = -1]; } message NetParameter { @@ -60,6 +63,7 @@ message SolverParameter { optional int32 test_iter = 3 [default = 0]; // The number of iterations between two testing phases. optional int32 test_interval = 4 [default = 0]; + optional bool test_compute_loss = 19 [default = false]; optional float base_lr = 5; // The base learning rate // the number of iterations between displaying info. If display = 0, no info // will be displayed. @@ -102,7 +106,7 @@ message LayerParameter { // line above the enum. Update the next available ID when you add a new // LayerType. // - // LayerType next available ID: 27 + // LayerType next available ID: 28 enum LayerType { // "NONE" layer type is 0th enum element so that we don't cause confusion // by defaulting to an existent LayerType (instead, should usually error if @@ -129,6 +133,7 @@ message LayerParameter { POWER = 26; RELU = 18; SIGMOID = 19; + SIGMOID_CROSS_ENTROPY_LOSS = 27; SOFTMAX = 20; SOFTMAX_LOSS = 21; SPLIT = 22; diff --git a/src/caffe/solver.cpp b/src/caffe/solver.cpp index ef5123cfe40..8daedf4af22 100644 --- a/src/caffe/solver.cpp +++ b/src/caffe/solver.cpp @@ -100,9 +100,14 @@ void Solver::Test() { CHECK_NOTNULL(test_net_.get())->CopyTrainedLayersFrom(net_param); vector test_score; vector*> bottom_vec; + Dtype loss = 0; for (int i = 0; i < param_.test_iter(); ++i) { + Dtype iter_loss; const vector*>& result = - test_net_->Forward(bottom_vec); + test_net_->Forward(bottom_vec, &iter_loss); + if (param_.test_compute_loss()) { + loss += iter_loss; + } if (i == 0) { for (int j = 0; j < result.size(); ++j) { const Dtype* result_vec = result[j]->cpu_data(); @@ -120,6 +125,10 @@ void Solver::Test() { } } } + if (param_.test_compute_loss()) { + loss /= param_.test_iter(); + LOG(INFO) << "Test loss: " << loss; + } for (int i = 0; i < test_score.size(); ++i) { LOG(INFO) << "Test score #" << i << ": " << test_score[i] / param_.test_iter(); diff --git a/src/caffe/test/test_sigmoid_cross_entropy_loss_layer.cpp b/src/caffe/test/test_sigmoid_cross_entropy_loss_layer.cpp new file mode 100644 index 00000000000..fe899d43d53 --- /dev/null +++ b/src/caffe/test/test_sigmoid_cross_entropy_loss_layer.cpp @@ -0,0 +1,76 @@ +// Copyright 2014 BVLC and contributors. + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/vision_layers.hpp" +#include "caffe/test/test_gradient_check_util.hpp" + +#include "caffe/test/test_caffe_main.hpp" + +namespace caffe { + +extern cudaDeviceProp CAFFE_TEST_CUDA_PROP; + +template +class SigmoidCrossEntropyLossLayerTest : public ::testing::Test { + protected: + SigmoidCrossEntropyLossLayerTest() + : blob_bottom_data_(new Blob(10, 5, 1, 1)), + blob_bottom_targets_(new Blob(10, 5, 1, 1)) { + // Fill the data vector + FillerParameter data_filler_param; + data_filler_param.set_std(10); + GaussianFiller data_filler(data_filler_param); + data_filler.Fill(blob_bottom_data_); + blob_bottom_vec_.push_back(blob_bottom_data_); + // Fill the targets vector + FillerParameter targets_filler_param; + targets_filler_param.set_min(0.0); + targets_filler_param.set_max(1.0); + UniformFiller targets_filler(targets_filler_param); + targets_filler.Fill(blob_bottom_targets_); + blob_bottom_vec_.push_back(blob_bottom_targets_); + } + virtual ~SigmoidCrossEntropyLossLayerTest() { + delete blob_bottom_data_; + delete blob_bottom_targets_; + } + Blob* const blob_bottom_data_; + Blob* const blob_bottom_targets_; + vector*> blob_bottom_vec_; + vector*> blob_top_vec_; +}; + +typedef ::testing::Types Dtypes; +TYPED_TEST_CASE(SigmoidCrossEntropyLossLayerTest, Dtypes); + + +TYPED_TEST(SigmoidCrossEntropyLossLayerTest, TestGradientCPU) { + LayerParameter layer_param; + Caffe::set_mode(Caffe::CPU); + SigmoidCrossEntropyLossLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + GradientChecker checker(1e-2, 1e-2, 1701); + checker.CheckGradientSingle(&layer, &(this->blob_bottom_vec_), + &(this->blob_top_vec_), 0, -1, -1); +} + +TYPED_TEST(SigmoidCrossEntropyLossLayerTest, TestGradientGPU) { + LayerParameter layer_param; + Caffe::set_mode(Caffe::GPU); + SigmoidCrossEntropyLossLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + GradientChecker checker(1e-2, 1e-2, 1701); + checker.CheckGradientSingle(&layer, &(this->blob_bottom_vec_), + &(this->blob_top_vec_), 0, -1, -1); +} + + +} // namespace caffe