From 12f6fd52a8cd5e4b5972f3bf78bb526403c05f77 Mon Sep 17 00:00:00 2001 From: linmin Date: Thu, 22 May 2014 00:15:32 +0800 Subject: [PATCH 1/9] add option for lmdb --- Makefile | 1 + include/caffe/data_layers.hpp | 10 +++ src/caffe/layers/data_layer.cpp | 142 ++++++++++++++++++++++++++------ src/caffe/proto/caffe.proto | 5 ++ 4 files changed, 132 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index c43b30ee33a..943165a4642 100644 --- a/Makefile +++ b/Makefile @@ -129,6 +129,7 @@ LIBRARY_DIRS += $(CUDA_LIB_DIR) LIBRARIES := cudart cublas curand \ pthread \ glog protobuf leveldb snappy \ + lmdb \ boost_system \ hdf5_hl hdf5 \ opencv_core opencv_highgui opencv_imgproc diff --git a/include/caffe/data_layers.hpp b/include/caffe/data_layers.hpp index 3d3ce94bc88..2b4c278eafe 100644 --- a/include/caffe/data_layers.hpp +++ b/include/caffe/data_layers.hpp @@ -8,6 +8,7 @@ #include #include "leveldb/db.h" +#include "lmdb.h" #include "pthread.h" #include "hdf5.h" #include "boost/scoped_ptr.hpp" @@ -133,8 +134,17 @@ class DataLayer : public Layer { virtual unsigned int PrefetchRand(); shared_ptr prefetch_rng_; + + // LEVELDB shared_ptr db_; shared_ptr iter_; + // LMDB + MDB_env* mdb_env_; + MDB_dbi mdb_dbi_; + MDB_txn* mdb_txn_; + MDB_cursor* mdb_cursor_; + MDB_val mdb_key_, mdb_value_; + int datum_channels_; int datum_height_; int datum_width_; diff --git a/src/caffe/layers/data_layer.cpp b/src/caffe/layers/data_layer.cpp index 348753e5db9..3b307ec7743 100644 --- a/src/caffe/layers/data_layer.cpp +++ b/src/caffe/layers/data_layer.cpp @@ -12,6 +12,7 @@ #include "caffe/util/math_functions.hpp" #include "caffe/util/rng.hpp" #include "caffe/vision_layers.hpp" +#include "caffe/proto/caffe.pb.h" using std::string; @@ -46,9 +47,22 @@ void* DataLayerPrefetch(void* layer_pointer) { const Dtype* mean = layer->data_mean_.cpu_data(); for (int item_id = 0; item_id < batch_size; ++item_id) { // get a blob - CHECK(layer->iter_); - CHECK(layer->iter_->Valid()); - datum.ParseFromString(layer->iter_->value().ToString()); + switch (layer->layer_param_.data_param().backend()) { + case DataParameter_DB_LEVELDB: + CHECK(layer->iter_); + CHECK(layer->iter_->Valid()); + datum.ParseFromString(layer->iter_->value().ToString()); + break; + case DataParameter_DB_LMDB: + CHECK_EQ(mdb_cursor_get(layer->mdb_cursor_, &layer->mdb_key_, + &layer->mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS); + datum.ParseFromArray(layer->mdb_value_.mv_data, + layer->mdb_value_.mv_size); + break; + default: + LOG(FATAL) << "Unknown database backend"; + } + const string& data = datum.data(); if (crop_size) { CHECK(data.size()) << "Image cropping only support uint8 data"; @@ -110,11 +124,26 @@ void* DataLayerPrefetch(void* layer_pointer) { top_label[item_id] = datum.label(); } // go to the next iter - layer->iter_->Next(); - if (!layer->iter_->Valid()) { - // We have reached the end. Restart from the first. - DLOG(INFO) << "Restarting data prefetching from start."; - layer->iter_->SeekToFirst(); + switch (layer->layer_param_.data_param().backend()) { + case DataParameter_DB_LEVELDB: + layer->iter_->Next(); + if (!layer->iter_->Valid()) { + // We have reached the end. Restart from the first. + DLOG(INFO) << "Restarting data prefetching from start."; + layer->iter_->SeekToFirst(); + } + break; + case DataParameter_DB_LMDB: + if (mdb_cursor_get(layer->mdb_cursor_, &layer->mdb_key_, + &layer->mdb_value_, MDB_NEXT) != MDB_SUCCESS) { + // We have reached the end. Restart from the first. + DLOG(INFO) << "Restarting data prefetching from start."; + CHECK_EQ(mdb_cursor_get(layer->mdb_cursor_, &layer->mdb_key_, + &layer->mdb_value_, MDB_FIRST), MDB_SUCCESS); + } + break; + default: + LOG(FATAL) << "Unknown database backend"; } } @@ -124,6 +153,19 @@ void* DataLayerPrefetch(void* layer_pointer) { template DataLayer::~DataLayer() { JoinPrefetchThread(); + // clean up the database resources + switch (this->layer_param_.data_param().backend()) { + case DataParameter_DB_LEVELDB: + break; // do nothing + case DataParameter_DB_LMDB: + mdb_cursor_close(mdb_cursor_); + mdb_close(mdb_env_, mdb_dbi_); + mdb_txn_abort(mdb_txn_); + mdb_env_close(mdb_env_); + break; + default: + LOG(FATAL) << "Unknown database backend"; + } } template @@ -135,35 +177,83 @@ void DataLayer::SetUp(const vector*>& bottom, } else { output_labels_ = true; } - // Initialize the leveldb - leveldb::DB* db_temp; - leveldb::Options options; - options.create_if_missing = false; - options.max_open_files = 100; - LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source(); - leveldb::Status status = leveldb::DB::Open( - options, this->layer_param_.data_param().source(), &db_temp); - CHECK(status.ok()) << "Failed to open leveldb " - << this->layer_param_.data_param().source() << std::endl - << status.ToString(); - db_.reset(db_temp); - iter_.reset(db_->NewIterator(leveldb::ReadOptions())); - iter_->SeekToFirst(); + // Initialize DB + switch (this->layer_param_.data_param().backend()) { + case DataParameter_DB_LEVELDB: + { + leveldb::DB* db_temp; + leveldb::Options options; + options.create_if_missing = false; + options.max_open_files = 100; + LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source(); + leveldb::Status status = leveldb::DB::Open( + options, this->layer_param_.data_param().source(), &db_temp); + CHECK(status.ok()) << "Failed to open leveldb " + << this->layer_param_.data_param().source() << std::endl + << status.ToString(); + db_.reset(db_temp); + iter_.reset(db_->NewIterator(leveldb::ReadOptions())); + iter_->SeekToFirst(); + } + break; + case DataParameter_DB_LMDB: + CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed"; + CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB + CHECK_EQ(mdb_env_open(mdb_env_, + this->layer_param_.data_param().source().c_str(), + MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed"; + CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS) + << "mdb_txn_begin failed"; + CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS) + << "mdb_open failed"; + CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS) + << "mdb_cursor_open failed"; + LOG(INFO) << "Opening lmdb " << this->layer_param_.data_param().source(); + CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST), + MDB_SUCCESS) << "mdb_cursor_get failed"; + break; + default: + LOG(FATAL) << "Unknown database backend"; + } + // Check if we would need to randomly skip a few data points if (this->layer_param_.data_param().rand_skip()) { unsigned int skip = caffe_rng_rand() % this->layer_param_.data_param().rand_skip(); LOG(INFO) << "Skipping first " << skip << " data points."; while (skip-- > 0) { - iter_->Next(); - if (!iter_->Valid()) { - iter_->SeekToFirst(); + switch (this->layer_param_.data_param().backend()) { + case DataParameter_DB_LEVELDB: + iter_->Next(); + if (!iter_->Valid()) { + iter_->SeekToFirst(); + } + break; + case DataParameter_DB_LMDB: + if(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT) + != MDB_SUCCESS) { + CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, + MDB_FIRST), MDB_SUCCESS); + } + break; + default: + LOG(FATAL) << "Unknown database backend"; } } } // Read a data point, and use it to initialize the top blob. Datum datum; - datum.ParseFromString(iter_->value().ToString()); + switch (this->layer_param_.data_param().backend()) { + case DataParameter_DB_LEVELDB: + datum.ParseFromString(iter_->value().ToString()); + break; + case DataParameter_DB_LMDB: + datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size); + break; + default: + LOG(FATAL) << "Unknown database backend"; + } + // image int crop_size = this->layer_param_.data_param().crop_size(); if (crop_size > 0) { diff --git a/src/caffe/proto/caffe.proto b/src/caffe/proto/caffe.proto index 60c7daa9031..b85b49c77f7 100644 --- a/src/caffe/proto/caffe.proto +++ b/src/caffe/proto/caffe.proto @@ -229,6 +229,10 @@ message ConvolutionParameter { // Message that stores parameters used by DataLayer message DataParameter { + enum DB { + LEVELDB = 0; + LMDB = 1; + } // Specify the data source. optional string source = 1; // For data pre-processing, we can do simple scaling and subtracting the @@ -247,6 +251,7 @@ message DataParameter { // point would be set as rand_skip * rand(0,1). Note that rand_skip should not // be larger than the number of keys in the leveldb. optional uint32 rand_skip = 7 [default = 0]; + optional DB backend = 8 [default = LEVELDB]; } // Message that stores parameters used by DropoutLayer From f43f555dc03a8bf616c21f9ae6225012bad3ad1b Mon Sep 17 00:00:00 2001 From: linmin Date: Fri, 13 Jun 2014 10:19:33 +0800 Subject: [PATCH 2/9] add tests for lmdb of datalayer (copied from test_data_layer.cpp) --- src/caffe/test/test_data_layer_lmdb.cpp | 565 ++++++++++++++++++++++++ 1 file changed, 565 insertions(+) create mode 100644 src/caffe/test/test_data_layer_lmdb.cpp diff --git a/src/caffe/test/test_data_layer_lmdb.cpp b/src/caffe/test/test_data_layer_lmdb.cpp new file mode 100644 index 00000000000..9b5b7d2bfc6 --- /dev/null +++ b/src/caffe/test/test_data_layer_lmdb.cpp @@ -0,0 +1,565 @@ +// Copyright 2014 BVLC and contributors. + +#include +#include +#include + +#include "cuda_runtime.h" +#include "leveldb/db.h" +#include "lmdb.h" +#include "gtest/gtest.h" +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/vision_layers.hpp" +#include "caffe/proto/caffe.pb.h" +#include "caffe/test/test_caffe_main.hpp" + +using std::string; +using std::stringstream; + +namespace caffe { + +extern cudaDeviceProp CAFFE_TEST_CUDA_PROP; + +template +class DataLayerTestLmdb : public ::testing::Test { + protected: + DataLayerTestLmdb() + : blob_top_data_(new Blob()), + blob_top_label_(new Blob()), + filename_(new string(tmpnam(NULL))), + seed_(1701) {} + virtual void SetUp() { + blob_top_vec_.push_back(blob_top_data_); + blob_top_vec_.push_back(blob_top_label_); + } + + // Fill the LMDB with data: if unique_pixels, each pixel is unique but + // all images are the same; else each image is unique but all pixels within + // an image are the same. + + void FillLMDB(const bool unique_pixels) { + LOG(INFO) << "Using temporary lmdb " << *filename_; + CHECK_EQ(mkdir(filename_->c_str(), 0744), 0) << "mkdir " << filename_ + << "failed"; + MDB_env *env; + MDB_dbi dbi; + MDB_val mdbkey, mdbdata; + MDB_txn *txn; + CHECK_EQ(mdb_env_create(&env), MDB_SUCCESS) << "mdb_env_create failed"; + CHECK_EQ(mdb_env_set_mapsize(env, 1099511627776), MDB_SUCCESS) // 1TB + << "mdb_env_set_mapsize failed"; + CHECK_EQ(mdb_env_open(env, filename_->c_str(), 0, 0664), MDB_SUCCESS) + << "mdb_env_open failed"; + CHECK_EQ(mdb_txn_begin(env, NULL, 0, &txn), MDB_SUCCESS) + << "mdb_txn_begin failed"; + CHECK_EQ(mdb_open(txn, NULL, 0, &dbi), MDB_SUCCESS) << "mdb_open failed"; + + for (int i = 0; i < 5; ++i) { + Datum datum; + datum.set_label(i); + datum.set_channels(2); + datum.set_height(3); + datum.set_width(4); + std::string* data = datum.mutable_data(); + for (int j = 0; j < 24; ++j) { + int datum = unique_pixels ? j : i; + data->push_back(static_cast(datum)); + } + stringstream ss; + ss << i; + + string value; + datum.SerializeToString(&value); + mdbdata.mv_size = value.size(); + mdbdata.mv_data = reinterpret_cast(&value[0]); + string keystr = ss.str(); + mdbkey.mv_size = keystr.size(); + mdbkey.mv_data = reinterpret_cast(&keystr[0]); + CHECK_EQ(mdb_put(txn, dbi, &mdbkey, &mdbdata, 0), MDB_SUCCESS) + << "mdb_put failed"; + } + CHECK_EQ(mdb_txn_commit(txn), MDB_SUCCESS) << "mdb_txn_commit failed"; + mdb_close(env, dbi); + mdb_env_close(env); + } + + virtual ~DataLayerTestLmdb() { delete blob_top_data_; delete blob_top_label_; } + + shared_ptr filename_; + Blob* const blob_top_data_; + Blob* const blob_top_label_; + vector*> blob_bottom_vec_; + vector*> blob_top_vec_; + int seed_; +}; + +typedef ::testing::Types Dtypes; +TYPED_TEST_CASE(DataLayerTestLmdb, Dtypes); + +TYPED_TEST(DataLayerTestLmdb, TestReadCPU) { + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = false; // all pixels the same; images different + this->FillLMDB(unique_pixels); + const TypeParam scale = 3; + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_scale(scale); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + DataLayer layer(param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + EXPECT_EQ(this->blob_top_data_->num(), 5); + EXPECT_EQ(this->blob_top_data_->channels(), 2); + EXPECT_EQ(this->blob_top_data_->height(), 3); + EXPECT_EQ(this->blob_top_data_->width(), 4); + EXPECT_EQ(this->blob_top_label_->num(), 5); + EXPECT_EQ(this->blob_top_label_->channels(), 1); + EXPECT_EQ(this->blob_top_label_->height(), 1); + EXPECT_EQ(this->blob_top_label_->width(), 1); + + for (int iter = 0; iter < 100; ++iter) { + layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 24; ++j) { + EXPECT_EQ(scale * i, this->blob_top_data_->cpu_data()[i * 24 + j]) + << "debug: iter " << iter << " i " << i << " j " << j; + } + } + } +} + +TYPED_TEST(DataLayerTestLmdb, TestReadGPU) { + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = false; // all pixels the same; images different + this->FillLMDB(unique_pixels); + const TypeParam scale = 3; + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_scale(scale); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + DataLayer layer(param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + EXPECT_EQ(this->blob_top_data_->num(), 5); + EXPECT_EQ(this->blob_top_data_->channels(), 2); + EXPECT_EQ(this->blob_top_data_->height(), 3); + EXPECT_EQ(this->blob_top_data_->width(), 4); + EXPECT_EQ(this->blob_top_label_->num(), 5); + EXPECT_EQ(this->blob_top_label_->channels(), 1); + EXPECT_EQ(this->blob_top_label_->height(), 1); + EXPECT_EQ(this->blob_top_label_->width(), 1); + + for (int iter = 0; iter < 100; ++iter) { + layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 24; ++j) { + EXPECT_EQ(scale * i, this->blob_top_data_->cpu_data()[i * 24 + j]) + << "debug: iter " << iter << " i " << i << " j " << j; + } + } + } +} + +TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainCPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + const TypeParam scale = 3; + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_scale(scale); + data_param->set_crop_size(1); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + DataLayer layer(param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + EXPECT_EQ(this->blob_top_data_->num(), 5); + EXPECT_EQ(this->blob_top_data_->channels(), 2); + EXPECT_EQ(this->blob_top_data_->height(), 1); + EXPECT_EQ(this->blob_top_data_->width(), 1); + EXPECT_EQ(this->blob_top_label_->num(), 5); + EXPECT_EQ(this->blob_top_label_->channels(), 1); + EXPECT_EQ(this->blob_top_label_->height(), 1); + EXPECT_EQ(this->blob_top_label_->width(), 1); + + for (int iter = 0; iter < 2; ++iter) { + layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + int num_with_center_value = 0; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + const TypeParam center_value = scale * (j ? 17 : 5); + num_with_center_value += + (center_value == this->blob_top_data_->cpu_data()[i * 2 + j]); + } + } + // Check we did not get the center crop all 10 times (occurs with + // probability 1-1/12^10 in working implementation). + EXPECT_LT(num_with_center_value, 10); + } +} + +TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainGPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + const TypeParam scale = 3; + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_scale(scale); + data_param->set_crop_size(1); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + DataLayer layer(param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + EXPECT_EQ(this->blob_top_data_->num(), 5); + EXPECT_EQ(this->blob_top_data_->channels(), 2); + EXPECT_EQ(this->blob_top_data_->height(), 1); + EXPECT_EQ(this->blob_top_data_->width(), 1); + EXPECT_EQ(this->blob_top_label_->num(), 5); + EXPECT_EQ(this->blob_top_label_->channels(), 1); + EXPECT_EQ(this->blob_top_label_->height(), 1); + EXPECT_EQ(this->blob_top_label_->width(), 1); + + for (int iter = 0; iter < 2; ++iter) { + layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + int num_with_center_value = 0; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + const TypeParam center_value = scale * (j ? 17 : 5); + num_with_center_value += + (center_value == this->blob_top_data_->cpu_data()[i * 2 + j]); + } + } + // Check we did not get the center crop all 10 times (occurs with + // probability 1-1/12^10 in working implementation). + EXPECT_LT(num_with_center_value, 10); + } +} + +// Test that the sequence of random crops is consistent when using +// Caffe::set_random_seed. +TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainSequenceSeededCPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_crop_size(1); + data_param->set_mirror(true); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + // Get crop sequence with Caffe seed 1701. + Caffe::set_random_seed(this->seed_); + vector > crop_sequence; + { + DataLayer layer1(param); + layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + vector iter_crop_sequence; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + iter_crop_sequence.push_back( + this->blob_top_data_->cpu_data()[i * 2 + j]); + } + } + crop_sequence.push_back(iter_crop_sequence); + } + } // destroy 1st data layer and unlock the leveldb + + // Get crop sequence after reseeding Caffe with 1701. + // Check that the sequence is the same as the original. + Caffe::set_random_seed(this->seed_); + DataLayer layer2(param); + layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + EXPECT_EQ(crop_sequence[iter][i * 2 + j], + this->blob_top_data_->cpu_data()[i * 2 + j]) + << "debug: iter " << iter << " i " << i << " j " << j; + } + } + } +} + +// Test that the sequence of random crops is consistent when using +// Caffe::set_random_seed. +TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainSequenceSeededGPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_crop_size(1); + data_param->set_mirror(true); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + // Get crop sequence with Caffe seed 1701. + Caffe::set_random_seed(this->seed_); + vector > crop_sequence; + { + DataLayer layer1(param); + layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + vector iter_crop_sequence; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + iter_crop_sequence.push_back( + this->blob_top_data_->cpu_data()[i * 2 + j]); + } + } + crop_sequence.push_back(iter_crop_sequence); + } + } // destroy 1st data layer and unlock the leveldb + + // Get crop sequence after reseeding Caffe with 1701. + // Check that the sequence is the same as the original. + Caffe::set_random_seed(this->seed_); + DataLayer layer2(param); + layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + EXPECT_EQ(crop_sequence[iter][i * 2 + j], + this->blob_top_data_->cpu_data()[i * 2 + j]) + << "debug: iter " << iter << " i " << i << " j " << j; + } + } + } +} + +// Test that the sequence of random crops differs across iterations when +// Caffe::set_random_seed isn't called (and seeds from srand are ignored). +TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainSequenceUnseededCPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_crop_size(1); + data_param->set_mirror(true); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + // Get crop sequence with Caffe seed 1701, srand seed 1701. + Caffe::set_random_seed(this->seed_); + srand(this->seed_); + vector > crop_sequence; + { + DataLayer layer1(param); + layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + vector iter_crop_sequence; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + iter_crop_sequence.push_back( + this->blob_top_data_->cpu_data()[i * 2 + j]); + } + } + crop_sequence.push_back(iter_crop_sequence); + } + } // destroy 1st data layer and unlock the leveldb + + // Get crop sequence continuing from previous Caffe RNG state; + // reseed srand with 1701. Check that the sequence differs from the original. + srand(this->seed_); + DataLayer layer2(param); + layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + int num_sequence_matches = 0; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + num_sequence_matches += (crop_sequence[iter][i * 2 + j] == + this->blob_top_data_->cpu_data()[i * 2 + j]); + } + } + EXPECT_LT(num_sequence_matches, 10); + } +} + +// Test that the sequence of random crops differs across iterations when +// Caffe::set_random_seed isn't called (and seeds from srand are ignored). +TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainSequenceUnseededGPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_crop_size(1); + data_param->set_mirror(true); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + // Get crop sequence with Caffe seed 1701, srand seed 1701. + Caffe::set_random_seed(this->seed_); + srand(this->seed_); + vector > crop_sequence; + { + DataLayer layer1(param); + layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + vector iter_crop_sequence; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + iter_crop_sequence.push_back( + this->blob_top_data_->cpu_data()[i * 2 + j]); + } + } + crop_sequence.push_back(iter_crop_sequence); + } + } // destroy 1st data layer and unlock the leveldb + + // Get crop sequence continuing from previous Caffe RNG state; + // reseed srand with 1701. Check that the sequence differs from the original. + srand(this->seed_); + DataLayer layer2(param); + layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + int num_sequence_matches = 0; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + num_sequence_matches += (crop_sequence[iter][i * 2 + j] == + this->blob_top_data_->cpu_data()[i * 2 + j]); + } + } + EXPECT_LT(num_sequence_matches, 10); + } +} + +TYPED_TEST(DataLayerTestLmdb, TestReadCropTestCPU) { + Caffe::set_phase(Caffe::TEST); + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + const TypeParam scale = 3; + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_scale(scale); + data_param->set_crop_size(1); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + DataLayer layer(param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + EXPECT_EQ(this->blob_top_data_->num(), 5); + EXPECT_EQ(this->blob_top_data_->channels(), 2); + EXPECT_EQ(this->blob_top_data_->height(), 1); + EXPECT_EQ(this->blob_top_data_->width(), 1); + EXPECT_EQ(this->blob_top_label_->num(), 5); + EXPECT_EQ(this->blob_top_label_->channels(), 1); + EXPECT_EQ(this->blob_top_label_->height(), 1); + EXPECT_EQ(this->blob_top_label_->width(), 1); + + for (int iter = 0; iter < 2; ++iter) { + layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + const TypeParam center_value = scale * (j ? 17 : 5); + EXPECT_EQ(center_value, this->blob_top_data_->cpu_data()[i * 2 + j]) + << "debug: iter " << iter << " i " << i << " j " << j; + } + } + } +} + +TYPED_TEST(DataLayerTestLmdb, TestReadCropTestGPU) { + Caffe::set_phase(Caffe::TEST); + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + const TypeParam scale = 3; + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_scale(scale); + data_param->set_crop_size(1); + data_param->set_source(this->filename_->c_str()); + data_param->set_backend(DataParameter_DB_LMDB); + DataLayer layer(param); + layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); + EXPECT_EQ(this->blob_top_data_->num(), 5); + EXPECT_EQ(this->blob_top_data_->channels(), 2); + EXPECT_EQ(this->blob_top_data_->height(), 1); + EXPECT_EQ(this->blob_top_data_->width(), 1); + EXPECT_EQ(this->blob_top_label_->num(), 5); + EXPECT_EQ(this->blob_top_label_->channels(), 1); + EXPECT_EQ(this->blob_top_label_->height(), 1); + EXPECT_EQ(this->blob_top_label_->width(), 1); + + for (int iter = 0; iter < 2; ++iter) { + layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); + } + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + const TypeParam center_value = scale * (j ? 17 : 5); + EXPECT_EQ(center_value, this->blob_top_data_->cpu_data()[i * 2 + j]) + << "debug: iter " << iter << " i " << i << " j " << j; + } + } + } +} + +} // namespace caffe From c77c6e12e49048e4cfcfac82dee0a8dae3e69d56 Mon Sep 17 00:00:00 2001 From: linmin Date: Fri, 13 Jun 2014 10:26:38 +0800 Subject: [PATCH 3/9] fixed cpplint error --- src/caffe/test/test_data_layer_lmdb.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/caffe/test/test_data_layer_lmdb.cpp b/src/caffe/test/test_data_layer_lmdb.cpp index 9b5b7d2bfc6..f3f9d723b5b 100644 --- a/src/caffe/test/test_data_layer_lmdb.cpp +++ b/src/caffe/test/test_data_layer_lmdb.cpp @@ -1,8 +1,8 @@ // Copyright 2014 BVLC and contributors. +#include #include #include -#include #include "cuda_runtime.h" #include "leveldb/db.h" @@ -48,7 +48,7 @@ class DataLayerTestLmdb : public ::testing::Test { MDB_val mdbkey, mdbdata; MDB_txn *txn; CHECK_EQ(mdb_env_create(&env), MDB_SUCCESS) << "mdb_env_create failed"; - CHECK_EQ(mdb_env_set_mapsize(env, 1099511627776), MDB_SUCCESS) // 1TB + CHECK_EQ(mdb_env_set_mapsize(env, 1099511627776), MDB_SUCCESS) // 1TB << "mdb_env_set_mapsize failed"; CHECK_EQ(mdb_env_open(env, filename_->c_str(), 0, 0664), MDB_SUCCESS) << "mdb_env_open failed"; @@ -85,7 +85,10 @@ class DataLayerTestLmdb : public ::testing::Test { mdb_env_close(env); } - virtual ~DataLayerTestLmdb() { delete blob_top_data_; delete blob_top_label_; } + virtual ~DataLayerTestLmdb() { + delete blob_top_data_; + delete blob_top_label_; + } shared_ptr filename_; Blob* const blob_top_data_; From c17e413b305c29f64fdaa348d070478414a208a6 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Thu, 12 Jun 2014 22:29:59 -0700 Subject: [PATCH 4/9] lint --- src/caffe/layers/data_layer.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/caffe/layers/data_layer.cpp b/src/caffe/layers/data_layer.cpp index 3b307ec7743..f12ae1c12e1 100644 --- a/src/caffe/layers/data_layer.cpp +++ b/src/caffe/layers/data_layer.cpp @@ -156,7 +156,7 @@ DataLayer::~DataLayer() { // clean up the database resources switch (this->layer_param_.data_param().backend()) { case DataParameter_DB_LEVELDB: - break; // do nothing + break; // do nothing case DataParameter_DB_LMDB: mdb_cursor_close(mdb_cursor_); mdb_close(mdb_env_, mdb_dbi_); @@ -198,10 +198,10 @@ void DataLayer::SetUp(const vector*>& bottom, break; case DataParameter_DB_LMDB: CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed"; - CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB + CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB CHECK_EQ(mdb_env_open(mdb_env_, - this->layer_param_.data_param().source().c_str(), - MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed"; + this->layer_param_.data_param().source().c_str(), + MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed"; CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS) << "mdb_txn_begin failed"; CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS) @@ -230,10 +230,10 @@ void DataLayer::SetUp(const vector*>& bottom, } break; case DataParameter_DB_LMDB: - if(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT) + if (mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT) != MDB_SUCCESS) { CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, - MDB_FIRST), MDB_SUCCESS); + MDB_FIRST), MDB_SUCCESS); } break; default: From f8ebd09d067d893289e34b0ccc8ac2d2ac2cca8f Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Thu, 12 Jun 2014 22:51:33 -0700 Subject: [PATCH 5/9] unify test_data_layer tests --- src/caffe/test/test_data_layer.cpp | 573 ++++++++++------------------- 1 file changed, 191 insertions(+), 382 deletions(-) diff --git a/src/caffe/test/test_data_layer.cpp b/src/caffe/test/test_data_layer.cpp index 1eae1a4e068..98ac60d47b4 100644 --- a/src/caffe/test/test_data_layer.cpp +++ b/src/caffe/test/test_data_layer.cpp @@ -63,6 +63,187 @@ class DataLayerTest : public ::testing::Test { delete db; } + void TestRead() { + const Dtype scale = 3; + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_scale(scale); + data_param->set_source(filename_->c_str()); + DataLayer layer(param); + layer.SetUp(blob_bottom_vec_, &blob_top_vec_); + EXPECT_EQ(blob_top_data_->num(), 5); + EXPECT_EQ(blob_top_data_->channels(), 2); + EXPECT_EQ(blob_top_data_->height(), 3); + EXPECT_EQ(blob_top_data_->width(), 4); + EXPECT_EQ(blob_top_label_->num(), 5); + EXPECT_EQ(blob_top_label_->channels(), 1); + EXPECT_EQ(blob_top_label_->height(), 1); + EXPECT_EQ(blob_top_label_->width(), 1); + + for (int iter = 0; iter < 100; ++iter) { + layer.Forward(blob_bottom_vec_, &blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, blob_top_label_->cpu_data()[i]); + } + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 24; ++j) { + EXPECT_EQ(scale * i, blob_top_data_->cpu_data()[i * 24 + j]) + << "debug: iter " << iter << " i " << i << " j " << j; + } + } + } + } + + void TestReadCrop() { + const Dtype scale = 3; + LayerParameter param; + Caffe::set_random_seed(1701); + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_scale(scale); + data_param->set_crop_size(1); + data_param->set_source(filename_->c_str()); + DataLayer layer(param); + layer.SetUp(blob_bottom_vec_, &blob_top_vec_); + EXPECT_EQ(blob_top_data_->num(), 5); + EXPECT_EQ(blob_top_data_->channels(), 2); + EXPECT_EQ(blob_top_data_->height(), 1); + EXPECT_EQ(blob_top_data_->width(), 1); + EXPECT_EQ(blob_top_label_->num(), 5); + EXPECT_EQ(blob_top_label_->channels(), 1); + EXPECT_EQ(blob_top_label_->height(), 1); + EXPECT_EQ(blob_top_label_->width(), 1); + + for (int iter = 0; iter < 2; ++iter) { + layer.Forward(blob_bottom_vec_, &blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, blob_top_label_->cpu_data()[i]); + } + int num_with_center_value = 0; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + const Dtype center_value = scale * (j ? 17 : 5); + num_with_center_value += + (center_value == blob_top_data_->cpu_data()[i * 2 + j]); + // At TEST time, check that we always get center value. + if (Caffe::phase() == Caffe::TEST) { + EXPECT_EQ(center_value, this->blob_top_data_->cpu_data()[i * 2 + j]) + << "debug: iter " << iter << " i " << i << " j " << j; + } + } + } + // At TRAIN time, check that we did not get the center crop all 10 times. + // (This check fails with probability 1-1/12^10 in a correct + // implementation, so we call set_random_seed.) + if (Caffe::phase() == Caffe::TRAIN) { + EXPECT_LT(num_with_center_value, 10); + } + } + } + + void TestReadCropTrainSequenceSeeded() { + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_crop_size(1); + data_param->set_mirror(true); + data_param->set_source(filename_->c_str()); + + // Get crop sequence with Caffe seed 1701. + Caffe::set_random_seed(seed_); + vector > crop_sequence; + { + DataLayer layer1(param); + layer1.SetUp(blob_bottom_vec_, &blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer1.Forward(blob_bottom_vec_, &blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, blob_top_label_->cpu_data()[i]); + } + vector iter_crop_sequence; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + iter_crop_sequence.push_back( + blob_top_data_->cpu_data()[i * 2 + j]); + } + } + crop_sequence.push_back(iter_crop_sequence); + } + } // destroy 1st data layer and unlock the leveldb + + // Get crop sequence after reseeding Caffe with 1701. + // Check that the sequence is the same as the original. + Caffe::set_random_seed(seed_); + DataLayer layer2(param); + layer2.SetUp(blob_bottom_vec_, &blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer2.Forward(blob_bottom_vec_, &blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, blob_top_label_->cpu_data()[i]); + } + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + EXPECT_EQ(crop_sequence[iter][i * 2 + j], + blob_top_data_->cpu_data()[i * 2 + j]) + << "debug: iter " << iter << " i " << i << " j " << j; + } + } + } + } + + void TestReadCropTrainSequenceUnseeded() { + LayerParameter param; + DataParameter* data_param = param.mutable_data_param(); + data_param->set_batch_size(5); + data_param->set_crop_size(1); + data_param->set_mirror(true); + data_param->set_source(filename_->c_str()); + + // Get crop sequence with Caffe seed 1701, srand seed 1701. + Caffe::set_random_seed(seed_); + srand(seed_); + vector > crop_sequence; + { + DataLayer layer1(param); + layer1.SetUp(blob_bottom_vec_, &blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer1.Forward(blob_bottom_vec_, &blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, blob_top_label_->cpu_data()[i]); + } + vector iter_crop_sequence; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + iter_crop_sequence.push_back( + blob_top_data_->cpu_data()[i * 2 + j]); + } + } + crop_sequence.push_back(iter_crop_sequence); + } + } // destroy 1st data layer and unlock the leveldb + + // Get crop sequence continuing from previous Caffe RNG state; + // reseed srand with 1701. Check that the sequence differs from the original. + srand(seed_); + DataLayer layer2(param); + layer2.SetUp(blob_bottom_vec_, &blob_top_vec_); + for (int iter = 0; iter < 2; ++iter) { + layer2.Forward(blob_bottom_vec_, &blob_top_vec_); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(i, blob_top_label_->cpu_data()[i]); + } + int num_sequence_matches = 0; + for (int i = 0; i < 5; ++i) { + for (int j = 0; j < 2; ++j) { + num_sequence_matches += (crop_sequence[iter][i * 2 + j] == + blob_top_data_->cpu_data()[i * 2 + j]); + } + } + EXPECT_LT(num_sequence_matches, 10); + } + } + virtual ~DataLayerTest() { delete blob_top_data_; delete blob_top_label_; } shared_ptr filename_; @@ -80,70 +261,14 @@ TYPED_TEST(DataLayerTest, TestReadCPU) { Caffe::set_mode(Caffe::CPU); const bool unique_pixels = false; // all pixels the same; images different this->FillLevelDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_source(this->filename_->c_str()); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 3); - EXPECT_EQ(this->blob_top_data_->width(), 4); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 100; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 24; ++j) { - EXPECT_EQ(scale * i, this->blob_top_data_->cpu_data()[i * 24 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } + this->TestRead(); } TYPED_TEST(DataLayerTest, TestReadGPU) { Caffe::set_mode(Caffe::GPU); const bool unique_pixels = false; // all pixels the same; images different this->FillLevelDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_source(this->filename_->c_str()); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 3); - EXPECT_EQ(this->blob_top_data_->width(), 4); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 100; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 24; ++j) { - EXPECT_EQ(scale * i, this->blob_top_data_->cpu_data()[i * 24 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } + this->TestRead(); } TYPED_TEST(DataLayerTest, TestReadCropTrainCPU) { @@ -151,41 +276,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainCPU) { Caffe::set_mode(Caffe::CPU); const bool unique_pixels = true; // all images the same; pixels different this->FillLevelDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_crop_size(1); - data_param->set_source(this->filename_->c_str()); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 1); - EXPECT_EQ(this->blob_top_data_->width(), 1); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 2; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - int num_with_center_value = 0; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - const TypeParam center_value = scale * (j ? 17 : 5); - num_with_center_value += - (center_value == this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - // Check we did not get the center crop all 10 times (occurs with - // probability 1-1/12^10 in working implementation). - EXPECT_LT(num_with_center_value, 10); - } + this->TestReadCrop(); } TYPED_TEST(DataLayerTest, TestReadCropTrainGPU) { @@ -193,41 +284,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainGPU) { Caffe::set_mode(Caffe::GPU); const bool unique_pixels = true; // all images the same; pixels different this->FillLevelDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_crop_size(1); - data_param->set_source(this->filename_->c_str()); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 1); - EXPECT_EQ(this->blob_top_data_->width(), 1); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 2; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - int num_with_center_value = 0; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - const TypeParam center_value = scale * (j ? 17 : 5); - num_with_center_value += - (center_value == this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - // Check we did not get the center crop all 10 times (occurs with - // probability 1-1/12^10 in working implementation). - EXPECT_LT(num_with_center_value, 10); - } + this->TestReadCrop(); } // Test that the sequence of random crops is consistent when using @@ -237,53 +294,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededCPU) { Caffe::set_mode(Caffe::CPU); const bool unique_pixels = true; // all images the same; pixels different this->FillLevelDB(unique_pixels); - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_crop_size(1); - data_param->set_mirror(true); - data_param->set_source(this->filename_->c_str()); - - // Get crop sequence with Caffe seed 1701. - Caffe::set_random_seed(this->seed_); - vector > crop_sequence; - { - DataLayer layer1(param); - layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - vector iter_crop_sequence; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - iter_crop_sequence.push_back( - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - crop_sequence.push_back(iter_crop_sequence); - } - } // destroy 1st data layer and unlock the leveldb - - // Get crop sequence after reseeding Caffe with 1701. - // Check that the sequence is the same as the original. - Caffe::set_random_seed(this->seed_); - DataLayer layer2(param); - layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - EXPECT_EQ(crop_sequence[iter][i * 2 + j], - this->blob_top_data_->cpu_data()[i * 2 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } + this->TestReadCropTrainSequenceSeeded(); } // Test that the sequence of random crops is consistent when using @@ -293,53 +304,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededGPU) { Caffe::set_mode(Caffe::GPU); const bool unique_pixels = true; // all images the same; pixels different this->FillLevelDB(unique_pixels); - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_crop_size(1); - data_param->set_mirror(true); - data_param->set_source(this->filename_->c_str()); - - // Get crop sequence with Caffe seed 1701. - Caffe::set_random_seed(this->seed_); - vector > crop_sequence; - { - DataLayer layer1(param); - layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - vector iter_crop_sequence; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - iter_crop_sequence.push_back( - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - crop_sequence.push_back(iter_crop_sequence); - } - } // destroy 1st data layer and unlock the leveldb - - // Get crop sequence after reseeding Caffe with 1701. - // Check that the sequence is the same as the original. - Caffe::set_random_seed(this->seed_); - DataLayer layer2(param); - layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - EXPECT_EQ(crop_sequence[iter][i * 2 + j], - this->blob_top_data_->cpu_data()[i * 2 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } + this->TestReadCropTrainSequenceSeeded(); } // Test that the sequence of random crops differs across iterations when @@ -349,55 +314,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededCPU) { Caffe::set_mode(Caffe::CPU); const bool unique_pixels = true; // all images the same; pixels different this->FillLevelDB(unique_pixels); - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_crop_size(1); - data_param->set_mirror(true); - data_param->set_source(this->filename_->c_str()); - - // Get crop sequence with Caffe seed 1701, srand seed 1701. - Caffe::set_random_seed(this->seed_); - srand(this->seed_); - vector > crop_sequence; - { - DataLayer layer1(param); - layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - vector iter_crop_sequence; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - iter_crop_sequence.push_back( - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - crop_sequence.push_back(iter_crop_sequence); - } - } // destroy 1st data layer and unlock the leveldb - - // Get crop sequence continuing from previous Caffe RNG state; - // reseed srand with 1701. Check that the sequence differs from the original. - srand(this->seed_); - DataLayer layer2(param); - layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - int num_sequence_matches = 0; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - num_sequence_matches += (crop_sequence[iter][i * 2 + j] == - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - EXPECT_LT(num_sequence_matches, 10); - } + this->TestReadCropTrainSequenceUnseeded(); } // Test that the sequence of random crops differs across iterations when @@ -407,55 +324,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededGPU) { Caffe::set_mode(Caffe::GPU); const bool unique_pixels = true; // all images the same; pixels different this->FillLevelDB(unique_pixels); - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_crop_size(1); - data_param->set_mirror(true); - data_param->set_source(this->filename_->c_str()); - - // Get crop sequence with Caffe seed 1701, srand seed 1701. - Caffe::set_random_seed(this->seed_); - srand(this->seed_); - vector > crop_sequence; - { - DataLayer layer1(param); - layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - vector iter_crop_sequence; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - iter_crop_sequence.push_back( - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - crop_sequence.push_back(iter_crop_sequence); - } - } // destroy 1st data layer and unlock the leveldb - - // Get crop sequence continuing from previous Caffe RNG state; - // reseed srand with 1701. Check that the sequence differs from the original. - srand(this->seed_); - DataLayer layer2(param); - layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - int num_sequence_matches = 0; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - num_sequence_matches += (crop_sequence[iter][i * 2 + j] == - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - EXPECT_LT(num_sequence_matches, 10); - } + this->TestReadCropTrainSequenceUnseeded(); } TYPED_TEST(DataLayerTest, TestReadCropTestCPU) { @@ -463,37 +332,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTestCPU) { Caffe::set_mode(Caffe::CPU); const bool unique_pixels = true; // all images the same; pixels different this->FillLevelDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_crop_size(1); - data_param->set_source(this->filename_->c_str()); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 1); - EXPECT_EQ(this->blob_top_data_->width(), 1); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 2; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - const TypeParam center_value = scale * (j ? 17 : 5); - EXPECT_EQ(center_value, this->blob_top_data_->cpu_data()[i * 2 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } + this->TestReadCrop(); } TYPED_TEST(DataLayerTest, TestReadCropTestGPU) { @@ -501,37 +340,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTestGPU) { Caffe::set_mode(Caffe::GPU); const bool unique_pixels = true; // all images the same; pixels different this->FillLevelDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_crop_size(1); - data_param->set_source(this->filename_->c_str()); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 1); - EXPECT_EQ(this->blob_top_data_->width(), 1); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 2; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - const TypeParam center_value = scale * (j ? 17 : 5); - EXPECT_EQ(center_value, this->blob_top_data_->cpu_data()[i * 2 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } + this->TestReadCrop(); } } // namespace caffe From 1dd0d94bbea53bd080ad48ae15d53e4e9d47cad7 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Thu, 12 Jun 2014 23:07:23 -0700 Subject: [PATCH 6/9] unify data layer tests: was copied four times for all combinations of cpu/gpu and leveldb/lmdb; now just one copy of each test body --- src/caffe/test/test_data_layer.cpp | 167 ++++++- src/caffe/test/test_data_layer_lmdb.cpp | 568 ------------------------ 2 files changed, 154 insertions(+), 581 deletions(-) delete mode 100644 src/caffe/test/test_data_layer_lmdb.cpp diff --git a/src/caffe/test/test_data_layer.cpp b/src/caffe/test/test_data_layer.cpp index 98ac60d47b4..68f2618d323 100644 --- a/src/caffe/test/test_data_layer.cpp +++ b/src/caffe/test/test_data_layer.cpp @@ -27,6 +27,7 @@ class DataLayerTest : public ::testing::Test { : blob_top_data_(new Blob()), blob_top_label_(new Blob()), filename_(new string(tmpnam(NULL))), + backend_(DataParameter_DB_LEVELDB), seed_(1701) {} virtual void SetUp() { blob_top_vec_.push_back(blob_top_data_); @@ -37,6 +38,7 @@ class DataLayerTest : public ::testing::Test { // all images are the same; else each image is unique but all pixels within // an image are the same. void FillLevelDB(const bool unique_pixels) { + backend_ = DataParameter_DB_LEVELDB; LOG(INFO) << "Using temporary leveldb " << *filename_; leveldb::DB* db; leveldb::Options options; @@ -63,6 +65,54 @@ class DataLayerTest : public ::testing::Test { delete db; } + // Fill the LMDB with data: unique_pixels has same meaning as in FillLevelDB. + void FillLMDB(const bool unique_pixels) { + backend_ = DataParameter_DB_LMDB; + LOG(INFO) << "Using temporary lmdb " << *filename_; + CHECK_EQ(mkdir(filename_->c_str(), 0744), 0) << "mkdir " << filename_ + << "failed"; + MDB_env *env; + MDB_dbi dbi; + MDB_val mdbkey, mdbdata; + MDB_txn *txn; + CHECK_EQ(mdb_env_create(&env), MDB_SUCCESS) << "mdb_env_create failed"; + CHECK_EQ(mdb_env_set_mapsize(env, 1099511627776), MDB_SUCCESS) // 1TB + << "mdb_env_set_mapsize failed"; + CHECK_EQ(mdb_env_open(env, filename_->c_str(), 0, 0664), MDB_SUCCESS) + << "mdb_env_open failed"; + CHECK_EQ(mdb_txn_begin(env, NULL, 0, &txn), MDB_SUCCESS) + << "mdb_txn_begin failed"; + CHECK_EQ(mdb_open(txn, NULL, 0, &dbi), MDB_SUCCESS) << "mdb_open failed"; + + for (int i = 0; i < 5; ++i) { + Datum datum; + datum.set_label(i); + datum.set_channels(2); + datum.set_height(3); + datum.set_width(4); + std::string* data = datum.mutable_data(); + for (int j = 0; j < 24; ++j) { + int datum = unique_pixels ? j : i; + data->push_back(static_cast(datum)); + } + stringstream ss; + ss << i; + + string value; + datum.SerializeToString(&value); + mdbdata.mv_size = value.size(); + mdbdata.mv_data = reinterpret_cast(&value[0]); + string keystr = ss.str(); + mdbkey.mv_size = keystr.size(); + mdbkey.mv_data = reinterpret_cast(&keystr[0]); + CHECK_EQ(mdb_put(txn, dbi, &mdbkey, &mdbdata, 0), MDB_SUCCESS) + << "mdb_put failed"; + } + CHECK_EQ(mdb_txn_commit(txn), MDB_SUCCESS) << "mdb_txn_commit failed"; + mdb_close(env, dbi); + mdb_env_close(env); + } + void TestRead() { const Dtype scale = 3; LayerParameter param; @@ -70,6 +120,7 @@ class DataLayerTest : public ::testing::Test { data_param->set_batch_size(5); data_param->set_scale(scale); data_param->set_source(filename_->c_str()); + data_param->set_backend(backend_); DataLayer layer(param); layer.SetUp(blob_bottom_vec_, &blob_top_vec_); EXPECT_EQ(blob_top_data_->num(), 5); @@ -104,6 +155,7 @@ class DataLayerTest : public ::testing::Test { data_param->set_scale(scale); data_param->set_crop_size(1); data_param->set_source(filename_->c_str()); + data_param->set_backend(backend_); DataLayer layer(param); layer.SetUp(blob_bottom_vec_, &blob_top_vec_); EXPECT_EQ(blob_top_data_->num(), 5); @@ -134,7 +186,7 @@ class DataLayerTest : public ::testing::Test { } } // At TRAIN time, check that we did not get the center crop all 10 times. - // (This check fails with probability 1-1/12^10 in a correct + // (This check fails with probability 1-1/12^10 in a correct // implementation, so we call set_random_seed.) if (Caffe::phase() == Caffe::TRAIN) { EXPECT_LT(num_with_center_value, 10); @@ -149,6 +201,7 @@ class DataLayerTest : public ::testing::Test { data_param->set_crop_size(1); data_param->set_mirror(true); data_param->set_source(filename_->c_str()); + data_param->set_backend(backend_); // Get crop sequence with Caffe seed 1701. Caffe::set_random_seed(seed_); @@ -199,6 +252,7 @@ class DataLayerTest : public ::testing::Test { data_param->set_crop_size(1); data_param->set_mirror(true); data_param->set_source(filename_->c_str()); + data_param->set_backend(backend_); // Get crop sequence with Caffe seed 1701, srand seed 1701. Caffe::set_random_seed(seed_); @@ -223,8 +277,8 @@ class DataLayerTest : public ::testing::Test { } } // destroy 1st data layer and unlock the leveldb - // Get crop sequence continuing from previous Caffe RNG state; - // reseed srand with 1701. Check that the sequence differs from the original. + // Get crop sequence continuing from previous Caffe RNG state; reseed + // srand with 1701. Check that the sequence differs from the original. srand(seed_); DataLayer layer2(param); layer2.SetUp(blob_bottom_vec_, &blob_top_vec_); @@ -246,6 +300,7 @@ class DataLayerTest : public ::testing::Test { virtual ~DataLayerTest() { delete blob_top_data_; delete blob_top_label_; } + DataParameter_DB backend_; shared_ptr filename_; Blob* const blob_top_data_; Blob* const blob_top_label_; @@ -257,21 +312,21 @@ class DataLayerTest : public ::testing::Test { typedef ::testing::Types Dtypes; TYPED_TEST_CASE(DataLayerTest, Dtypes); -TYPED_TEST(DataLayerTest, TestReadCPU) { +TYPED_TEST(DataLayerTest, TestReadLevelDBCPU) { Caffe::set_mode(Caffe::CPU); const bool unique_pixels = false; // all pixels the same; images different this->FillLevelDB(unique_pixels); this->TestRead(); } -TYPED_TEST(DataLayerTest, TestReadGPU) { +TYPED_TEST(DataLayerTest, TestReadLevelDBGPU) { Caffe::set_mode(Caffe::GPU); const bool unique_pixels = false; // all pixels the same; images different this->FillLevelDB(unique_pixels); this->TestRead(); } -TYPED_TEST(DataLayerTest, TestReadCropTrainCPU) { +TYPED_TEST(DataLayerTest, TestReadCropTrainLevelDBCPU) { Caffe::set_phase(Caffe::TRAIN); Caffe::set_mode(Caffe::CPU); const bool unique_pixels = true; // all images the same; pixels different @@ -279,7 +334,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainCPU) { this->TestReadCrop(); } -TYPED_TEST(DataLayerTest, TestReadCropTrainGPU) { +TYPED_TEST(DataLayerTest, TestReadCropTrainLevelDBGPU) { Caffe::set_phase(Caffe::TRAIN); Caffe::set_mode(Caffe::GPU); const bool unique_pixels = true; // all images the same; pixels different @@ -289,7 +344,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainGPU) { // Test that the sequence of random crops is consistent when using // Caffe::set_random_seed. -TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededCPU) { +TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededLevelDBCPU) { Caffe::set_phase(Caffe::TRAIN); Caffe::set_mode(Caffe::CPU); const bool unique_pixels = true; // all images the same; pixels different @@ -299,7 +354,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededCPU) { // Test that the sequence of random crops is consistent when using // Caffe::set_random_seed. -TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededGPU) { +TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededLevelDBGPU) { Caffe::set_phase(Caffe::TRAIN); Caffe::set_mode(Caffe::GPU); const bool unique_pixels = true; // all images the same; pixels different @@ -309,7 +364,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededGPU) { // Test that the sequence of random crops differs across iterations when // Caffe::set_random_seed isn't called (and seeds from srand are ignored). -TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededCPU) { +TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededLevelDBCPU) { Caffe::set_phase(Caffe::TRAIN); Caffe::set_mode(Caffe::CPU); const bool unique_pixels = true; // all images the same; pixels different @@ -319,7 +374,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededCPU) { // Test that the sequence of random crops differs across iterations when // Caffe::set_random_seed isn't called (and seeds from srand are ignored). -TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededGPU) { +TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededLevelDBGPU) { Caffe::set_phase(Caffe::TRAIN); Caffe::set_mode(Caffe::GPU); const bool unique_pixels = true; // all images the same; pixels different @@ -327,7 +382,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededGPU) { this->TestReadCropTrainSequenceUnseeded(); } -TYPED_TEST(DataLayerTest, TestReadCropTestCPU) { +TYPED_TEST(DataLayerTest, TestReadCropTestLevelDBCPU) { Caffe::set_phase(Caffe::TEST); Caffe::set_mode(Caffe::CPU); const bool unique_pixels = true; // all images the same; pixels different @@ -335,7 +390,7 @@ TYPED_TEST(DataLayerTest, TestReadCropTestCPU) { this->TestReadCrop(); } -TYPED_TEST(DataLayerTest, TestReadCropTestGPU) { +TYPED_TEST(DataLayerTest, TestReadCropTestLevelDBGPU) { Caffe::set_phase(Caffe::TEST); Caffe::set_mode(Caffe::GPU); const bool unique_pixels = true; // all images the same; pixels different @@ -343,4 +398,90 @@ TYPED_TEST(DataLayerTest, TestReadCropTestGPU) { this->TestReadCrop(); } +TYPED_TEST(DataLayerTest, TestReadLMDBCPU) { + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = false; // all pixels the same; images different + this->FillLMDB(unique_pixels); + this->TestRead(); +} + +TYPED_TEST(DataLayerTest, TestReadLMDBGPU) { + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = false; // all pixels the same; images different + this->FillLMDB(unique_pixels); + this->TestRead(); +} + +TYPED_TEST(DataLayerTest, TestReadCropTrainLMDBCPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + this->TestReadCrop(); +} + +TYPED_TEST(DataLayerTest, TestReadCropTrainLMDBGPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + this->TestReadCrop(); +} + +// Test that the sequence of random crops is consistent when using +// Caffe::set_random_seed. +TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededLMDBCPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + this->TestReadCropTrainSequenceSeeded(); +} + +// Test that the sequence of random crops is consistent when using +// Caffe::set_random_seed. +TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceSeededLMDBGPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + this->TestReadCropTrainSequenceSeeded(); +} + +// Test that the sequence of random crops differs across iterations when +// Caffe::set_random_seed isn't called (and seeds from srand are ignored). +TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededLMDBCPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + this->TestReadCropTrainSequenceUnseeded(); +} + +// Test that the sequence of random crops differs across iterations when +// Caffe::set_random_seed isn't called (and seeds from srand are ignored). +TYPED_TEST(DataLayerTest, TestReadCropTrainSequenceUnseededLMDBGPU) { + Caffe::set_phase(Caffe::TRAIN); + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + this->TestReadCropTrainSequenceUnseeded(); +} + +TYPED_TEST(DataLayerTest, TestReadCropTestLMDBCPU) { + Caffe::set_phase(Caffe::TEST); + Caffe::set_mode(Caffe::CPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + this->TestReadCrop(); +} + +TYPED_TEST(DataLayerTest, TestReadCropTestLMDBGPU) { + Caffe::set_phase(Caffe::TEST); + Caffe::set_mode(Caffe::GPU); + const bool unique_pixels = true; // all images the same; pixels different + this->FillLMDB(unique_pixels); + this->TestReadCrop(); +} + } // namespace caffe diff --git a/src/caffe/test/test_data_layer_lmdb.cpp b/src/caffe/test/test_data_layer_lmdb.cpp deleted file mode 100644 index f3f9d723b5b..00000000000 --- a/src/caffe/test/test_data_layer_lmdb.cpp +++ /dev/null @@ -1,568 +0,0 @@ -// Copyright 2014 BVLC and contributors. - -#include -#include -#include - -#include "cuda_runtime.h" -#include "leveldb/db.h" -#include "lmdb.h" -#include "gtest/gtest.h" -#include "caffe/blob.hpp" -#include "caffe/common.hpp" -#include "caffe/filler.hpp" -#include "caffe/vision_layers.hpp" -#include "caffe/proto/caffe.pb.h" -#include "caffe/test/test_caffe_main.hpp" - -using std::string; -using std::stringstream; - -namespace caffe { - -extern cudaDeviceProp CAFFE_TEST_CUDA_PROP; - -template -class DataLayerTestLmdb : public ::testing::Test { - protected: - DataLayerTestLmdb() - : blob_top_data_(new Blob()), - blob_top_label_(new Blob()), - filename_(new string(tmpnam(NULL))), - seed_(1701) {} - virtual void SetUp() { - blob_top_vec_.push_back(blob_top_data_); - blob_top_vec_.push_back(blob_top_label_); - } - - // Fill the LMDB with data: if unique_pixels, each pixel is unique but - // all images are the same; else each image is unique but all pixels within - // an image are the same. - - void FillLMDB(const bool unique_pixels) { - LOG(INFO) << "Using temporary lmdb " << *filename_; - CHECK_EQ(mkdir(filename_->c_str(), 0744), 0) << "mkdir " << filename_ - << "failed"; - MDB_env *env; - MDB_dbi dbi; - MDB_val mdbkey, mdbdata; - MDB_txn *txn; - CHECK_EQ(mdb_env_create(&env), MDB_SUCCESS) << "mdb_env_create failed"; - CHECK_EQ(mdb_env_set_mapsize(env, 1099511627776), MDB_SUCCESS) // 1TB - << "mdb_env_set_mapsize failed"; - CHECK_EQ(mdb_env_open(env, filename_->c_str(), 0, 0664), MDB_SUCCESS) - << "mdb_env_open failed"; - CHECK_EQ(mdb_txn_begin(env, NULL, 0, &txn), MDB_SUCCESS) - << "mdb_txn_begin failed"; - CHECK_EQ(mdb_open(txn, NULL, 0, &dbi), MDB_SUCCESS) << "mdb_open failed"; - - for (int i = 0; i < 5; ++i) { - Datum datum; - datum.set_label(i); - datum.set_channels(2); - datum.set_height(3); - datum.set_width(4); - std::string* data = datum.mutable_data(); - for (int j = 0; j < 24; ++j) { - int datum = unique_pixels ? j : i; - data->push_back(static_cast(datum)); - } - stringstream ss; - ss << i; - - string value; - datum.SerializeToString(&value); - mdbdata.mv_size = value.size(); - mdbdata.mv_data = reinterpret_cast(&value[0]); - string keystr = ss.str(); - mdbkey.mv_size = keystr.size(); - mdbkey.mv_data = reinterpret_cast(&keystr[0]); - CHECK_EQ(mdb_put(txn, dbi, &mdbkey, &mdbdata, 0), MDB_SUCCESS) - << "mdb_put failed"; - } - CHECK_EQ(mdb_txn_commit(txn), MDB_SUCCESS) << "mdb_txn_commit failed"; - mdb_close(env, dbi); - mdb_env_close(env); - } - - virtual ~DataLayerTestLmdb() { - delete blob_top_data_; - delete blob_top_label_; - } - - shared_ptr filename_; - Blob* const blob_top_data_; - Blob* const blob_top_label_; - vector*> blob_bottom_vec_; - vector*> blob_top_vec_; - int seed_; -}; - -typedef ::testing::Types Dtypes; -TYPED_TEST_CASE(DataLayerTestLmdb, Dtypes); - -TYPED_TEST(DataLayerTestLmdb, TestReadCPU) { - Caffe::set_mode(Caffe::CPU); - const bool unique_pixels = false; // all pixels the same; images different - this->FillLMDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 3); - EXPECT_EQ(this->blob_top_data_->width(), 4); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 100; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 24; ++j) { - EXPECT_EQ(scale * i, this->blob_top_data_->cpu_data()[i * 24 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } -} - -TYPED_TEST(DataLayerTestLmdb, TestReadGPU) { - Caffe::set_mode(Caffe::GPU); - const bool unique_pixels = false; // all pixels the same; images different - this->FillLMDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 3); - EXPECT_EQ(this->blob_top_data_->width(), 4); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 100; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 24; ++j) { - EXPECT_EQ(scale * i, this->blob_top_data_->cpu_data()[i * 24 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } -} - -TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainCPU) { - Caffe::set_phase(Caffe::TRAIN); - Caffe::set_mode(Caffe::CPU); - const bool unique_pixels = true; // all images the same; pixels different - this->FillLMDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_crop_size(1); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 1); - EXPECT_EQ(this->blob_top_data_->width(), 1); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 2; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - int num_with_center_value = 0; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - const TypeParam center_value = scale * (j ? 17 : 5); - num_with_center_value += - (center_value == this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - // Check we did not get the center crop all 10 times (occurs with - // probability 1-1/12^10 in working implementation). - EXPECT_LT(num_with_center_value, 10); - } -} - -TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainGPU) { - Caffe::set_phase(Caffe::TRAIN); - Caffe::set_mode(Caffe::GPU); - const bool unique_pixels = true; // all images the same; pixels different - this->FillLMDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_crop_size(1); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 1); - EXPECT_EQ(this->blob_top_data_->width(), 1); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 2; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - int num_with_center_value = 0; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - const TypeParam center_value = scale * (j ? 17 : 5); - num_with_center_value += - (center_value == this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - // Check we did not get the center crop all 10 times (occurs with - // probability 1-1/12^10 in working implementation). - EXPECT_LT(num_with_center_value, 10); - } -} - -// Test that the sequence of random crops is consistent when using -// Caffe::set_random_seed. -TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainSequenceSeededCPU) { - Caffe::set_phase(Caffe::TRAIN); - Caffe::set_mode(Caffe::CPU); - const bool unique_pixels = true; // all images the same; pixels different - this->FillLMDB(unique_pixels); - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_crop_size(1); - data_param->set_mirror(true); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - // Get crop sequence with Caffe seed 1701. - Caffe::set_random_seed(this->seed_); - vector > crop_sequence; - { - DataLayer layer1(param); - layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - vector iter_crop_sequence; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - iter_crop_sequence.push_back( - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - crop_sequence.push_back(iter_crop_sequence); - } - } // destroy 1st data layer and unlock the leveldb - - // Get crop sequence after reseeding Caffe with 1701. - // Check that the sequence is the same as the original. - Caffe::set_random_seed(this->seed_); - DataLayer layer2(param); - layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - EXPECT_EQ(crop_sequence[iter][i * 2 + j], - this->blob_top_data_->cpu_data()[i * 2 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } -} - -// Test that the sequence of random crops is consistent when using -// Caffe::set_random_seed. -TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainSequenceSeededGPU) { - Caffe::set_phase(Caffe::TRAIN); - Caffe::set_mode(Caffe::GPU); - const bool unique_pixels = true; // all images the same; pixels different - this->FillLMDB(unique_pixels); - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_crop_size(1); - data_param->set_mirror(true); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - // Get crop sequence with Caffe seed 1701. - Caffe::set_random_seed(this->seed_); - vector > crop_sequence; - { - DataLayer layer1(param); - layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - vector iter_crop_sequence; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - iter_crop_sequence.push_back( - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - crop_sequence.push_back(iter_crop_sequence); - } - } // destroy 1st data layer and unlock the leveldb - - // Get crop sequence after reseeding Caffe with 1701. - // Check that the sequence is the same as the original. - Caffe::set_random_seed(this->seed_); - DataLayer layer2(param); - layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - EXPECT_EQ(crop_sequence[iter][i * 2 + j], - this->blob_top_data_->cpu_data()[i * 2 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } -} - -// Test that the sequence of random crops differs across iterations when -// Caffe::set_random_seed isn't called (and seeds from srand are ignored). -TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainSequenceUnseededCPU) { - Caffe::set_phase(Caffe::TRAIN); - Caffe::set_mode(Caffe::CPU); - const bool unique_pixels = true; // all images the same; pixels different - this->FillLMDB(unique_pixels); - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_crop_size(1); - data_param->set_mirror(true); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - // Get crop sequence with Caffe seed 1701, srand seed 1701. - Caffe::set_random_seed(this->seed_); - srand(this->seed_); - vector > crop_sequence; - { - DataLayer layer1(param); - layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - vector iter_crop_sequence; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - iter_crop_sequence.push_back( - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - crop_sequence.push_back(iter_crop_sequence); - } - } // destroy 1st data layer and unlock the leveldb - - // Get crop sequence continuing from previous Caffe RNG state; - // reseed srand with 1701. Check that the sequence differs from the original. - srand(this->seed_); - DataLayer layer2(param); - layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - int num_sequence_matches = 0; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - num_sequence_matches += (crop_sequence[iter][i * 2 + j] == - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - EXPECT_LT(num_sequence_matches, 10); - } -} - -// Test that the sequence of random crops differs across iterations when -// Caffe::set_random_seed isn't called (and seeds from srand are ignored). -TYPED_TEST(DataLayerTestLmdb, TestReadCropTrainSequenceUnseededGPU) { - Caffe::set_phase(Caffe::TRAIN); - Caffe::set_mode(Caffe::GPU); - const bool unique_pixels = true; // all images the same; pixels different - this->FillLMDB(unique_pixels); - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_crop_size(1); - data_param->set_mirror(true); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - // Get crop sequence with Caffe seed 1701, srand seed 1701. - Caffe::set_random_seed(this->seed_); - srand(this->seed_); - vector > crop_sequence; - { - DataLayer layer1(param); - layer1.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer1.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - vector iter_crop_sequence; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - iter_crop_sequence.push_back( - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - crop_sequence.push_back(iter_crop_sequence); - } - } // destroy 1st data layer and unlock the leveldb - - // Get crop sequence continuing from previous Caffe RNG state; - // reseed srand with 1701. Check that the sequence differs from the original. - srand(this->seed_); - DataLayer layer2(param); - layer2.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int iter = 0; iter < 2; ++iter) { - layer2.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - int num_sequence_matches = 0; - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - num_sequence_matches += (crop_sequence[iter][i * 2 + j] == - this->blob_top_data_->cpu_data()[i * 2 + j]); - } - } - EXPECT_LT(num_sequence_matches, 10); - } -} - -TYPED_TEST(DataLayerTestLmdb, TestReadCropTestCPU) { - Caffe::set_phase(Caffe::TEST); - Caffe::set_mode(Caffe::CPU); - const bool unique_pixels = true; // all images the same; pixels different - this->FillLMDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_crop_size(1); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 1); - EXPECT_EQ(this->blob_top_data_->width(), 1); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 2; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - const TypeParam center_value = scale * (j ? 17 : 5); - EXPECT_EQ(center_value, this->blob_top_data_->cpu_data()[i * 2 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } -} - -TYPED_TEST(DataLayerTestLmdb, TestReadCropTestGPU) { - Caffe::set_phase(Caffe::TEST); - Caffe::set_mode(Caffe::GPU); - const bool unique_pixels = true; // all images the same; pixels different - this->FillLMDB(unique_pixels); - const TypeParam scale = 3; - LayerParameter param; - DataParameter* data_param = param.mutable_data_param(); - data_param->set_batch_size(5); - data_param->set_scale(scale); - data_param->set_crop_size(1); - data_param->set_source(this->filename_->c_str()); - data_param->set_backend(DataParameter_DB_LMDB); - DataLayer layer(param); - layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_); - EXPECT_EQ(this->blob_top_data_->num(), 5); - EXPECT_EQ(this->blob_top_data_->channels(), 2); - EXPECT_EQ(this->blob_top_data_->height(), 1); - EXPECT_EQ(this->blob_top_data_->width(), 1); - EXPECT_EQ(this->blob_top_label_->num(), 5); - EXPECT_EQ(this->blob_top_label_->channels(), 1); - EXPECT_EQ(this->blob_top_label_->height(), 1); - EXPECT_EQ(this->blob_top_label_->width(), 1); - - for (int iter = 0; iter < 2; ++iter) { - layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_); - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, this->blob_top_label_->cpu_data()[i]); - } - for (int i = 0; i < 5; ++i) { - for (int j = 0; j < 2; ++j) { - const TypeParam center_value = scale * (j ? 17 : 5); - EXPECT_EQ(center_value, this->blob_top_data_->cpu_data()[i * 2 + j]) - << "debug: iter " << iter << " i " << i << " j " << j; - } - } - } -} - -} // namespace caffe From e0ce93383e7b09d1f90db89b10f269a47fec360e Mon Sep 17 00:00:00 2001 From: linmin Date: Sat, 14 Jun 2014 16:15:18 +0800 Subject: [PATCH 7/9] add lmdb support for convert_imageset --- tools/convert_imageset.cpp | 112 ++++++++++++++++++++++++++++++------- 1 file changed, 92 insertions(+), 20 deletions(-) diff --git a/tools/convert_imageset.cpp b/tools/convert_imageset.cpp index 524d1950cb0..a381eaf4c03 100644 --- a/tools/convert_imageset.cpp +++ b/tools/convert_imageset.cpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include // NOLINT(readability/streams) @@ -33,12 +35,13 @@ using std::string; int main(int argc, char** argv) { ::google::InitGoogleLogging(argv[0]); - if (argc < 4 || argc > 8) { + if (argc < 4 || argc > 9) { printf("Convert a set of images to the leveldb format used\n" "as input for Caffe.\n" "Usage:\n" " convert_imageset [-g] ROOTFOLDER/ LISTFILE DB_NAME" - " RANDOM_SHUFFLE_DATA[0 or 1] [resize_height] [resize_width]\n" + " RANDOM_SHUFFLE_DATA[0 or 1] DB_BACKEND[leveldb or lmdb]" + " [resize_height] [resize_width]\n" "The ImageNet dataset for the training demo is at\n" " http://www.image-net.org/download-images\n"); return 1; @@ -60,33 +63,71 @@ int main(int argc, char** argv) { std::random_shuffle(lines.begin(), lines.end()); } LOG(INFO) << "A total of " << lines.size() << " images."; - int resize_height = 0; - int resize_width = 0; + + string db_backend = "leveldb"; if (argc >= (arg_offset+6)) { - resize_height = atoi(argv[arg_offset+5]); + db_backend = string(argv[arg_offset+5]); + if (db_backend != "leveldb" || db_backend != "lmdb") { + LOG(FATAL) << "Unknown db backend " << db_backend; + } } + + int resize_height = 0; + int resize_width = 0; if (argc >= (arg_offset+7)) { - resize_width = atoi(argv[arg_offset+6]); + resize_height = atoi(argv[arg_offset+6]); + } + if (argc >= (arg_offset+8)) { + resize_width = atoi(argv[arg_offset+7]); } + // Open new db + // lmdb + MDB_env *mdb_env; + MDB_dbi mdb_dbi; + MDB_val mdb_key, mdb_data; + MDB_txn *mdb_txn; + // leveldb leveldb::DB* db; leveldb::Options options; options.error_if_exists = true; options.create_if_missing = true; options.write_buffer_size = 268435456; - LOG(INFO) << "Opening leveldb " << argv[arg_offset+3]; - leveldb::Status status = leveldb::DB::Open( - options, argv[arg_offset+3], &db); - CHECK(status.ok()) << "Failed to open leveldb " << argv[arg_offset+3]; + leveldb::WriteBatch* batch; + // Open db + if (db_backend == "leveldb") { // leveldb + LOG(INFO) << "Opening leveldb " << argv[arg_offset+3]; + leveldb::Status status = leveldb::DB::Open( + options, argv[arg_offset+3], &db); + CHECK(status.ok()) << "Failed to open leveldb " << argv[arg_offset+3]; + batch = new leveldb::WriteBatch(); + } else if (db_backend == "lmdb") { // lmdb + LOG(INFO) << "Opening lmdb " << argv[arg_offset+3]; + CHECK_EQ(mkdir(argv[arg_offset+3], 0744), 0) + << "mkdir " << argv[arg_offset+3] << "failed"; + CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed"; + CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS) // 1TB + << "mdb_env_set_mapsize failed"; + CHECK_EQ(mdb_env_open(mdb_env, argv[3], 0, 0664), MDB_SUCCESS) + << "mdb_env_open failed"; + CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS) + << "mdb_txn_begin failed"; + CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS) + << "mdb_open failed"; + } else { + LOG(FATAL) << "Unknown db backend " << db_backend; + } + + // Storing to db string root_folder(argv[arg_offset+1]); Datum datum; int count = 0; const int kMaxKeyLength = 256; char key_cstr[kMaxKeyLength]; - leveldb::WriteBatch* batch = new leveldb::WriteBatch(); int data_size; bool data_size_initialized = false; + for (int line_id = 0; line_id < lines.size(); ++line_id) { if (!ReadImageToDatum(root_folder + lines[line_id].first, lines[line_id].second, resize_height, resize_width, is_color, &datum)) { @@ -104,23 +145,54 @@ int main(int argc, char** argv) { snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id, lines[line_id].first.c_str()); string value; - // get the value datum.SerializeToString(&value); - batch->Put(string(key_cstr), value); + string keystr(key_cstr); + + // Put in db + if (db_backend == "leveldb") { // leveldb + batch->Put(keystr, value); + } else if (db_backend == "lmdb") { // lmdb + mdb_data.mv_size = value.size(); + mdb_data.mv_data = reinterpret_cast(&value[0]); + mdb_key.mv_size = keystr.size(); + mdb_key.mv_data = reinterpret_cast(&keystr[0]); + CHECK_EQ(mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0), MDB_SUCCESS) + << "mdb_put failed"; + } else { + LOG(FATAL) << "Unknown db backend " << db_backend; + } + if (++count % 1000 == 0) { - db->Write(leveldb::WriteOptions(), batch); + // Commit txn + if (db_backend == "leveldb") { // leveldb + db->Write(leveldb::WriteOptions(), batch); + delete batch; + batch = new leveldb::WriteBatch(); + } else if (db_backend == "lmdb") { // lmdb + CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS) + << "mdb_txn_commit failed"; + CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS) + << "mdb_txn_begin failed"; + } else { + LOG(FATAL) << "Unknown db backend " << db_backend; + } LOG(ERROR) << "Processed " << count << " files."; - delete batch; - batch = new leveldb::WriteBatch(); } } // write the last batch if (count % 1000 != 0) { - db->Write(leveldb::WriteOptions(), batch); + if (db_backend == "leveldb") { // leveldb + db->Write(leveldb::WriteOptions(), batch); + delete batch; + delete db; + } else if (db_backend == "lmdb") { // lmdb + CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS) << "mdb_txn_commit failed"; + mdb_close(mdb_env, mdb_dbi); + mdb_env_close(mdb_env); + } else { + LOG(FATAL) << "Unknown db backend " << db_backend; + } LOG(ERROR) << "Processed " << count << " files."; } - - delete batch; - delete db; return 0; } From 01481d581b7b333c15f00f8c3597194eb285eeb9 Mon Sep 17 00:00:00 2001 From: linmin Date: Sat, 14 Jun 2014 16:15:44 +0800 Subject: [PATCH 8/9] add lmdb support for compute_image_mean --- tools/compute_image_mean.cpp | 145 +++++++++++++++++++++++++++-------- 1 file changed, 115 insertions(+), 30 deletions(-) diff --git a/tools/compute_image_mean.cpp b/tools/compute_image_mean.cpp index 7cf5fe5f222..5b212b66edd 100644 --- a/tools/compute_image_mean.cpp +++ b/tools/compute_image_mean.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -16,28 +17,69 @@ using std::max; int main(int argc, char** argv) { ::google::InitGoogleLogging(argv[0]); - if (argc != 3) { - LOG(ERROR) << "Usage: compute_image_mean input_leveldb output_file"; + if (argc < 3 || argc > 4) { + LOG(ERROR) << "Usage: compute_image_mean input_leveldb output_file" + << " db_backend[leveldb or lmdb]"; return 1; } + string db_backend = "leveldb"; + if (argc == 4) { + db_backend = string(argv[3]); + } + + // leveldb leveldb::DB* db; leveldb::Options options; options.create_if_missing = false; + leveldb::Iterator* it; + // lmdb + MDB_env* mdb_env; + MDB_dbi mdb_dbi; + MDB_val mdb_key, mdb_value; + MDB_txn* mdb_txn; + MDB_cursor* mdb_cursor; - LOG(INFO) << "Opening leveldb " << argv[1]; - leveldb::Status status = leveldb::DB::Open( - options, argv[1], &db); - CHECK(status.ok()) << "Failed to open leveldb " << argv[1]; + // Open db + if (db_backend == "leveldb") { // leveldb + LOG(INFO) << "Opening leveldb " << argv[1]; + leveldb::Status status = leveldb::DB::Open( + options, argv[1], &db); + CHECK(status.ok()) << "Failed to open leveldb " << argv[1]; + leveldb::ReadOptions read_options; + read_options.fill_cache = false; + it = db->NewIterator(read_options); + it->SeekToFirst(); + } else if (db_backend == "lmdb") { // lmdb + LOG(INFO) << "Opening lmdb " << argv[1]; + CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed"; + CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS); // 1TB + CHECK_EQ(mdb_env_open(mdb_env, argv[1], MDB_RDONLY, 0664), + MDB_SUCCESS) << "mdb_env_open failed"; + CHECK_EQ(mdb_txn_begin(mdb_env, NULL, MDB_RDONLY, &mdb_txn), MDB_SUCCESS) + << "mdb_txn_begin failed"; + CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS) + << "mdb_open failed"; + CHECK_EQ(mdb_cursor_open(mdb_txn, mdb_dbi, &mdb_cursor), MDB_SUCCESS) + << "mdb_cursor_open failed"; + CHECK_EQ(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_FIRST), + MDB_SUCCESS); + } else { + LOG(FATAL) << "Unknown db backend " << db_backend; + } - leveldb::ReadOptions read_options; - read_options.fill_cache = false; - leveldb::Iterator* it = db->NewIterator(read_options); - it->SeekToFirst(); Datum datum; BlobProto sum_blob; int count = 0; - datum.ParseFromString(it->value().ToString()); + // load first datum + if (db_backend == "leveldb") { + datum.ParseFromString(it->value().ToString()); + } else if (db_backend == "lmdb") { + datum.ParseFromArray(mdb_value.mv_data, mdb_value.mv_size); + } else { + LOG(FATAL) << "Unknown db backend " << db_backend; + } + sum_blob.set_num(1); sum_blob.set_channels(datum.channels()); sum_blob.set_height(datum.height()); @@ -49,28 +91,61 @@ int main(int argc, char** argv) { sum_blob.add_data(0.); } LOG(INFO) << "Starting Iteration"; - for (it->SeekToFirst(); it->Valid(); it->Next()) { - // just a dummy operation - datum.ParseFromString(it->value().ToString()); - const string& data = datum.data(); - size_in_datum = std::max(datum.data().size(), datum.float_data_size()); - CHECK_EQ(size_in_datum, data_size) << "Incorrect data field size " << - size_in_datum; - if (data.size() != 0) { - for (int i = 0; i < size_in_datum; ++i) { - sum_blob.set_data(i, sum_blob.data(i) + (uint8_t)data[i]); + if (db_backend == "leveldb") { // leveldb + for (it->SeekToFirst(); it->Valid(); it->Next()) { + // just a dummy operation + datum.ParseFromString(it->value().ToString()); + const string& data = datum.data(); + size_in_datum = std::max(datum.data().size(), + datum.float_data_size()); + CHECK_EQ(size_in_datum, data_size) << "Incorrect data field size " << + size_in_datum; + if (data.size() != 0) { + for (int i = 0; i < size_in_datum; ++i) { + sum_blob.set_data(i, sum_blob.data(i) + (uint8_t)data[i]); + } + } else { + for (int i = 0; i < size_in_datum; ++i) { + sum_blob.set_data(i, sum_blob.data(i) + + static_cast(datum.float_data(i))); + } } - } else { - for (int i = 0; i < size_in_datum; ++i) { - sum_blob.set_data(i, sum_blob.data(i) + - static_cast(datum.float_data(i))); + ++count; + if (count % 10000 == 0) { + LOG(ERROR) << "Processed " << count << " files."; } } - ++count; - if (count % 10000 == 0) { - LOG(ERROR) << "Processed " << count << " files."; - } + } else if (db_backend == "lmdb") { // lmdb + CHECK_EQ(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_FIRST), + MDB_SUCCESS); + do { + // just a dummy operation + datum.ParseFromArray(mdb_value.mv_data, mdb_value.mv_size); + const string& data = datum.data(); + size_in_datum = std::max(datum.data().size(), + datum.float_data_size()); + CHECK_EQ(size_in_datum, data_size) << "Incorrect data field size " << + size_in_datum; + if (data.size() != 0) { + for (int i = 0; i < size_in_datum; ++i) { + sum_blob.set_data(i, sum_blob.data(i) + (uint8_t)data[i]); + } + } else { + for (int i = 0; i < size_in_datum; ++i) { + sum_blob.set_data(i, sum_blob.data(i) + + static_cast(datum.float_data(i))); + } + } + ++count; + if (count % 10000 == 0) { + LOG(ERROR) << "Processed " << count << " files."; + } + } while (mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_NEXT) + == MDB_SUCCESS); + } else { + LOG(FATAL) << "Unknown db backend " << db_backend; } + if (count % 10000 != 0) { LOG(ERROR) << "Processed " << count << " files."; } @@ -81,6 +156,16 @@ int main(int argc, char** argv) { LOG(INFO) << "Write to " << argv[2]; WriteProtoToBinaryFile(sum_blob, argv[2]); - delete db; + // Clean up + if (db_backend == "leveldb") { + delete db; + } else if (db_backend == "lmdb") { + mdb_cursor_close(mdb_cursor); + mdb_close(mdb_env, mdb_dbi); + mdb_txn_abort(mdb_txn); + mdb_env_close(mdb_env); + } else { + LOG(FATAL) << "Unknown db backend " << db_backend; + } return 0; } From dbc5ebf31b529db5690dc0d1dad1498ff269e799 Mon Sep 17 00:00:00 2001 From: linmin Date: Sat, 14 Jun 2014 16:35:48 +0800 Subject: [PATCH 9/9] fix string compare error --- tools/convert_imageset.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/convert_imageset.cpp b/tools/convert_imageset.cpp index a381eaf4c03..aa9515514c3 100644 --- a/tools/convert_imageset.cpp +++ b/tools/convert_imageset.cpp @@ -67,7 +67,7 @@ int main(int argc, char** argv) { string db_backend = "leveldb"; if (argc >= (arg_offset+6)) { db_backend = string(argv[arg_offset+5]); - if (db_backend != "leveldb" || db_backend != "lmdb") { + if (!(db_backend == "leveldb") && !(db_backend == "lmdb")) { LOG(FATAL) << "Unknown db backend " << db_backend; } }