diff --git a/Makefile b/Makefile index 54437f1e914..d40b1873ae7 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ LIBRARIES := cudart cublas curand \ glog protobuf leveldb \ snappy \ boost_system \ - hdf5_hl hdf5 \ + hdf5_cpp hdf5_hl hdf5 \ opencv_core opencv_highgui opencv_imgproc PYTHON_LIBRARIES := boost_python python2.7 WARNINGS := -Wall diff --git a/docs/installation.md b/docs/installation.md index e115f4853ab..3640706ac0a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -43,6 +43,12 @@ You will also need other packages, most of which can be installed via apt-get us sudo apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libboost-all-dev libhdf5-serial-dev +Note libhdf5-serial-dev does not include the C++ library. Download the latest source code from http://www.hdfgroup.org/HDF5/release/obtainsrc.html. Take the 1.8.12 version as an example, build the C++ library as follows. + + axel -n 100 http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.12.tar.bz2 + tar xvjf hdf5-1.8.12.tar.bz2 && cd hdf5-1.8.12 && mkdir build && cd build + cmake -C ../config/cmake/cacheinit.cmake -G "Unix Makefiles" -DHDF5_ENABLE_SZIP_SUPPORT=OFF -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF -DHDF5_BUILD_CPP_LIB=ON -DBUILD_SHARED_LIBS=ON .. + The only exception being the google logging library, which does not exist in the Ubuntu 12.04 repository. To install it, do: wget https://google-glog.googlecode.com/files/glog-0.3.3.tar.gz diff --git a/include/caffe/util/format.hpp b/include/caffe/util/format.hpp new file mode 100644 index 00000000000..1e3f938072d --- /dev/null +++ b/include/caffe/util/format.hpp @@ -0,0 +1,78 @@ +// Copyright 2014 kloudkl@github + +#ifndef CAFFE_FORMAT_IO_H_ +#define CAFFE_FORMAT_IO_H_ + +#include + +#include "caffe/blob.hpp" + +namespace caffe { +using namespace H5; +using std::string; +using std::vector; + +template +int BlobToHDF5(const Blob& blob, const DataType hdf5_data_type, + const DataSpace& hdf5_data_space, DataSet* data_set) { + try { + Exception::dontPrint(); + const Dtype* data = blob.cpu_data(); + data_set->write(data, hdf5_data_type, hdf5_data_space, hdf5_data_space); + } catch(FileIException& error) { + error.printError(); + return -1; + } catch(DataSetIException& error) { + error.printError(); + return -2; + } catch(DataSpaceIException& error) { + error.printError(); + return -3; + } catch (Exception& error) { + error.printError(); + return -4; + } catch (...) { + return 1; + } + return 0; +} + +template +int HDF5ToBlob(const DataSet& data_set, const PredType hdf5_data_type, + Blob* blob) { + LOG(ERROR) << "HDF5ToBlob"; + try { + Exception::dontPrint(); + LOG(ERROR) << "HDF5ToBlob"; + DataSpace dataspace = data_set.getSpace(); + hsize_t dims_out[4]; + LOG(ERROR) << "HDF5ToBlob"; + int ndims = dataspace.getSimpleExtentDims(dims_out, NULL); + LOG(ERROR) << "blob->Reshape "; + LOG(ERROR) << dims_out[0] << dims_out[1] << dims_out[2] << dims_out[3]; + blob->Reshape(dims_out[0], dims_out[1], dims_out[2], dims_out[3]); + Dtype* data_out = blob->mutable_cpu_data(); + data_set.read(data_out, hdf5_data_type, dataspace, dataspace); +// data_set.read(data_out, hdf5_data_type); + } catch(FileIException& error) { + error.printError(); + return -1; + } catch(DataSetIException& error) { + error.printError(); + return -2; + } catch(DataSpaceIException& error) { + error.printError(); + return -3; + } catch (Exception& error) { + error.printError(); + return -4; + } catch (...) { + LOG(ERROR) << "HDF5ToBlob Unknown error "; + return 1; + } + return 0; +} + +} // namespace caffe + +#endif // CAFFE_FORMAT_IO_H_ diff --git a/include/caffe/util/io.hpp b/include/caffe/util/io.hpp index 269fbd9df0b..85ef255fbe6 100644 --- a/include/caffe/util/io.hpp +++ b/include/caffe/util/io.hpp @@ -5,18 +5,22 @@ #include -#include "google/protobuf/message.h" -#include "hdf5.h" -#include "hdf5_hl.h" -#include "caffe/proto/caffe.pb.h" +#include +#include +#include + +#include +#include -#include "boost/scoped_ptr.hpp" +#include "caffe/util/format.hpp" +#include "caffe/proto/caffe.pb.h" #include "caffe/blob.hpp" using std::string; using ::google::protobuf::Message; namespace caffe { +using namespace H5; void ReadProtoFromTextFile(const char* filename, Message* proto); @@ -61,6 +65,84 @@ void hd5_load_nd_dataset( std::vector& dims ); +template +int WriteBlobToHDF5File( + const Blob& blob, const string& hdf5_file, + const string& hdf5_dataset_name, const PredType hdf5_data_type) { + LOG(ERROR) << "WriteBlobToHDF5File"; + try { + Exception::dontPrint(); + H5File* file = new H5File(hdf5_file, H5F_ACC_TRUNC); + int fill_value = 0; + DSetCreatPropList plist; + plist.setFillValue(hdf5_data_type, &fill_value); + int num = blob.num(); + int channels = blob.channels(); + int height = blob.height(); + int width = blob.width(); + const int num_dims = 4; + hsize_t dims[] = {num, channels, height, width}; + DataSpace data_space(num_dims, dims); + DataSet* data_set = new DataSet( + file->createDataSet(hdf5_dataset_name, hdf5_data_type, data_space, + plist)); + BlobToHDF5(blob, hdf5_data_type, data_space, data_set); + file->close(); + delete data_set; + delete file; + } catch(FileIException& error) { + error.printError(); + return -1; + } catch(DataSetIException& error) { + error.printError(); + return -2; + } catch(DataSpaceIException& error) { + error.printError(); + return -3; + } catch (Exception& error) { + error.printError(); + return -4; + } catch (...) { + return 1; + } + return 0; +} + +template +int ReadBlobFromHDF5File(const string& hdf5_file, + const string& hdf5_dataset_name, + Blob* blob) { + LOG(ERROR) << "ReadBlobFromHDF5File"; + try { + Exception::dontPrint(); + H5File file(hdf5_file, H5F_ACC_RDONLY); + DataSet data_set = file.openDataSet(hdf5_dataset_name); + LOG(ERROR) << "ReadBlobFromHDF5File"; + PredType hdf5_data_type = PredType::NATIVE_FLOAT; + if (sizeof(Dtype) == sizeof(double)) { + hdf5_data_type = PredType::NATIVE_DOUBLE; + } + HDF5ToBlob(data_set, hdf5_data_type, blob); + LOG(ERROR) << "ReadBlobFromHDF5File"; + file.close(); + } catch(FileIException& error) { + error.printError(); + return -1; + } catch(DataSetIException& error) { + error.printError(); + return -2; + } catch(DataSpaceIException& error) { + error.printError(); + return -3; + } catch (Exception& error) { + error.printError(); + return -4; + } catch (...) { + return 1; + } + return 0; +} + } // namespace caffe #endif // CAFFE_UTIL_IO_H_ diff --git a/src/caffe/test/test_io.cpp b/src/caffe/test/test_io.cpp new file mode 100644 index 00000000000..b342220f51b --- /dev/null +++ b/src/caffe/test/test_io.cpp @@ -0,0 +1,76 @@ +// Copyright 2014 kloudkl@github + +#include +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/util/io.hpp" +#include "caffe/proto/caffe.pb.h" +#include "caffe/test/test_caffe_main.hpp" +#include "gtest/gtest.h" + +namespace caffe { +using namespace H5; + +template +class IOTest : public ::testing::Test { + protected: + IOTest(): num_(11), channels_(17), height_(19), + width_(23), hdf5_file_name_("src/caffe/test/test_data/write_blob.h5"), + hdf5_dataset_name_("hdn"), data_type_(PredType::NATIVE_FLOAT), + blob_(num_, channels_, height_, width_) {} + + virtual void SetUp() { + FillerParameter filler_param; + GaussianFiller filler(filler_param); + filler.Fill(&blob_); + if (sizeof(Dtype) == sizeof(float)) { + data_type_ = PredType::NATIVE_FLOAT; + } else { + data_type_ = PredType::NATIVE_DOUBLE; + } + } + + virtual ~IOTest() {} + + int num_; + int channels_; + int height_ ; + int width_; + string hdf5_file_name_; + string hdf5_dataset_name_; + PredType data_type_; + Blob blob_; +}; + +typedef ::testing::Types Dtypes; +TYPED_TEST_CASE(IOTest, Dtypes); + +TYPED_TEST(IOTest, TestWriteAndReadBlobToHDF5File) { + WriteBlobToHDF5File(this->blob_, this->hdf5_file_name_, + this->hdf5_dataset_name_, this->data_type_); + Blob blob; + ReadBlobFromHDF5File( + this->hdf5_file_name_, this->hdf5_dataset_name_, &blob); + EXPECT_EQ(blob.num(), this->num_); + EXPECT_EQ(blob.channels(), this->channels_); + EXPECT_EQ(blob.height(), this->height_); + EXPECT_EQ(blob.width(), this->width_); + const TypeParam* data_gt = this->blob_.cpu_data(); + const TypeParam* data = blob.cpu_data(); + int idx = 0; + for (int i = 0; i < this->num_; ++i) { + for (int j = 0; j < this->channels_; ++j) { + for (int h = 0; h < this->height_; ++h) { + for (int w = 0; w < this->width_; ++w, ++idx) { + EXPECT_EQ(data[idx], data_gt[idx]) + << "debug: i " << i << " j " << j << " h " << h << " w " << w; + } + } + } + } +} + +} // namespace caffe diff --git a/src/caffe/util/format.cpp b/src/caffe/util/format.cpp new file mode 100644 index 00000000000..f2c6181216c --- /dev/null +++ b/src/caffe/util/format.cpp @@ -0,0 +1,24 @@ +// Copyright 2014 kloudkl@github + +#include +#include + +#include "caffe/util/format.hpp" + +namespace caffe { + +template <> +int BlobToHDF5(const Blob& blob, const DataType hdf5_data_type, + const DataSpace& hdf5_data_space, DataSet* data_set); +template <> +int BlobToHDF5(const Blob& blob, const DataType hdf5_data_type, + const DataSpace& hdf5_data_space, DataSet* data_set); + +template<> +int HDF5ToBlob(const DataSet& data_set, const PredType hdf5_data_type, +Blob* blob); +template<> +int HDF5ToBlob(const DataSet& data_set, const PredType hdf5_data_type, + Blob* blob); + +} // namespace caffe diff --git a/src/caffe/util/io.cpp b/src/caffe/util/io.cpp index c301c55bba1..2ba07d9a98c 100644 --- a/src/caffe/util/io.cpp +++ b/src/caffe/util/io.cpp @@ -158,4 +158,23 @@ void hd5_load_nd_dataset(hid_t file_id, const char* dataset_name_, file_id, dataset_name_, array->get()); } +template <> +int WriteBlobToHDF5File( + const Blob& blob, const string& hdf5_file, + const string& hdf5_dataset_name, const PredType hdf5_data_type); +template <> +int WriteBlobToHDF5File( + const Blob& blob, const string& hdf5_file, + const string& hdf5_dataset_name, const PredType hdf5_data_type); + +template <> +int ReadBlobFromHDF5File( + const string& hdf5_file, const string& hdf5_dataset_name, + Blob* blob); +template <> +int ReadBlobFromHDF5File( + const string& hdf5_file, const string& hdf5_dataset_name, + Blob* blob); + + } // namespace caffe