Util functions converting formats between HDF5 and Blob

Makefile

@@ -91,7 +91,7 @@ LIBRARIES := cudart cublas curand \
 	glog protobuf leveldb \
 	snappy \
 	boost_system \
-	hdf5_hl hdf5 \
+	hdf5_cpp hdf5_hl hdf5 \
 	opencv_core opencv_highgui opencv_imgproc
 PYTHON_LIBRARIES := boost_python python2.7
 WARNINGS := -Wall

docs/installation.md

@@ -43,6 +43,12 @@ You will also need other packages, most of which can be installed via apt-get us
 
     sudo apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libboost-all-dev libhdf5-serial-dev
 
+Note libhdf5-serial-dev does not include the C++ library. Download the latest source code from http://www.hdfgroup.org/HDF5/release/obtainsrc.html. Take the 1.8.12 version as an example, build the C++ library as follows.
+
+    axel -n 100 http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.12.tar.bz2
+    tar xvjf hdf5-1.8.12.tar.bz2  &&  cd hdf5-1.8.12  &&  mkdir build  &&  cd build
+    cmake -C ../config/cmake/cacheinit.cmake -G "Unix Makefiles" -DHDF5_ENABLE_SZIP_SUPPORT=OFF -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF -DHDF5_BUILD_CPP_LIB=ON -DBUILD_SHARED_LIBS=ON ..
+
 The only exception being the google logging library, which does not exist in the Ubuntu 12.04 repository. To install it, do:
 
     wget https://google-glog.googlecode.com/files/glog-0.3.3.tar.gz

include/caffe/util/format.hpp

@@ -0,0 +1,78 @@
+// Copyright 2014 kloudkl@github
+
+#ifndef CAFFE_FORMAT_IO_H_
+#define CAFFE_FORMAT_IO_H_
+
+#include <H5Cpp.h>
+
+#include "caffe/blob.hpp"
+
+namespace caffe {
+using namespace H5;
+using std::string;
+using std::vector;
+
+template<typename Dtype>
+int BlobToHDF5(const Blob<Dtype>& blob, const DataType hdf5_data_type,
+               const DataSpace& hdf5_data_space, DataSet* data_set) {
+  try {
+    Exception::dontPrint();
+    const Dtype* data = blob.cpu_data();
+    data_set->write(data, hdf5_data_type, hdf5_data_space, hdf5_data_space);
+  } catch(FileIException& error) {
+    error.printError();
+    return -1;
+  } catch(DataSetIException& error) {
+    error.printError();
+    return -2;
+  } catch(DataSpaceIException& error) {
+    error.printError();
+    return -3;
+  } catch (Exception& error) {
+    error.printError();
+    return -4;
+  } catch (...) {
+    return 1;
+  }
+  return 0;
+}
+
+template<typename Dtype>
+int HDF5ToBlob(const DataSet& data_set, const PredType hdf5_data_type,
+               Blob<Dtype>* blob) {
+  LOG(ERROR) << "HDF5ToBlob";
+  try {
+    Exception::dontPrint();
+    LOG(ERROR) << "HDF5ToBlob";
+    DataSpace dataspace = data_set.getSpace();
+    hsize_t dims_out[4];
+    LOG(ERROR) << "HDF5ToBlob";
+    int ndims = dataspace.getSimpleExtentDims(dims_out, NULL);
+    LOG(ERROR) << "blob->Reshape ";
+    LOG(ERROR) << dims_out[0] << dims_out[1]  << dims_out[2] << dims_out[3];
+    blob->Reshape(dims_out[0], dims_out[1], dims_out[2], dims_out[3]);
+    Dtype* data_out = blob->mutable_cpu_data();
+    data_set.read(data_out, hdf5_data_type, dataspace, dataspace);
+//    data_set.read(data_out, hdf5_data_type);
+  } catch(FileIException& error) {
+    error.printError();
+    return -1;
+  } catch(DataSetIException& error) {
+    error.printError();
+    return -2;
+  } catch(DataSpaceIException& error) {
+    error.printError();
+    return -3;
+  } catch (Exception& error) {
+    error.printError();
+    return -4;
+  } catch (...) {
+    LOG(ERROR) << "HDF5ToBlob Unknown error ";
+    return 1;
+  }
+  return 0;
+}
+
+}  // namespace caffe
+
+#endif   // CAFFE_FORMAT_IO_H_

include/caffe/util/io.hpp

@@ -5,18 +5,22 @@
 
 #include <string>
 
-#include "google/protobuf/message.h"
-#include "hdf5.h"
-#include "hdf5_hl.h"
-#include "caffe/proto/caffe.pb.h"
+#include <google/protobuf/message.h>
+#include <hdf5.h>
+#include <hdf5_hl.h>
+
+#include <H5Cpp.h>
+#include <boost/scoped_ptr.hpp>
 
-#include "boost/scoped_ptr.hpp"
+#include "caffe/util/format.hpp"
+#include "caffe/proto/caffe.pb.h"
 #include "caffe/blob.hpp"
 
 using std::string;
 using ::google::protobuf::Message;
 
 namespace caffe {
+using namespace H5;
 
 void ReadProtoFromTextFile(const char* filename,
     Message* proto);
@@ -61,6 +65,84 @@ void hd5_load_nd_dataset(
   std::vector<hsize_t>& dims
   );
 
+template <typename Dtype>
+int WriteBlobToHDF5File(
+    const Blob<Dtype>& blob, const string& hdf5_file,
+    const string& hdf5_dataset_name,  const PredType hdf5_data_type) {
+  LOG(ERROR) << "WriteBlobToHDF5File";
+  try {
+     Exception::dontPrint();
+     H5File* file = new H5File(hdf5_file, H5F_ACC_TRUNC);
+     int fill_value = 0;
+     DSetCreatPropList plist;
+     plist.setFillValue(hdf5_data_type, &fill_value);
+     int num = blob.num();
+     int channels = blob.channels();
+     int height = blob.height();
+     int width = blob.width();
+     const int num_dims = 4;
+     hsize_t dims[] = {num, channels, height, width};
+     DataSpace data_space(num_dims, dims);
+     DataSet* data_set = new DataSet(
+         file->createDataSet(hdf5_dataset_name, hdf5_data_type, data_space,
+                             plist));
+     BlobToHDF5(blob, hdf5_data_type, data_space, data_set);
+     file->close();
+     delete data_set;
+     delete file;
+   } catch(FileIException& error) {
+     error.printError();
+     return -1;
+   } catch(DataSetIException& error) {
+     error.printError();
+     return -2;
+   } catch(DataSpaceIException& error) {
+     error.printError();
+     return -3;
+   } catch (Exception& error) {
+     error.printError();
+     return -4;
+   } catch (...) {
+     return 1;
+   }
+   return 0;
+}
+
+template <typename Dtype>
+int ReadBlobFromHDF5File(const string& hdf5_file,
+                          const string& hdf5_dataset_name,
+                          Blob<Dtype>* blob) {
+  LOG(ERROR) << "ReadBlobFromHDF5File";
+  try {
+     Exception::dontPrint();
+     H5File file(hdf5_file, H5F_ACC_RDONLY);
+     DataSet data_set = file.openDataSet(hdf5_dataset_name);
+     LOG(ERROR) << "ReadBlobFromHDF5File";
+     PredType hdf5_data_type = PredType::NATIVE_FLOAT;
+     if (sizeof(Dtype) == sizeof(double)) {
+       hdf5_data_type = PredType::NATIVE_DOUBLE;
+     }
+     HDF5ToBlob(data_set, hdf5_data_type, blob);
+     LOG(ERROR) << "ReadBlobFromHDF5File";
+     file.close();
+   } catch(FileIException& error) {
+     error.printError();
+     return -1;
+   } catch(DataSetIException& error) {
+     error.printError();
+     return -2;
+   } catch(DataSpaceIException& error) {
+     error.printError();
+     return -3;
+   } catch (Exception& error) {
+     error.printError();
+     return -4;
+   } catch (...) {
+     return 1;
+   }
+   return 0;
+}
+
 }  // namespace caffe
 
 #endif   // CAFFE_UTIL_IO_H_

src/caffe/test/test_io.cpp

@@ -0,0 +1,76 @@
+// Copyright 2014 kloudkl@github
+
+#include <H5Cpp.h>
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/common.hpp"
+#include "caffe/filler.hpp"
+#include "caffe/util/io.hpp"
+#include "caffe/proto/caffe.pb.h"
+#include "caffe/test/test_caffe_main.hpp"
+#include "gtest/gtest.h"
+
+namespace caffe {
+using namespace H5;
+
+template <typename Dtype>
+class IOTest : public ::testing::Test {
+ protected:
+  IOTest(): num_(11), channels_(17), height_(19),
+  width_(23), hdf5_file_name_("src/caffe/test/test_data/write_blob.h5"),
+  hdf5_dataset_name_("hdn"), data_type_(PredType::NATIVE_FLOAT),
+  blob_(num_, channels_, height_, width_) {}
+
+  virtual void SetUp() {
+    FillerParameter filler_param;
+    GaussianFiller<Dtype> filler(filler_param);
+    filler.Fill(&blob_);
+    if (sizeof(Dtype) == sizeof(float)) {
+      data_type_ = PredType::NATIVE_FLOAT;
+    } else {
+      data_type_ = PredType::NATIVE_DOUBLE;
+    }
+  }
+
+  virtual ~IOTest() {}
+
+  int num_;
+  int channels_;
+  int height_ ;
+  int width_;
+  string hdf5_file_name_;
+  string hdf5_dataset_name_;
+  PredType data_type_;
+  Blob<Dtype> blob_;
+};
+
+typedef ::testing::Types<float, double> Dtypes;
+TYPED_TEST_CASE(IOTest, Dtypes);
+
+TYPED_TEST(IOTest, TestWriteAndReadBlobToHDF5File) {
+  WriteBlobToHDF5File<TypeParam>(this->blob_, this->hdf5_file_name_,
+                      this->hdf5_dataset_name_, this->data_type_);
+  Blob<TypeParam> blob;
+  ReadBlobFromHDF5File<TypeParam>(
+      this->hdf5_file_name_, this->hdf5_dataset_name_, &blob);
+  EXPECT_EQ(blob.num(), this->num_);
+  EXPECT_EQ(blob.channels(), this->channels_);
+  EXPECT_EQ(blob.height(), this->height_);
+  EXPECT_EQ(blob.width(), this->width_);
+  const TypeParam* data_gt = this->blob_.cpu_data();
+  const TypeParam* data = blob.cpu_data();
+  int idx = 0;
+  for (int i = 0; i < this->num_; ++i) {
+    for (int j = 0; j < this->channels_; ++j) {
+      for (int h = 0; h < this->height_; ++h) {
+        for (int w = 0; w < this->width_; ++w, ++idx) {
+          EXPECT_EQ(data[idx], data_gt[idx])
+              << "debug: i " << i << " j " << j << " h " << h << " w " << w;
+        }
+      }
+    }
+  }
+}
+
+}  // namespace caffe

src/caffe/util/format.cpp

@@ -0,0 +1,24 @@
+// Copyright 2014 kloudkl@github
+
+#include <H5Cpp.h>
+#include <opencv2/opencv.hpp>
+
+#include "caffe/util/format.hpp"
+
+namespace caffe {
+
+template <>
+int BlobToHDF5<float>(const Blob<float>& blob, const DataType hdf5_data_type,
+               const DataSpace& hdf5_data_space, DataSet* data_set);
+template <>
+int BlobToHDF5<double>(const Blob<double>& blob, const DataType hdf5_data_type,
+               const DataSpace& hdf5_data_space, DataSet* data_set);
+
+template<>
+int HDF5ToBlob<float>(const DataSet& data_set, const PredType hdf5_data_type,
+Blob<float>* blob);
+template<>
+int HDF5ToBlob<double>(const DataSet& data_set, const PredType hdf5_data_type,
+                       Blob<double>* blob);
+
+}  // namespace caffe

src/caffe/util/io.cpp

@@ -158,4 +158,23 @@ void hd5_load_nd_dataset<double>(hid_t file_id, const char* dataset_name_,
         file_id, dataset_name_, array->get());
 }
 
+template <>
+int WriteBlobToHDF5File<float>(
+    const Blob<float>& blob, const string& hdf5_file,
+      const string& hdf5_dataset_name, const PredType hdf5_data_type);
+template <>
+int WriteBlobToHDF5File<double>(
+    const Blob<double>& blob, const string& hdf5_file,
+      const string& hdf5_dataset_name, const PredType hdf5_data_type);
+
+template <>
+int ReadBlobFromHDF5File<float>(
+    const string& hdf5_file, const string& hdf5_dataset_name,
+    Blob<float>* blob);
+template <>
+int ReadBlobFromHDF5File<double>(
+    const string& hdf5_file, const string& hdf5_dataset_name,
+    Blob<double>* blob);
+
+
 }  // namespace caffe