diff --git a/include/caffe/net.hpp b/include/caffe/net.hpp index 075afebc9b0..167541c4506 100644 --- a/include/caffe/net.hpp +++ b/include/caffe/net.hpp @@ -23,12 +23,15 @@ namespace caffe { template class Net { public: - explicit Net(const NetParameter& param); - explicit Net(const string& param_file, Phase phase); + explicit Net(const NetParameter& param, + const size_t force_input_count = 0); + explicit Net(const string& param_file, Phase phase, + const size_t force_input_count = 0); virtual ~Net() {} /// @brief Initialize a network with a NetParameter. - void Init(const NetParameter& param); + void Init(const NetParameter& param, + const size_t force_input_count = 0); /** * @brief Run Forward with the input Blob%s already fed separately. @@ -256,6 +259,8 @@ class Net { /// Whether to compute and display debug info for the net. bool debug_info_; + size_t force_input_count_; + DISABLE_COPY_AND_ASSIGN(Net); }; diff --git a/matlab/caffe/matcaffe.cpp b/matlab/caffe/matcaffe.cpp index da37d920b20..344d7a9ce88 100644 --- a/matlab/caffe/matcaffe.cpp +++ b/matlab/caffe/matcaffe.cpp @@ -3,6 +3,7 @@ // caffe::Caffe functions so that one could easily call it from matlab. // Note that for matlab, we will simply use float as the data type. +#include #include #include #include @@ -22,9 +23,79 @@ inline void mex_error(const std::string &msg) { using namespace caffe; // NOLINT(build/namespaces) // The pointer to the internal caffe::Net instance +// used by default unless a net reference is passed in static shared_ptr > net_; static int init_key = -2; +// Storage for nets by reference +std::map > > nets_; +pthread_mutex_t mutex_add_net_; + +/** ----------------------------------------------------------------- + ** net handle management + **/ + +static uint16_t add_net(shared_ptr >* net, uint16_t handle = 0) { + static uint16_t handle_ctr = 0; + + if (handle < 1) { + pthread_mutex_lock(&mutex_add_net_); + uint16_t next_handle = ++handle_ctr; + pthread_mutex_unlock(&mutex_add_net_); + + nets_[next_handle] = (*net); + + CHECK_GE(next_handle, 1); + return next_handle; + } else { + nets_[handle] = (*net); + return handle; + } +} + +static shared_ptr > get_net(const uint16_t handle = 0) { + if (handle == 0) { + return net_; + } else { + std::map > >::iterator it; + it = nets_.find(handle); + + if (it == nets_.end()) { + mexErrMsgTxt("Invalid net handle"); + } + + shared_ptr > net = it->second; + CHECK(net) << "Returned null pointer for lookup: " << handle; + + return net; + } +} + +static void remove_net(const uint16_t handle) { + CHECK_GE(handle, 1); + + std::map > >::iterator it; + it = nets_.find(handle); + + if (it == nets_.end()) { + mexErrMsgTxt("Invalid net handle"); + } + + nets_.erase(it); +} + +static uint16_t get_handle(const mxArray* array) { + if (!mxIsUint16(array)) { + mexErrMsgTxt("Handle should be of type uint16_t"); + } + uint16_t* handle_arr = static_cast(mxGetData(array)); + return handle_arr[0]; +} + +/** ----------------------------------------------------------------- + ** main module functions + **/ + // Five things to be aware of: // caffe uses row-major order // matlab uses column-major order @@ -51,8 +122,15 @@ static int init_key = -2; // The actual forward function. It takes in a cell array of 4-D arrays as // input and outputs a cell array. -static mxArray* do_forward(const mxArray* const bottom) { - const vector*>& input_blobs = net_->input_blobs(); +static mxArray* do_forward(const mxArray* const bottom, + const std::vector& blob_names = + std::vector(), + shared_ptr > net = net_) { + if (!net) { + mex_error("Calling 'forward' on an uninitialized net - call 'init' first"); + } + + const vector*>& input_blobs = net->input_blobs(); if (static_cast(mxGetDimensions(bottom)[0]) != input_blobs.size()) { mex_error("Invalid input size"); @@ -84,36 +162,66 @@ static mxArray* do_forward(const mxArray* const bottom) { mex_error("Unknown Caffe mode"); } // switch (Caffe::mode()) } - const vector*>& output_blobs = net_->ForwardPrefilled(); - mxArray* mx_out = mxCreateCellMatrix(output_blobs.size(), 1); - for (unsigned int i = 0; i < output_blobs.size(); ++i) { + + // do forward + const vector*>& output_blobs = net->ForwardPrefilled(); + + // pointer to store blobs for return + const vector*>* result_blobs = 0; + + const bool retrieve_custom_blobs = (!blob_names.empty()); + vector*> extract_blobs(blob_names.size()); + + if (!retrieve_custom_blobs) { + // by default, only output blobs will be returned + result_blobs = &output_blobs; + } else { + for (size_t i = 0; i < blob_names.size(); ++i) { + const shared_ptr > custom_output_blob = + net->blob_by_name(blob_names[i]); + extract_blobs[i] = const_cast*>(custom_output_blob.get()); + } + // but get pointers to requested custom output blobs if specified + result_blobs = &extract_blobs; + } + + mxArray* mx_out = mxCreateCellMatrix(result_blobs->size(), 1); + + for (unsigned int i = 0; i < result_blobs->size(); ++i) { // internally data is stored as (width, height, channels, num) // where width is the fastest dimension - mwSize dims[4] = {output_blobs[i]->width(), output_blobs[i]->height(), - output_blobs[i]->channels(), output_blobs[i]->num()}; + const vector*>& result_blobs_ref = *result_blobs; + + mwSize dims[4] = {result_blobs_ref[i]->width(), + result_blobs_ref[i]->height(), + result_blobs_ref[i]->channels(), + result_blobs_ref[i]->num()}; mxArray* mx_blob = mxCreateNumericArray(4, dims, mxSINGLE_CLASS, mxREAL); - mxSetCell(mx_out, i, mx_blob); + float* data_ptr = reinterpret_cast(mxGetPr(mx_blob)); switch (Caffe::mode()) { case Caffe::CPU: - caffe_copy(output_blobs[i]->count(), output_blobs[i]->cpu_data(), + caffe_copy(result_blobs_ref[i]->count(), result_blobs_ref[i]->cpu_data(), data_ptr); break; case Caffe::GPU: - caffe_copy(output_blobs[i]->count(), output_blobs[i]->gpu_data(), + caffe_copy(result_blobs_ref[i]->count(), result_blobs_ref[i]->gpu_data(), data_ptr); break; default: mex_error("Unknown Caffe mode"); } // switch (Caffe::mode()) + + mxSetCell(mx_out, i, mx_blob); } return mx_out; } -static mxArray* do_backward(const mxArray* const top_diff) { - const vector*>& output_blobs = net_->output_blobs(); - const vector*>& input_blobs = net_->input_blobs(); +static mxArray* do_backward(const mxArray* const top_diff, + shared_ptr > net = net_) { + const vector*>& output_blobs = net->output_blobs(); + const vector*>& input_blobs = net->input_blobs(); if (static_cast(mxGetDimensions(top_diff)[0]) != output_blobs.size()) { mex_error("Invalid input size"); @@ -137,7 +245,7 @@ static mxArray* do_backward(const mxArray* const top_diff) { } // switch (Caffe::mode()) } // LOG(INFO) << "Start"; - net_->Backward(); + net->Backward(); // LOG(INFO) << "End"; mxArray* mx_out = mxCreateCellMatrix(input_blobs.size(), 1); for (unsigned int i = 0; i < input_blobs.size(); ++i) { @@ -163,9 +271,9 @@ static mxArray* do_backward(const mxArray* const top_diff) { return mx_out; } -static mxArray* do_get_weights() { - const vector > >& layers = net_->layers(); - const vector& layer_names = net_->layer_names(); +static mxArray* do_get_weights(shared_ptr > net = net_) { + const vector > >& layers = net->layers(); + const vector& layer_names = net->layer_names(); // Step 1: count the number of layers with weights int num_layers = 0; @@ -243,7 +351,13 @@ static mxArray* do_get_weights() { } static void get_weights(MEX_ARGS) { - plhs[0] = do_get_weights(); + uint16_t handle = 0; + if ((nrhs >= 1) && (!mxIsEmpty(prhs[0]))) { + handle = get_handle(prhs[0]); + } + shared_ptr > net = get_net(handle); + + plhs[0] = do_get_weights(net); } static void set_mode_cpu(MEX_ARGS) { @@ -270,16 +384,36 @@ static void get_init_key(MEX_ARGS) { } static void init(MEX_ARGS) { - if (nrhs != 3) { + if ((nrhs < 3) || (nrhs > 5)) { ostringstream error_msg; - error_msg << "Expected 3 arguments, got " << nrhs; + error_msg << "Wrong number of arguments. Usage: caffe('init', " + << "model_def_file, model_file, phase_name," + << "[, input_count=<0=from model>, net_handle=<0>])" + << "Where net_handle=N and " + << "N=0 (use singleton global net instance), " + << "N=-1 (create new net instance and return handle), " + << "N>0 (reinitialize net instance with handle N)"; mex_error(error_msg.str()); } + const int32_t GLOBAL_NET = 0; + const int32_t CREATE_NET = -1; + + // Handle input params char* param_file = mxArrayToString(prhs[0]); char* model_file = mxArrayToString(prhs[1]); char* phase_name = mxArrayToString(prhs[2]); + size_t input_count = 0; + int32_t net_handle = 0; + if (nrhs >= 4) { + input_count = mxGetScalar(prhs[3]); + } + if (nrhs >= 5) { + net_handle = static_cast(mxGetScalar(prhs[4])); + } + + // Get phase Phase phase; if (strcmp(phase_name, "train") == 0) { phase = TRAIN; @@ -289,50 +423,154 @@ static void init(MEX_ARGS) { mex_error("Unknown phase."); } - net_.reset(new Net(string(param_file), phase)); - net_->CopyTrainedLayersFrom(string(model_file)); + // Handle output params + + if (((net_handle == GLOBAL_NET) || (net_handle > 0)) && (nlhs != 0)) + mexErrMsgTxt("0 output arugments if net_handle>=0" + "(use global net/existing handle)"); + if (net_handle == CREATE_NET) { + if (nlhs != 1) { + LOG(ERROR) << "Must provide output arugment if using net_handle<0" + << " to store the created net pointer"; + mexErrMsgTxt("Wrong number of output arugments"); + } + } + + LOG(INFO) << "Starting instantiation..."; + // Instantiate a new net + if (net_handle == GLOBAL_NET) { + LOG(INFO) << "Instantiating global net..."; + net_.reset(new Net(string(param_file), phase, input_count)); + net_->CopyTrainedLayersFrom(string(model_file)); + + init_key = random(); // NOLINT(caffe/random_fn) + + if (nlhs == 1) { + plhs[0] = mxCreateDoubleScalar(init_key); + } + } else if (net_handle == CREATE_NET) { + shared_ptr > + net(new Net(string(param_file), + phase, input_count)); + net->CopyTrainedLayersFrom(string(model_file)); + + uint16_t* created_handle; + mwSize dims[3] = {1, 1}; + plhs[0] = mxCreateNumericArray(2, dims, mxUINT16_CLASS, mxREAL); + created_handle = static_cast(mxGetData(plhs[0])); + + (*created_handle) = add_net(&net); + } else if (net_handle > 0) { + shared_ptr > + net(new Net(string(param_file), + phase, input_count)); + net->CopyTrainedLayersFrom(string(model_file)); + + add_net(&net, net_handle); + + uint16_t* return_handle; + mwSize dims[3] = {1, 1}; + if (nlhs == 1) { + plhs[0] = mxCreateNumericArray(2, dims, mxUINT16_CLASS, mxREAL); + return_handle = static_cast(mxGetData(plhs[0])); + (*return_handle) = net_handle; + } + } else { + mex_error("Unrecognized value for net_handle"); + } mxFree(param_file); mxFree(model_file); mxFree(phase_name); +} - init_key = random(); // NOLINT(caffe/random_fn) +static void reset(MEX_ARGS) { + if ((nrhs != 0) || (nrhs != 1)) { + mex_error("Wrong number of arguments. " + "Usage: caffe('reset'[, net_handle=]"); + } - if (nlhs == 1) { - plhs[0] = mxCreateDoubleScalar(init_key); + uint16_t handle = 0; + if ((nrhs >= 1) && (!mxIsEmpty(prhs[0]))) { + handle = get_handle(prhs[0]); } -} + shared_ptr > net = get_net(handle); -static void reset(MEX_ARGS) { - if (net_) { - net_.reset(); + if (net) { + net.reset(); init_key = -2; LOG(INFO) << "Network reset, call init before use it again"; } } static void forward(MEX_ARGS) { - if (nrhs != 1) { + if ((nrhs < 1) || (nrhs > 3)) { ostringstream error_msg; - error_msg << "Expected 1 argument, got " << nrhs; + error_msg << "Wrong number of arguments. Usage: caffe('forward', images" + << "[, blob_names=<[]=final>, net_handle=])"; mex_error(error_msg.str()); } - plhs[0] = do_forward(prhs[0]); + // Handle params + std::vector blob_names; + uint16_t handle = 0; + + if ((nrhs >= 2) && (!mxIsEmpty(prhs[1]))) { + if (!mxIsCell(prhs[1])) { + mex_error("blob_names should be a cell array of strings"); + } + size_t blob_count = mxGetNumberOfElements(prhs[1]); + for (size_t i = 0; i < blob_count; ++i) { + blob_names.push_back(std::string(mxArrayToString(mxGetCell(prhs[1], i)))); + } + } + + if ((nrhs >= 3) && (!mxIsEmpty(prhs[2]))) { + handle = get_handle(prhs[2]); + } + + shared_ptr > net = get_net(handle); + if (!net) { + mex_error("Net was uninitialized in call to 'forward'"); + } + + // Do forward + plhs[0] = do_forward(prhs[0], blob_names, net); } static void backward(MEX_ARGS) { - if (nrhs != 1) { - ostringstream error_msg; - error_msg << "Expected 1 argument, got " << nrhs; - mex_error(error_msg.str()); + if ((nrhs < 1) && (nrhs > 2)) { + mex_error("Wrong number of arguments. " + "Usage: caffe('backward', top_diff" + "[, net_handle=])"); } - plhs[0] = do_backward(prhs[0]); + uint16_t handle = 0; + if ((nrhs >= 2) && (!mxIsEmpty(prhs[1]))) { + handle = get_handle(prhs[0]); + } + shared_ptr > net = get_net(handle); + if (!net) { + mex_error("Net was uninitialized in call to 'backward'"); + } + + plhs[0] = do_backward(prhs[0], net); } static void is_initialized(MEX_ARGS) { - if (!net_) { + if ((nrhs != 0) && (nrhs != 1)) { + mex_error("Wrong number of arguments. " + "Usage: caffe('is_initialized'" + "[, net_handle=])"); + } + + uint16_t handle = 0; + if ((nrhs >= 1) && (!mxIsEmpty(prhs[0]))) { + handle = get_handle(prhs[0]); + } + shared_ptr > net = get_net(handle); + + if (!net) { plhs[0] = mxCreateDoubleScalar(0); } else { plhs[0] = mxCreateDoubleScalar(1); @@ -341,8 +579,9 @@ static void is_initialized(MEX_ARGS) { static void read_mean(MEX_ARGS) { if (nrhs != 1) { - mexErrMsgTxt("Usage: caffe('read_mean', 'path_to_binary_mean_file'"); - return; + mex_error("Wrong number of arguments. " + "Usage: caffe('read_mean', " + "'path_to_binary_mean_file')"); } const string& mean_file = mxArrayToString(prhs[0]); Blob data_mean; @@ -350,7 +589,7 @@ static void read_mean(MEX_ARGS) { BlobProto blob_proto; bool result = ReadProtoFromBinaryFile(mean_file.c_str(), &blob_proto); if (!result) { - mexErrMsgTxt("Couldn't read the file"); + mex_error("Couldn't read mean file"); return; } data_mean.FromProto(blob_proto); @@ -364,6 +603,87 @@ static void read_mean(MEX_ARGS) { plhs[0] = mx_blob; } +static void get_backend(MEX_ARGS) { + if ((nlhs != 1) || (nrhs != 0)) { + mex_error("Wrong number of arguments. " + "Usage: backend = caffe('get_backend')"); + } + + switch (Caffe::mode()) { + case Caffe::CPU: + plhs[0] = mxCreateString("cpu"); + break; + case Caffe::GPU: + plhs[0] = mxCreateString("gpu"); + break; + default: + mex_error("Could not recognize backend"); + } +} + +static void get_blob_size(MEX_ARGS) { + if (((nrhs != 1) && (nrhs != 2)) || (nlhs != 1)) { + mex_error("Wrong number of arguments. " + "Usage: size = caffe('get_blob_size', " + "blob_names=<[]=final>[, net_handle=])"); + } + + // Handle params + std::vector blob_names; + uint16_t handle = 0; + + if ((nrhs >= 2) && (!mxIsEmpty(prhs[1]))) { + handle = get_handle(prhs[1]); + } + + shared_ptr > net = get_net(handle); + if (!net) { + mex_error("Net was uninitialized in call to 'get_blob_size'"); + } + + if ((nrhs >= 1) && (!mxIsEmpty(prhs[0]))) { + if (!mxIsCell(prhs[0])) { + mex_error("blob_names should be a cell array of strings"); + } + size_t blob_count = mxGetNumberOfElements(prhs[0]); + for (size_t i = 0; i < blob_count; ++i) { + std::string blob_name = + std::string(mxArrayToString(mxGetCell(prhs[0], i))); + blob_names.push_back(blob_name); + } + } else { + const vector& all_blob_names = net->blob_names(); + blob_names.push_back(all_blob_names[all_blob_names.size()-1]); + } + + // Get named blob size + plhs[0] = mxCreateCellMatrix(blob_names.size(), 1); + + for (size_t i = 0; i < blob_names.size(); ++i) { + const shared_ptr > custom_output_blob = + net->blob_by_name(blob_names[i]); + mxArray* mx_blob_sz = + mxCreateNumericMatrix(4, 1, mxDOUBLE_CLASS, mxREAL); + double* data = static_cast(mxGetData(mx_blob_sz)); + data[0] = custom_output_blob->num(); + data[1] = custom_output_blob->channels(); + data[2] = custom_output_blob->height(); + data[3] = custom_output_blob->width(); + mxSetCell(plhs[0], i, mx_blob_sz); + } +} + +static void destroy_net_by_handle(MEX_ARGS) { + if (nrhs != 1) { + mex_error("Wrong number of arguments. " + "Usage: caffe('destroy_net_by_handle', " + "net_handle)"); + } + + remove_net(get_handle(prhs[0])); +} + + /** ----------------------------------------------------------------- ** Available commands. **/ @@ -385,6 +705,10 @@ static handler_registry handlers[] = { { "get_init_key", get_init_key }, { "reset", reset }, { "read_mean", read_mean }, + { "get_backend", get_backend }, + { "get_blob_size", get_blob_size }, + // Custom handle management + { "destroy_net_by_handle", destroy_net_by_handle}, // The end. { "END", NULL }, }; diff --git a/src/caffe/net.cpp b/src/caffe/net.cpp index e8f7c05e09d..9e60f2903a9 100644 --- a/src/caffe/net.cpp +++ b/src/caffe/net.cpp @@ -19,22 +19,27 @@ namespace caffe { template -Net::Net(const NetParameter& param) { - Init(param); +Net::Net(const NetParameter& param, + const size_t force_input_count) { + Init(param, force_input_count); } template -Net::Net(const string& param_file, Phase phase) { +Net::Net(const string& param_file, Phase phase, + const size_t force_input_count) { NetParameter param; ReadNetParamsFromTextFileOrDie(param_file, ¶m); param.mutable_state()->set_phase(phase); - Init(param); + Init(param, force_input_count); } template -void Net::Init(const NetParameter& in_param) { +void Net::Init(const NetParameter& in_param, + const size_t force_input_count) { // Set phase from the state. phase_ = in_param.state().phase(); + // Explicitly set input count + force_input_count_ = force_input_count; // Filter layers based on their include/exclude rules and // the current NetState. NetParameter filtered_param; @@ -348,12 +353,23 @@ void Net::AppendTop(const NetParameter& param, const int layer_id, if (layer_id == -1) { // Set the (explicitly specified) dimensions of the input blob. if (param.input_dim_size() > 0) { - blob_pointer->Reshape(param.input_dim(top_id * 4), + blob_pointer->Reshape(((force_input_count_ > 0) ? + force_input_count_ : + param.input_dim(top_id * 4)), param.input_dim(top_id * 4 + 1), param.input_dim(top_id * 4 + 2), param.input_dim(top_id * 4 + 3)); } else { - blob_pointer->Reshape(param.input_shape(top_id)); + const BlobShape& input_shape = param.input_shape(top_id); + + vector shape_vec(input_shape.dim_size()); + for (int i = 0; i < input_shape.dim_size(); ++i) { + shape_vec[i] = input_shape.dim(i); + } + if (force_input_count_ > 0) { + shape_vec[0] = force_input_count_; + } + blob_pointer->Reshape(shape_vec); } net_input_blob_indices_.push_back(blob_id); net_input_blobs_.push_back(blob_pointer.get()); diff --git a/src/caffe/solver.cpp b/src/caffe/solver.cpp index 034390e6824..f4d6ed26bc7 100644 --- a/src/caffe/solver.cpp +++ b/src/caffe/solver.cpp @@ -347,7 +347,7 @@ template void Solver::Restore(const char* state_file) { SolverState state; NetParameter net_param; - ReadProtoFromBinaryFile(state_file, &state); + caffe::ReadProtoFromBinaryFile(state_file, &state); if (state.has_learned_net()) { ReadProtoFromBinaryFile(state.learned_net().c_str(), &net_param); net_->CopyTrainedLayersFrom(net_param);