diff --git a/.gitignore b/.gitignore index 73bba6cb364..28f2aca854b 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,9 @@ # QtCreator files *.user +# PyCharm files +.idea + # OSX dir files .DS_Store diff --git a/CMakeLists.txt b/CMakeLists.txt index adea37be565..54b044d347b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ caffe_option(CPU_ONLY "Build Caffe wihtout CUDA support" OFF) # TODO: rename to caffe_option(USE_CUDNN "Build Caffe with cuDNN libary support" ON IF NOT CPU_ONLY) caffe_option(BUILD_SHARED_LIBS "Build shared libraries" ON) caffe_option(BUILD_python "Build Python wrapper" ON) +set(python_version "2" CACHE STRING "Specify which python version to use") caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE) caffe_option(BUILD_docs "Build documentation" ON IF UNIX OR APPLE) @@ -24,7 +25,7 @@ caffe_option(BUILD_docs "Build documentation" ON IF UNIX OR APPLE) include(cmake/Dependencies.cmake) # ---[ Flags -if(UNIX OR APLE) +if(UNIX OR APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall") endif() diff --git a/Makefile b/Makefile index 29827270baf..cb396794e37 100644 --- a/Makefile +++ b/Makefile @@ -261,7 +261,8 @@ ifneq (,$(findstring clang++,$(CXX))) else ifneq (,$(findstring g++,$(CXX))) STATIC_LINK_COMMAND := -Wl,--whole-archive $(STATIC_NAME) -Wl,--no-whole-archive else - $(error Cannot static link with the $(CXX) compiler.) + # The following line must not be indented with a tab, since we are not inside a target + $(error Cannot static link with the $(CXX) compiler) endif # Debugging @@ -319,7 +320,7 @@ else # 10.10 has accelerate while 10.9 has veclib XCODE_CLT_VER := $(shell pkgutil --pkg-info=com.apple.pkg.CLTools_Executables | grep -o 'version: 6') ifneq (,$(findstring version: 6,$(XCODE_CLT_VER))) - BLAS_INCLUDE ?= /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.10.sdk/System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Headers/ + BLAS_INCLUDE ?= /System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Headers/ LDFLAGS += -framework Accelerate else BLAS_INCLUDE ?= /System/Library/Frameworks/vecLib.framework/Versions/Current/Headers/ @@ -450,6 +451,7 @@ $(MAT$(PROJECT)_SO): $(MAT$(PROJECT)_SRC) $(STATIC_NAME) CXXLIBS="\$$CXXLIBS $(STATIC_LINK_COMMAND) $(LDFLAGS)" -output $@ runtest: $(TEST_ALL_BIN) + $(TOOL_BUILD_DIR)/caffe $(TEST_ALL_BIN) $(TEST_GPUID) --gtest_shuffle $(TEST_FILTER) pytest: py @@ -537,7 +539,12 @@ $(TOOL_BUILD_DIR)/%: $(TOOL_BUILD_DIR)/%.bin | $(TOOL_BUILD_DIR) @ $(RM) $@ @ ln -s $(abspath $<) $@ -$(TOOL_BINS) $(EXAMPLE_BINS): %.bin : %.o | $(DYNAMIC_NAME) +$(TOOL_BINS): %.bin : %.o | $(DYNAMIC_NAME) + @ echo CXX/LD -o $@ + $(Q)$(CXX) $< -o $@ $(LINKFLAGS) -l$(PROJECT) $(LDFLAGS) \ + -Wl,-rpath,$(ORIGIN)/../lib + +$(EXAMPLE_BINS): %.bin : %.o | $(DYNAMIC_NAME) @ echo CXX/LD -o $@ $(Q)$(CXX) $< -o $@ $(LINKFLAGS) -l$(PROJECT) $(LDFLAGS) \ -Wl,-rpath,$(ORIGIN)/../../lib diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index aa2dcbe1d0d..b1ac96c6777 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -92,12 +92,39 @@ endif() # ---[ Python if(BUILD_python) - # disable Python 3 search - find_package(PythonInterp 2.7) - find_package(PythonLibs 2.7) - find_package(NumPy 1.7.1) - find_package(Boost 1.46 COMPONENTS python) - + if(NOT "${python_version}" VERSION_LESS "3.0.0") + # use python3 + find_package(PythonInterp 3.0) + find_package(PythonLibs 3.0) + find_package(NumPy 1.7.1) + # Find the matching boost python implementation + set(version ${PYTHONLIBS_VERSION_STRING}) + + STRING( REPLACE "." "" boost_py_version ${version} ) + find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}") + set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND}) + + while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND) + STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} ) + STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} ) + if("${has_more_version}" STREQUAL "") + break() + endif() + + STRING( REPLACE "." "" boost_py_version ${version} ) + find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}") + set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND}) + endwhile() + if(NOT Boost_PYTHON_FOUND) + find_package(Boost 1.46 COMPONENTS python) + endif() + else() + # disable Python 3 search + find_package(PythonInterp 2.7) + find_package(PythonLibs 2.7) + find_package(NumPy 1.7.1) + find_package(Boost 1.46 COMPONENTS python) + endif() if(PYTHONLIBS_FOUND AND NUMPY_FOUND AND Boost_PYTHON_FOUND) set(HAVE_PYTHON TRUE) endif() diff --git a/cmake/Misc.cmake b/cmake/Misc.cmake index 608a5f13a79..39569eaf996 100644 --- a/cmake/Misc.cmake +++ b/cmake/Misc.cmake @@ -32,6 +32,11 @@ endif() set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE BOOLEAN "Use link paths for shared library rpath") set(CMAKE_MACOSX_RPATH TRUE) +list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES ${CMAKE_INSTALL_PREFIX}/lib __is_systtem_dir) +if(${__is_systtem_dir} STREQUAL -1) + set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_PREFIX}/lib) +endif() + # ---[ Funny target if(UNIX OR APPLE) add_custom_target(symlink_to_build COMMAND "ln" "-sf" "${PROJECT_BINARY_DIR}" "${PROJECT_SOURCE_DIR}/build" diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index 3f7dff6b6e0..32931942846 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -107,8 +107,9 @@ function(caffe_print_configuration_summary) caffe_status(" C++ compiler : ${CMAKE_CXX_COMPILER}") caffe_status(" Release CXX flags : ${__flags_rel}") caffe_status(" Debug CXX flags : ${__flags_deb}") - caffe_status(" BUILD_SHARED_LIBS : ${BUILD_SHARED_LIBS}") caffe_status(" Build type : ${CMAKE_BUILD_TYPE}") + caffe_status("") + caffe_status(" BUILD_SHARED_LIBS : ${BUILD_SHARED_LIBS}") caffe_status(" BUILD_python : ${BUILD_python}") caffe_status(" BUILD_matlab : ${BUILD_matlab}") caffe_status(" BUILD_docs : ${BUILD_docs}") @@ -116,8 +117,9 @@ function(caffe_print_configuration_summary) caffe_status("") caffe_status("Dependencies:") caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})") + caffe_status(" Boost : Yes (ver. ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION})") caffe_status(" glog : Yes") - caffe_status(" gflags : Yes") + caffe_status(" gflags : Yes") caffe_status(" protobuf : " PROTOBUF_FOUND THEN "Yes (ver. ${PROTOBUF_VERSION})" ELSE "No" ) caffe_status(" lmdb : " LMDB_FOUND THEN "Yes (ver. ${LMDB_VERSION})" ELSE "No") caffe_status(" Snappy : " SNAPPY_FOUND THEN "Yes (ver. ${Snappy_VERSION})" ELSE "No" ) diff --git a/docs/installation.md b/docs/installation.md index 16575b54029..144e6a34f67 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -30,7 +30,7 @@ Caffe has several dependencies. Pycaffe and Matcaffe interfaces have their own natural needs. -* For Python Caffe: `Python 2.7`, `numpy (>= 1.7)`, boost-provided `boost.python` +* For Python Caffe: `Python 2.7` or `Python 3.3+`, `numpy (>= 1.7)`, boost-provided `boost.python` * For MATLAB Caffe: MATLAB with the `mex` compiler. **cuDNN Caffe**: for fastest operation Caffe is accelerated by drop-in integration of [NVIDIA cuDNN](https://developer.nvidia.com/cudnn). To speed up your Caffe models, install cuDNN then uncomment the `USE_CUDNN := 1` flag in `Makefile.config` when installing Caffe. Acceleration is automatic. For now cuDNN v1 is integrated but see [PR #1731](https://github.com/BVLC/caffe/pull/1731) for v2. @@ -69,7 +69,7 @@ but we suggest first installing the [Anaconda](https://store.continuum.io/cshop/ To import the `caffe` Python module after completing the installation, add the module directory to your `$PYTHONPATH` by `export PYTHONPATH=/path/to/caffe/python:$PYTHONPATH` or the like. You should not import the module in the `caffe/python/caffe` directory! -*Caffe's Python interface works with Python 2.7. Python 3 or earlier Pythons are your own adventure.* +*Caffe's Python interface works with Python 2.7. Python 3.3+ should work out of the box without protobuf support. For protobuf support please install protobuf 3.0 alpha (https://developers.google.com/protocol-buffers/). Earlier Pythons are your own adventure.* #### MATLAB diff --git a/docs/tutorial/layers.md b/docs/tutorial/layers.md index 34bb48050e8..839939f5ad6 100644 --- a/docs/tutorial/layers.md +++ b/docs/tutorial/layers.md @@ -453,20 +453,20 @@ The `SLICE` layer is a utility layer that slices an input layer to multiple outp * Sample - layers { - name: "slicer_label" - type: SLICE - bottom: "label" - ## Example of label with a shape N x 3 x 1 x 1 - top: "label1" - top: "label2" - top: "label3" - slice_param { - slice_dim: 1 - slice_point: 1 - slice_point: 2 - } - } + layers { + name: "slicer_label" + type: SLICE + bottom: "label" + ## Example of label with a shape N x 3 x 1 x 1 + top: "label1" + top: "label2" + top: "label3" + slice_param { + slice_dim: 1 + slice_point: 1 + slice_point: 2 + } + } `slice_dim` indicates the target dimension and can assume only two values: 0 for num or 1 for channel; `slice_point` indicates indexes in the selected dimension (the number of indexes must be equal to the number of top blobs minus one). diff --git a/examples/imagenet/readme.md b/examples/imagenet/readme.md index c2dd62ec963..a6bdf49ca4d 100644 --- a/examples/imagenet/readme.md +++ b/examples/imagenet/readme.md @@ -26,7 +26,7 @@ We assume that you already have downloaded the ImageNet training data and valida You will first need to prepare some auxiliary data for training. This data can be downloaded by: - ./data/get_ilsvrc_aux.sh + ./data/ilsvrc12/get_ilsvrc_aux.sh The training and validation input are described in `train.txt` and `val.txt` as text listing all the files and their labels. Note that we use a different indexing for labels than the ILSVRC devkit: we sort the synset names in their ASCII order, and then label them from 0 to 999. See `synset_words.txt` for the synset/name mapping. diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp index 6cb507a5780..9cc23d1ff42 100644 --- a/include/caffe/vision_layers.hpp +++ b/include/caffe/vision_layers.hpp @@ -107,7 +107,7 @@ class BaseConvolutionLayer : public Layer { int col_offset_; int output_offset_; - Blob col_buffer_; + static Blob col_buffer_; Blob bias_multiplier_; }; diff --git a/matlab/caffe/matcaffe.cpp b/matlab/caffe/matcaffe.cpp index 996d3d2149c..da37d920b20 100644 --- a/matlab/caffe/matcaffe.cpp +++ b/matlab/caffe/matcaffe.cpp @@ -272,7 +272,7 @@ static void get_init_key(MEX_ARGS) { static void init(MEX_ARGS) { if (nrhs != 3) { ostringstream error_msg; - error_msg << "Expected 2 arguments, got " << nrhs; + error_msg << "Expected 3 arguments, got " << nrhs; mex_error(error_msg.str()); } diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 6afed4fa183..a2f82089cac 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,7 +4,7 @@ if(NOT HAVE_PYTHON) endif() include_directories(${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR} ${Boost_INCLUDE_DIRS}) -file(GLOB_RECURSE python_srcs ${PROJECT_SOURCE_DIR}/python/*.cpp) +file(GLOB_RECURSE python_srcs ${CMAKE_SOURCE_DIR}/python/*.cpp) add_library(pycaffe SHARED ${python_srcs}) target_link_libraries(pycaffe ${Caffe_LINK} ${PYTHON_LIBRARIES} ${Boost_LIBRARIES}) @@ -22,9 +22,13 @@ if(UNIX OR APPLE) endif() # ---[ Install -file(GLOB files *.py requirements.txt) -install(FILES ${files} DESTINATION python) -install(DIRECTORY caffe DESTINATION python) -install(TARGETS pycaffe DESTINATION python/caffe) +file(GLOB files1 *.py requirements.txt) +install(FILES ${files1} DESTINATION python) + +file(GLOB files2 caffe/*.py) +install(FILES ${files2} DESTINATION python/caffe) +install(TARGETS pycaffe DESTINATION python/caffe) +install(DIRECTORY caffe/imagenet caffe/proto caffe/test DESTINATION python/caffe) + diff --git a/python/caffe/_caffe.cpp b/python/caffe/_caffe.cpp index a5d0e64605e..03967a21029 100644 --- a/python/caffe/_caffe.cpp +++ b/python/caffe/_caffe.cpp @@ -275,7 +275,9 @@ BOOST_PYTHON_MODULE(_caffe) { bp::class_ >("BoolVec") .def(bp::vector_indexing_suite >()); - import_array(); + // boost python expects a void (missing) return value, while import_array + // returns NULL for python3. import_array1() forces a void return value. + import_array1(); } } // namespace caffe diff --git a/python/caffe/io.py b/python/caffe/io.py index 0ce9ecfeeed..f51e3a64d36 100644 --- a/python/caffe/io.py +++ b/python/caffe/io.py @@ -3,7 +3,14 @@ from scipy.ndimage import zoom from skimage.transform import resize -from caffe.proto import caffe_pb2 +try: + # Python3 will most likely not be able to load protobuf + from caffe.proto import caffe_pb2 +except: + if sys.version_info >= (3,0): + print("Failed to include caffe_pb2, things might go wrong!") + else: + raise ## proto / datum / ndarray conversion diff --git a/python/caffe/pycaffe.py b/python/caffe/pycaffe.py index 31c145d77a5..d662d6cc282 100644 --- a/python/caffe/pycaffe.py +++ b/python/caffe/pycaffe.py @@ -4,7 +4,10 @@ """ from collections import OrderedDict -from itertools import izip_longest +try: + from itertools import izip_longest +except: + from itertools import zip_longest as izip_longest import numpy as np from ._caffe import Net, SGDSolver diff --git a/python/classify.py b/python/classify.py index d435a572266..81d06369341 100755 --- a/python/classify.py +++ b/python/classify.py @@ -103,7 +103,7 @@ def main(argv): channel_swap=channel_swap) if args.gpu: - print 'GPU mode' + print('GPU mode') # Load numpy array (.npy), directory glob (*.jpg), or image file. args.input_file = os.path.expanduser(args.input_file) @@ -115,12 +115,12 @@ def main(argv): else: inputs = [caffe.io.load_image(args.input_file)] - print "Classifying %d inputs." % len(inputs) + print("Classifying %d inputs." % len(inputs)) # Classify. start = time.time() predictions = classifier.predict(inputs, not args.center_only) - print "Done in %.2f s." % (time.time() - start) + print("Done in %.2f s." % (time.time() - start)) # Save np.save(args.output_file, predictions) diff --git a/python/detect.py b/python/detect.py index cb0c2645761..d395bd97abf 100755 --- a/python/detect.py +++ b/python/detect.py @@ -115,7 +115,7 @@ def main(argv): context_pad=args.context_pad) if args.gpu: - print 'GPU mode' + print('GPU mode') # Load input. t = time.time() diff --git a/python/draw_net.py b/python/draw_net.py index 4457b793e86..6320f775ef7 100755 --- a/python/draw_net.py +++ b/python/draw_net.py @@ -36,7 +36,7 @@ def main(): args = parse_args() net = caffe_pb2.NetParameter() text_format.Merge(open(args.input_net_proto_file).read(), net) - print 'Drawing net to %s' % args.output_image_file + print('Drawing net to %s' % args.output_image_file) caffe.draw.draw_net_to_file(net, args.output_image_file, args.rankdir) diff --git a/scripts/cpp_lint.py b/scripts/cpp_lint.py index 1b7c6c0536c..f750489f4f9 100755 --- a/scripts/cpp_lint.py +++ b/scripts/cpp_lint.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # Copyright (c) 2009 Google Inc. All rights reserved. # diff --git a/src/caffe/layers/base_conv_layer.cpp b/src/caffe/layers/base_conv_layer.cpp index dccd5170c11..a454d42cbb3 100644 --- a/src/caffe/layers/base_conv_layer.cpp +++ b/src/caffe/layers/base_conv_layer.cpp @@ -8,6 +8,9 @@ namespace caffe { +template +Blob BaseConvolutionLayer::col_buffer_; + template void BaseConvolutionLayer::LayerSetUp(const vector*>& bottom, const vector*>& top) { diff --git a/src/caffe/layers/data_layer.cpp b/src/caffe/layers/data_layer.cpp index 8877caf89c8..1861090f953 100644 --- a/src/caffe/layers/data_layer.cpp +++ b/src/caffe/layers/data_layer.cpp @@ -89,9 +89,17 @@ void DataLayer::InternalThreadEntry() { // Reshape on single input batches for inputs of varying dimension. const int batch_size = this->layer_param_.data_param().batch_size(); const int crop_size = this->layer_param_.transform_param().crop_size(); + bool force_color = this->layer_param_.data_param().force_encoded_color(); if (batch_size == 1 && crop_size == 0) { Datum datum; datum.ParseFromString(cursor_->value()); + if (datum.encoded()) { + if (force_color) { + DecodeDatum(&datum, true); + } else { + DecodeDatumNative(&datum); + } + } this->prefetch_data_.Reshape(1, datum.channels(), datum.height(), datum.width()); this->transformed_data_.Reshape(1, datum.channels(), @@ -104,7 +112,6 @@ void DataLayer::InternalThreadEntry() { if (this->output_labels_) { top_label = this->prefetch_label_.mutable_cpu_data(); } - bool force_color = this->layer_param_.data_param().force_encoded_color(); for (int item_id = 0; item_id < batch_size; ++item_id) { timer.Start(); // get a blob