Skip to content

Commit

Permalink
Caffe without the patch, cpp-package fixed also with Caffe plugin (#5…
Browse files Browse the repository at this point in the history
…573)

* PEP8 indentation fix

* Remove executable flag

* Remove executable flags

* Fixing cpp-package including problems with caffe converter

* Fix warnings

* Remove need for caffe patch for caffe plugin

* Ignore cmake paths, remove rebuildable op.h for cpp-package

* cpp-package examples fixed. Makefile and CMake of op.h and examples added.

* cpp-package examples fixed. Makefile and CMake of op.h and examples added.

* Fix source file

* Better example.mk

* Turn off caffe by default

* lint fixes

* Trying to figure out how to fix submodules

* nohub problem on travis so force retry

* Edited test to just run make instead of obsolete make example

* Fix cpp-package op.h bootstrapping with CMake

* Build tweaking cpp-package

* Lint fixes

* op.h generator

* FIx cpp-package for latest merge from master

* Fix lint

* static link

* link whole static lib

* Trigger another build attempt

* Trigger another build attempt

* Add caffe plugin support (disabled until dependencies can be added to build machine)

* Add cufft library

* win32 cufft library

* don't link unit tests as static

* Add include for rebuildable protobuf header caffe.pb.h
  • Loading branch information
cjolivier01 authored and piiswrong committed Mar 31, 2017
1 parent 7f688f7 commit a5abd79
Show file tree
Hide file tree
Showing 66 changed files with 559 additions and 7,914 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ __pycache__
*.json
*.d
build
cmake-build-debug
cmake-build-release
cmake-build
data
recommonmark
deps
Expand Down
66 changes: 37 additions & 29 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}"
include(cmake/Utils.cmake)
mxnet_option(USE_OPENCV "Build with OpenCV support" ON)
mxnet_option(USE_OPENMP "Build with Openmp support" ON)
mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path
mxnet_option(USE_CUDA "Build with CUDA support" ON)
mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path
mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
mxnet_option(USE_MKLML_MKL "Use MKLML variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE))
mxnet_option(USE_MKL_EXPERIMENTAL "Use experimental MKL (if MKL enabled and found)" OFF)
Expand All @@ -21,6 +21,7 @@ mxnet_option(USE_PROFILER "Build with Profiler support" OFF)
mxnet_option(USE_DIST_KVSTORE "Build with DIST_KVSTORE support" OFF)
mxnet_option(USE_PLUGINS_WARPCTC "Use WARPCTC Plugins" OFF)
mxnet_option(USE_PLUGIN_CAFFE "Use Caffe Plugin" OFF)
mxnet_option(USE_CPP_PACKAGE "Build C++ Package" OFF)
mxnet_option(USE_MXNET_LIB_NAMING "Use MXNet library naming conventions." ON)

SET(EXTRA_OPERATORS "" CACHE PATH "EXTRA OPERATORS PATH")
Expand Down Expand Up @@ -291,6 +292,10 @@ if(USE_PLUGINS_WARPCTC)
endif()

if(USE_PLUGIN_CAFFE)
if(NOT USE_CUDA)
set(CPU_ONLY ON)
add_definitions(-DCPU_ONLY=1)
endif()
if(NOT DEFINED CAFFE_PATH)
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/caffe)
# Need newer FindCUDA.cmake that correctly handles -std=c++11
Expand All @@ -314,6 +319,7 @@ if(USE_PLUGIN_CAFFE)
list(APPEND SOURCE ${PLUGINS_SOURCE})
list(APPEND CUDA ${PLUGINS_CUSRC})
include_directories(${CMAKE_BINARY_DIR}/include)
add_definitions(-DMXNET_USE_CAFFE=1)
list(APPEND mxnet_LINKER_LIBS
protobuf boost_system boost_thread boost_filesystem
gflags glog caffe
Expand Down Expand Up @@ -348,8 +354,10 @@ if(USE_CUDA)
list(APPEND mxnet_LINKER_LIBS ${CUDA_nvrtc_LIBRARY})
set(CUDA_cuda_LIBRARY "${CUDA_nvrtc_LIBRARY}/../cuda.lib")
list(APPEND mxnet_LINKER_LIBS ${CUDA_cuda_LIBRARY})
FIND_LIBRARY(CUDA_cufft_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64" "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32")
list(APPEND mxnet_LINKER_LIBS "${CUDA_cufft_LIBRARY}/../cufft.lib") # For fft operator
else(MSVC)
list(APPEND mxnet_LINKER_LIBS nvrtc cuda)
list(APPEND mxnet_LINKER_LIBS nvrtc cuda cufft)
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
endif()
list(APPEND SOURCE ${cuda_objs} ${CUDA})
Expand Down Expand Up @@ -395,19 +403,26 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" AND USE_MXNET_LIB_NAMING)
add_library(mxnet MODULE ${SOURCE})
else()
if(UNIX)
add_library(mxnet_static STATIC ${SOURCE})
# Need an arbitrary source file to trigger CMake to build the library
add_library(mxnet SHARED src/initialize.cc)
target_link_libraries(mxnet
-Wl,--whole-archive mxnet_static -Wl,--no-whole-archive
)
add_custom_target(
Name ALL
BYPRODUCTS ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/libmxnet.a
WORKING_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}
COMMAND ln -sf libmxnet_static.a libmxnet.a
DEPENDS mxnet_static
)
set(MXNET_DYNAMIC_ONLY ON)
if(MXNET_DYNAMIC_ONLY)
add_library(mxnet SHARED ${SOURCE})
else()
set(INITIALIZE_SOURCE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/src/initialize.cc)
list(REMOVE_ITEM SOURCE ${INITIALIZE_SOURCE_FILE})
add_library(mxnet_static STATIC ${INITIALIZE_SOURCE_FILE} ${SOURCE})
# Need an arbitrary source file to trigger CMake to build the library
add_library(mxnet SHARED ${INITIALIZE_SOURCE_FILE})
# This has prolems, as it adds libmxnet_static to INTERFACE_LINK_LIBRARIES
target_link_libraries(mxnet "-Wl,--whole-archive $<TARGET_FILE:mxnet_static> -Wl,--no-whole-archive")
#target_link_libraries(mxnet mxnet_static)
add_custom_target(
StaticallyLinkStaticMXNetLibrary ALL
BYPRODUCTS ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/libmxnet.a
WORKING_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}
COMMAND ln -sf libmxnet_static.a libmxnet.a
DEPENDS mxnet_static
)
endif()
else()
add_library(mxnet SHARED ${SOURCE})
endif()
Expand All @@ -429,7 +444,7 @@ endif()
if(USE_DIST_KVSTORE)
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ps-lite/CMakeLists.txt)
add_subdirectory("ps-lite")
list(APPEND pslite_LINKER_LIBS pslite)
list(APPEND pslite_LINKER_LIBS pslite)
target_link_libraries(mxnet debug ${pslite_LINKER_LIBS_DEBUG})
target_link_libraries(mxnet optimized ${pslite_LINKER_LIBS_RELEASE})
else()
Expand Down Expand Up @@ -473,6 +488,12 @@ if(INSTALL_PYTHON_VERSIONS)
endforeach()
endif()

if(USE_CPP_PACKAGE)
add_subdirectory(cpp-package)
endif()

add_subdirectory(example/image-classification/predict-cpp)

# ---[ Linter target
if(MSVC)
find_package(PythonInterp)
Expand All @@ -481,16 +502,3 @@ endif()
set(LINT_DIRS include src scripts python)
add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DLINT_DIRS=${LINT_DIRS} -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR} -DPROJECT_NAME=mxnet -P ${PROJECT_SOURCE_DIR}/dmlc-core/cmake/lint.cmake)

add_subdirectory(example/image-classification/predict-cpp)

if(NOT MSVC)
add_custom_command(TARGET mxnet
POST_BUILD
COMMAND cp $<TARGET_FILE:mxnet> .
COMMAND python2 OpWrapperGenerator.py $<TARGET_FILE_NAME:mxnet>
COMMAND rm $<TARGET_FILE_NAME:mxnet>
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cpp-package/src/OpWrapperGenerator/
)
endif()


22 changes: 17 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ ifeq ($(USE_PROFILER), 1)
CFLAGS += -DMXNET_USE_PROFILER=1
endif

# Caffe Plugin
ifdef CAFFE_PATH
CFLAGS += -DMXNET_USE_CAFFE=1
endif

ifndef LINT_LANG
LINT_LANG="all"
endif
Expand Down Expand Up @@ -130,10 +135,10 @@ ifeq ($(USE_DIST_KVSTORE), 1)
LDFLAGS += $(PS_LDFLAGS_A)
endif

.PHONY: clean all test lint doc clean_all rcpplint rcppexport roxygen\
.PHONY: clean all extra-packages test lint doc clean_all rcpplint rcppexport roxygen\
cython2 cython3 cython cyclean

all: lib/libmxnet.a lib/libmxnet.so $(BIN)
all: lib/libmxnet.a lib/libmxnet.so $(BIN) extra-packages

SRC = $(wildcard src/*/*/*.cc src/*/*.cc src/*.cc)
OBJ = $(patsubst %.cc, build/%.o, $(SRC))
Expand Down Expand Up @@ -259,14 +264,21 @@ $(BIN) :
@mkdir -p $(@D)
$(CXX) $(CFLAGS) -std=c++11 -o $@ $(filter %.cpp %.o %.c %.a %.cc, $^) $(LDFLAGS)

# CPP Package
ifeq ($(USE_CPP_PACKAGE), 1)
include cpp-package/cpp-package.mk
endif

include tests/cpp/unittest.mk

extra-packages: $(EXTRA_PACKAGES)

test: $(TEST)

lint: cpplint rcpplint jnilint pylint

cpplint:
python2 dmlc-core/scripts/lint.py mxnet cpp include src plugin
python2 dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package

pylint:
# ideally we want to check all, such as: python tools example tests
Expand Down Expand Up @@ -342,7 +354,7 @@ jnilint:
python2 dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src

ifneq ($(EXTRA_OPERATORS),)
clean: cyclean
clean: cyclean $(EXTRA_PACKAGES_CLEAN)
$(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \
R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz
cd $(DMLC_CORE); $(MAKE) clean; cd -
Expand All @@ -351,7 +363,7 @@ clean: cyclean
$(RM) -r $(patsubst %, %/*.d, $(EXTRA_OPERATORS)) $(patsubst %, %/*/*.d, $(EXTRA_OPERATORS))
$(RM) -r $(patsubst %, %/*.o, $(EXTRA_OPERATORS)) $(patsubst %, %/*/*.o, $(EXTRA_OPERATORS))
else
clean: cyclean
clean: cyclean $(EXTRA_PACKAGES_CLEAN)
$(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \
R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz
cd $(DMLC_CORE); $(MAKE) clean; cd -
Expand Down
18 changes: 18 additions & 0 deletions cpp-package/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@

if(USE_CPP_PACKAGE AND NOT MSVC)

set(CPP_PACKAGE_OP_H_HEADER ${CMAKE_CURRENT_LIST_DIR}/include/mxnet-cpp/op.h)

add_custom_target(
cpp_package_op_h ALL
BYPRODUCTS ${CPP_PACKAGE_OP_H_HEADER}
MAIN_DEPENDENCY mxnet
DEPENDS mxnet ${CMAKE_SOURCE_DIR}/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.py
COMMAND echo "Running: OpWrapperGenerator.py"
COMMAND python OpWrapperGenerator.py ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/$<TARGET_FILE:mxnet>
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cpp-package/src/OpWrapperGenerator/
)

add_subdirectory(example)

endif()
Empty file modified cpp-package/LICENSE
100755 → 100644
Empty file.
11 changes: 0 additions & 11 deletions cpp-package/Makefile

This file was deleted.

28 changes: 28 additions & 0 deletions cpp-package/cpp-package.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
ifndef LINT_LANG
LINT_LANG="all"
endif

ifdef CAFFE_PATH
export LD_LIBRARY_PATH=$(CAFFE_PATH)/lib
endif

CPP_PACKAGE_OP_H_FILE = cpp-package/include/mxnet-cpp/op.h

EXTRA_PACKAGES += cpp-package-all
EXTRA_PACKAGES_CLEAN += cpp-package-clean

.PHONY: cpp-package-all cpp-package-lint cpp-package-clean

cpp-package-all: $(CPP_PACKAGE_OP_H_FILE)

cpp-package-clean:
rm -f $(CPP_PACKAGE_OP_H_FILE)

$(CPP_PACKAGE_OP_H_FILE): lib/libmxnet.so cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.py
(cd cpp-package/src/OpWrapperGenerator; python OpWrapperGenerator.py $(ROOTDIR)/lib/libmxnet.so)

cpp-package-lint:
(cd cpp-package; python scripts/lint.py dmlc ${LINT_LANG} include example)

include cpp-package/example/example.mk

54 changes: 54 additions & 0 deletions cpp-package/example/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@

if(NOT MSVC)
set(UNITTEST_STATIC_LINK ON)
endif()

set(CPP_EXAMPLE_LIBS
rt
${BEGIN_WHOLE_ARCHIVE} mxnet_static ${END_WHOLE_ARCHIVE}
dmlccore
${mxnet_LINKER_LIBS}
)

set(CPP_PACKAGE_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/../include/mxnet-cpp/)

set(CPPEX_DEPS cpp_package_op_h)

file(GLOB_RECURSE CPP_PACKAGE_HEADERS
"${CPP_PACKAGE_INCLUDE_DIR}/*.h"
"${CPP_PACKAGE_INCLUDE_DIR}/*.hpp"
)

include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../include)

add_executable(lenet lenet.cpp ${CPP_PACKAGE_HEADERS})
target_link_libraries(lenet ${CPP_EXAMPLE_LIBS})
add_dependencies(lenet ${CPPEX_DEPS})

add_executable(lenet_with_mxdataiter lenet_with_mxdataiter.cpp ${CPP_PACKAGE_HEADERS})
target_link_libraries(lenet_with_mxdataiter ${CPP_EXAMPLE_LIBS})
add_dependencies(lenet_with_mxdataiter ${CPPEX_DEPS})

add_executable(alexnet alexnet.cpp ${CPP_PACKAGE_HEADERS})
target_link_libraries(alexnet ${CPP_EXAMPLE_LIBS})
add_dependencies(alexnet ${CPPEX_DEPS})

add_executable(charRNN charRNN.cpp ${CPP_PACKAGE_HEADERS})
target_link_libraries(charRNN ${CPP_EXAMPLE_LIBS})
add_dependencies(charRNN ${CPPEX_DEPS})

add_executable(googlenet googlenet.cpp ${CPP_PACKAGE_HEADERS})
target_link_libraries(googlenet ${CPP_EXAMPLE_LIBS})
add_dependencies(googlenet ${CPPEX_DEPS})

add_executable(inception_bn inception_bn.cpp ${CPP_PACKAGE_HEADERS})
target_link_libraries(inception_bn ${CPP_EXAMPLE_LIBS})
add_dependencies(inception_bn ${CPPEX_DEPS})

add_executable(mlp mlp.cpp ${CPP_PACKAGE_HEADERS})
target_link_libraries(mlp ${CPP_EXAMPLE_LIBS})
add_dependencies(mlp ${CPPEX_DEPS})

add_executable(resnet resnet.cpp ${CPP_PACKAGE_HEADERS})
target_link_libraries(resnet ${CPP_EXAMPLE_LIBS})
add_dependencies(resnet ${CPPEX_DEPS})
70 changes: 0 additions & 70 deletions cpp-package/example/Makefile

This file was deleted.

2 changes: 2 additions & 0 deletions cpp-package/example/alexnet.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
#include <map>
#include <string>
#include "mxnet-cpp/MxNetCpp.h"
// Allow IDE to parse the types
#include "../include/mxnet-cpp/op.h"

using namespace std;
using namespace mxnet::cpp;
Expand Down
Loading

0 comments on commit a5abd79

Please sign in to comment.