diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..370aae686 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "thirdparty/grpc"] + path = thirdparty/grpc + url = https://github.com/grpc/grpc diff --git a/CMakeLists.txt b/CMakeLists.txt index 31b74e291..419ada6f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,8 @@ set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty") list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) -find_package(Protobuf REQUIRED) +set(CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/" ${CMAKE_PREFIX_PATH}) + find_package(PythonInterp REQUIRED) find_package(PythonLibs REQUIRED) find_package(NumPy REQUIRED) @@ -15,6 +16,8 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") include_directories("${CMAKE_SOURCE_DIR}/include") include_directories("/usr/local/include") +include_directories("${CMAKE_SOURCE_DIR}/thirdparty/grpc/include/") +include_directories("${CMAKE_SOURCE_DIR}/thirdparty/grpc/third_party/protobuf/src") include_directories("${PYTHON_INCLUDE_DIRS}") include_directories("${NUMPY_INCLUDE_DIR}") @@ -40,14 +43,14 @@ add_custom_command( "${HALO_PB_CPP_FILE}" "${HALO_GRPC_PB_H_FILE}" "${HALO_GRPC_PB_CPP_FILE}" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + COMMAND ${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/protobuf/protoc ARGS "--proto_path=${PROTO_PATH}" "--cpp_out=${GENERATED_PROTOBUF_PATH}" "${HALO_PROTO}" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + COMMAND ${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/protobuf/protoc ARGS "--proto_path=${PROTO_PATH}" "--grpc_out=${GENERATED_PROTOBUF_PATH}" - "--plugin=protoc-gen-grpc=/usr/local/bin/grpc_cpp_plugin" + "--plugin=protoc-gen-grpc=${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/grpc_cpp_plugin" "${HALO_PROTO}" ) @@ -56,14 +59,14 @@ add_custom_command( "${TYPES_PB_CPP_FILE}" "${TYPES_GRPC_PB_H_FILE}" "${TYPES_GRPC_PB_CPP_FILE}" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + COMMAND ${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/protobuf/protoc ARGS "--proto_path=${PROTO_PATH}" "--cpp_out=${GENERATED_PROTOBUF_PATH}" "${TYPES_PROTO}" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + COMMAND ${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/protobuf/protoc ARGS "--proto_path=${PROTO_PATH}" "--grpc_out=${GENERATED_PROTOBUF_PATH}" - "--plugin=protoc-gen-grpc=/usr/local/bin/grpc_cpp_plugin" + "--plugin=protoc-gen-grpc=${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/grpc_cpp_plugin" "${TYPES_PROTO}" ) @@ -73,15 +76,41 @@ set(GENERATED_PROTOBUF_FILES ${HALO_PB_H_FILE} ${HALO_PB_CPP_FILE} ${TYPES_GRPC_PB_H_FILE} ${TYPES_GRPC_PB_CPP_FILE}) include_directories(${GENERATED_PROTOBUF_PATH}) -link_libraries(grpc++_unsecure grpc pthread protobuf) -if (UNIX AND NOT APPLE) - link_libraries(rt) + +link_libraries(${CMAKE_SOURCE_DIR}/thirdparty/grpc/libs/opt/libgrpc++_unsecure.a + ${CMAKE_SOURCE_DIR}/thirdparty/grpc/libs/opt/libgrpc++.a + ${CMAKE_SOURCE_DIR}/thirdparty/grpc/libs/opt/libgrpc.a + ${CMAKE_SOURCE_DIR}/thirdparty/grpc/libs/opt/protobuf/libprotobuf.a + pthread) + +if(UNIX AND NOT APPLE) + link_libraries(rt) endif() +if(APPLE) + SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif(APPLE) + +set(ARROW_LIB ${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a) + +add_definitions(-fPIC) +include_directories("${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/src/") +include_directories("${CMAKE_SOURCE_DIR}/thirdparty/numbuf/cpp/src/") +include_directories("${CMAKE_SOURCE_DIR}/thirdparty/numbuf/python/src/") +add_library(pynumbuf STATIC ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/cpp/src/numbuf/tensor.cc + ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/cpp/src/numbuf/types.cc + ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/cpp/src/numbuf/metadata.cc + ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/cpp/src/numbuf/dict.cc + ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/python/src/pynumbuf/serialize.cc + ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/python/src/pynumbuf/numbuf.cc + ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/python/src/pynumbuf/adapters/numpy.cc + ${CMAKE_SOURCE_DIR}/thirdparty/numbuf/python/src/pynumbuf/adapters/python.cc) +target_link_libraries(pynumbuf ${ARROW_LIB} ${PYTHON_LIBRARIES}) + add_executable(objstore src/objstore.cc src/ipc.cc ${GENERATED_PROTOBUF_FILES}) -target_link_libraries(objstore arrow numbuf pynumbuf) +target_link_libraries(objstore ${ARROW_LIB} pynumbuf) add_executable(scheduler src/scheduler.cc src/computation_graph.cc ${GENERATED_PROTOBUF_FILES}) add_library(halolib SHARED src/halolib.cc src/worker.cc src/ipc.cc ${GENERATED_PROTOBUF_FILES}) -target_link_libraries(halolib arrow numbuf pynumbuf) +target_link_libraries(halolib ${ARROW_LIB} pynumbuf) install(TARGETS objstore scheduler halolib DESTINATION ${CMAKE_SOURCE_DIR}/lib/python/halo) diff --git a/lib/python/halo/datasets/__init__.py b/lib/python/halo/datasets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/python/setup.py b/lib/python/setup.py index 6fdd1b8b5..e5ba911a1 100644 --- a/lib/python/setup.py +++ b/lib/python/setup.py @@ -5,17 +5,13 @@ import setuptools # because of relative paths, this must be run from inside halo/lib/python/ -MACOSX = (sys.platform in ["darwin"]) - setup( name = "halo", version = "0.1.dev0", use_2to3=True, packages=find_packages(), package_data = { - "halo": ["libhalolib.dylib" if MACOSX else "libhalolib.so", - "scheduler", - "objstore"] + "halo": ["libhalolib.so", "scheduler", "objstore"] }, zip_safe=False ) diff --git a/requirements.txt b/requirements.txt index 85f1652e5..6105fd65f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -six >= 1.10 typing funcsigs subprocess32 -grpcio diff --git a/setup.sh b/setup.sh index 020659b88..d0ef95df9 100644 --- a/setup.sh +++ b/setup.sh @@ -1,6 +1,5 @@ sudo apt-get update -sudo apt-get install -y git cmake build-essential python-dev python-numpy automake autoconf libtool python-pip libboost-all-dev unzip -sudo pip install --ignore-installed six # getting rid of an old version of six, if it is installed (needed for Ubuntu 14.04) +sudo apt-get install -y git cmake build-essential autoconf libtool python-dev python-numpy python-pip libboost-all-dev unzip sudo pip install -r requirements.txt cd thirdparty bash download_thirdparty.sh diff --git a/test/arrays_test.py b/test/arrays_test.py index 5ab33c283..e395ae951 100644 --- a/test/arrays_test.py +++ b/test/arrays_test.py @@ -11,12 +11,6 @@ import os import halo.arrays.remote as ra import halo.arrays.distributed as da -from google.protobuf.text_format import * - -from grpc.beta import implementations -import halo_pb2 -import types_pb2 - class ArraysSingleTest(unittest.TestCase): def testMethods(self): diff --git a/test/runtest.py b/test/runtest.py index 67be6d6ca..c412ba014 100644 --- a/test/runtest.py +++ b/test/runtest.py @@ -8,11 +8,6 @@ import time import subprocess32 as subprocess import os -from google.protobuf.text_format import * - -import halo_pb2 -import types_pb2 - import test_functions import halo.arrays.remote as ra import halo.arrays.distributed as da diff --git a/thirdparty/build_thirdparty.sh b/thirdparty/build_thirdparty.sh index 937259c90..6ad79b648 100644 --- a/thirdparty/build_thirdparty.sh +++ b/thirdparty/build_thirdparty.sh @@ -16,37 +16,14 @@ else exit 1 fi -echo "installing arrow" -export FLATBUFFERS_HOME=$TP_DIR/arrow/cpp/thirdparty/installed/ -$TP_DIR/arrow/cpp/thirdparty/download_thirdparty.sh -$TP_DIR/arrow/cpp/thirdparty/build_thirdparty.sh +echo "building arrow" +cd $TP_DIR/arrow/cpp +source setup_build_env.sh mkdir -p $TP_DIR/arrow/cpp/build cd $TP_DIR/arrow/cpp/build -cmake -DCMAKE_BUILD_TYPE=Release .. +cmake -DLIBARROW_LINKAGE=STATIC -DCMAKE_BUILD_TYPE=Release .. make VERBOSE=1 -j$PARALLEL -sudo make VERBOSE=1 install -echo "installing numbuf" -mkdir -p $TP_DIR/numbuf/cpp/build -cd $TP_DIR/numbuf/cpp/build -cmake -DCMAKE_BUILD_TYPE=Release .. -make VERBOSE=1 -j$PARALLEL -sudo make VERBOSE=1 install -mkdir -p $TP_DIR/numbuf/python/build -cd $TP_DIR/numbuf/python/build -cmake -DCMAKE_BUILD_TYPE=Release .. -make VERBOSE=1 -j$PARALLEL -sudo make VERBOSE=1 install - -echo "installing GRPC" -cd $TP_DIR/grpc/third_party/protobuf -./autogen.sh -export CXXFLAGS="$CXXFLAGS -fPIC" -./configure --enable-static=no -sudo make install -sudo ldconfig -cd python -sudo python setup.py install +echo "building GRPC" cd $TP_DIR/grpc -make VERBOSE=1 -j$PARALLEL -sudo make VERBOSE=1 install +make static HAS_SYSTEM_PROTOBUF=false HAS_SYSTEM_ZLIB=false -j$PARALLEL diff --git a/thirdparty/download_thirdparty.sh b/thirdparty/download_thirdparty.sh index 219048cd1..786137d4f 100644 --- a/thirdparty/download_thirdparty.sh +++ b/thirdparty/download_thirdparty.sh @@ -7,9 +7,9 @@ TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) if [ ! -d arrow ]; then echo "Fetching arrow" - git clone https://github.com/apache/arrow.git + git clone https://github.com/pcmoritz/arrow.git cd arrow - git checkout 4bd13b852d376065fdb16c36fa821ab0e167f0fc + git checkout static cd .. fi @@ -18,10 +18,9 @@ if [ ! -d numbuf ]; then git clone https://github.com/amplab/numbuf.git fi -if [ ! -d grpc ]; then - echo "Fetching GRPC" - git clone https://github.com/grpc/grpc.git - cd grpc - git submodule update --init - cd .. -fi +git submodule update --init --recursive + +# this seems to be neeccessary for building on Mac OS X +cd grpc +git submodule update --init --recursive +cd .. diff --git a/thirdparty/grpc b/thirdparty/grpc new file mode 160000 index 000000000..d941fd878 --- /dev/null +++ b/thirdparty/grpc @@ -0,0 +1 @@ +Subproject commit d941fd8786b7eb9c20a299b84d7a2a7c9228549d