diff --git a/.travis.yml b/.travis.yml index d44a93d84..7e8293e8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,10 @@ matrix: install: - ./install-dependencies.sh - ./build.sh + + - cd numbuf + - sudo python setup.py install + - cd .. - cd src/common/lib/python - sudo python setup.py install @@ -62,6 +66,8 @@ install: - cd ../.. script: + - python numbuf/python/test/runtest.py + - python src/common/test/test.py - python src/plasma/test/test.py - python src/photon/test/test.py diff --git a/doc/install-on-macosx.md b/doc/install-on-macosx.md index bc208c9f0..02b3c9b74 100644 --- a/doc/install-on-macosx.md +++ b/doc/install-on-macosx.md @@ -1,6 +1,6 @@ # Installation on Mac OS X -Ray should work with Python 2 and Python 3. We have tested Ray on OS X 10.11. +Ray should work with Python 2. We have tested Ray on OS X 10.11. ## Dependencies @@ -14,7 +14,7 @@ sudo easy_install pip # If you're using Anaconda, then this is unnecessary. pip install numpy funcsigs colorama psutil redis --ignore-installed six pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4 # We use the latest version of cloudpickle because it can serialize named tuples. -pip install --upgrade --verbose git+git://github.com/ray-project/numbuf.git@488f881d708bc54e86ed375ee97aa94540808fa1 +pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=ray&subdirectory=numbuf" ``` # Install Ray diff --git a/doc/install-on-ubuntu.md b/doc/install-on-ubuntu.md index fc4638dcc..9d8c50a77 100644 --- a/doc/install-on-ubuntu.md +++ b/doc/install-on-ubuntu.md @@ -1,6 +1,6 @@ # Installation on Ubuntu -Ray should work with Python 2 and Python 3. We have tested Ray on Ubuntu 14.04 +Ray should work with Python 2. We have tested Ray on Ubuntu 14.04 and Ubuntu 16.04 ## Dependencies @@ -14,7 +14,7 @@ sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev p pip install numpy funcsigs colorama psutil redis pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4 # We use the latest version of cloudpickle because it can serialize named tuples. -pip install --upgrade --verbose git+git://github.com/ray-project/numbuf.git@488f881d708bc54e86ed375ee97aa94540808fa1 +pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=ray&subdirectory=numbuf" ``` # Install Ray diff --git a/install-dependencies.sh b/install-dependencies.sh index d6104f50b..546989d15 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -37,4 +37,3 @@ elif [[ $platform == "macosx" ]]; then fi sudo pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4 # We use the latest version of cloudpickle because it can serialize named tuples. -sudo pip install --upgrade --verbose git+git://github.com/ray-project/numbuf.git@488f881d708bc54e86ed375ee97aa94540808fa1 diff --git a/numbuf/.clang-format b/numbuf/.clang-format new file mode 100644 index 000000000..7d5b3cf30 --- /dev/null +++ b/numbuf/.clang-format @@ -0,0 +1,65 @@ +--- +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: false +AlignConsecutiveAssignments: false +AlignEscapedNewlinesLeft: true +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: true +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: true +BinPackParameters: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +ColumnLimit: 90 +CommentPragmas: '^ IWYU pragma:' +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +IndentCaseLabels: true +IndentWidth: 2 +IndentWrappedFunctionNames: false +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 1000 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 8 +UseTab: Never diff --git a/numbuf/CMakeLists.txt b/numbuf/CMakeLists.txt new file mode 100644 index 000000000..a8d400ee6 --- /dev/null +++ b/numbuf/CMakeLists.txt @@ -0,0 +1,113 @@ +cmake_minimum_required(VERSION 2.8) + +project(numbuf) + +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) + +# Make libnumbuf.so look for shared libraries in the folder libnumbuf.so is in +set(CMAKE_INSTALL_RPATH "$ORIGIN/") +set(CMAKE_MACOSX_RPATH 1) + +if(NOT APPLE) + find_package(PythonInterp REQUIRED) + find_package(PythonLibs REQUIRED) + set(CUSTOM_PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}) +else() + find_program(CUSTOM_PYTHON_EXECUTABLE python) + message("-- Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}") + execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c + "import sys; print 'python' + sys.version[0:3]" + OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c + "import sys; print sys.exec_prefix" + OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) + FIND_LIBRARY(PYTHON_LIBRARIES + NAMES ${PYTHON_LIBRARY_NAME} + HINTS "${PYTHON_PREFIX}" + PATH_SUFFIXES "lib" "libs" + NO_DEFAULT_PATH) + execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c + "from distutils.sysconfig import *; print get_python_inc()" + OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE) + if(PYTHON_LIBRARIES AND PYTHON_INCLUDE_DIRS) + SET(PYTHONLIBS_FOUND TRUE) + message("-- Found PythonLibs: " ${PYTHON_LIBRARIES}) + message("-- -- Used custom search path") + else() + find_package(PythonLibs REQUIRED) + message("-- -- Used find_package(PythonLibs)") + endif() +endif() + +find_package(NumPy REQUIRED) + +if(APPLE) + SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif(APPLE) + +include_directories("${PYTHON_INCLUDE_DIRS}") +include_directories("${NUMPY_INCLUDE_DIR}") + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + +if (UNIX AND NOT APPLE) + link_libraries(rt) +endif() + +set(ARROW_DIR "${CMAKE_SOURCE_DIR}/thirdparty/arrow/" CACHE STRING + "Path of the arrow source directory") + +if (APPLE) + set(ARROW_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.dylib" CACHE STRING + "Path to libarrow.dylib (needs to be changed if arrow is build in debug mode)") + + set(ARROW_IO_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_io.dylib" CACHE STRING + "Path to libarrow_io.dylib (needs to be changed if arrow is build in debug mode)") + + set(ARROW_IPC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_ipc.dylib" CACHE STRING + "Path to libarrow_ipc.dylib (needs to be changed if arrow is build in debug mode)") +else() + set(ARROW_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.so" CACHE STRING + "Path to libarrow.so (needs to be changed if arrow is build in debug mode)") + + set(ARROW_IO_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_io.so" CACHE STRING + "Path to libarrow_io.so (needs to be changed if arrow is build in debug mode)") + + set(ARROW_IPC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_ipc.so" CACHE STRING + "Path to libarrow_ipc.so (needs to be changed if arrow is build in debug mode)") +endif() + +include_directories("${ARROW_DIR}/cpp/src/") +include_directories("cpp/src/") +include_directories("python/src/") + +add_definitions(-fPIC) + +add_library(numbuf SHARED + cpp/src/numbuf/tensor.cc + cpp/src/numbuf/dict.cc + cpp/src/numbuf/sequence.cc + python/src/pynumbuf/numbuf.cc + python/src/pynumbuf/adapters/numpy.cc + python/src/pynumbuf/adapters/python.cc) + +get_filename_component(PYTHON_SHARED_LIBRARY ${PYTHON_LIBRARIES} NAME) +if(APPLE) + add_custom_command(TARGET numbuf + POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libnumbuf.so) + add_custom_command(TARGET numbuf + POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -change "@rpath/libarrow.dylib" "@loader_path/libarrow.dylib" libnumbuf.so) + add_custom_command(TARGET numbuf + POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -change "@rpath/libarrow_io.dylib" "@loader_path/libarrow_io.dylib" libnumbuf.so) + add_custom_command(TARGET numbuf + POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -change "@rpath/libarrow_ipc.dylib" "@loader_path/libarrow_ipc.dylib" libnumbuf.so) +endif(APPLE) + +target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES}) + +install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/) +install(FILES ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/) diff --git a/numbuf/build.sh b/numbuf/build.sh new file mode 100755 index 000000000..087c1471f --- /dev/null +++ b/numbuf/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +# Determine how many parallel jobs to use for make based on the number of cores +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + PARALLEL=$(nproc) +elif [[ "$unamestr" == "Darwin" ]]; then + PARALLEL=$(sysctl -n hw.ncpu) +else + echo "Unrecognized platform." + exit 1 +fi + +mkdir -p "$ROOT_DIR/build" +pushd "$ROOT_DIR/build" + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g" -DCMAKE_CXX_FLAGS="-g" .. + make install -j$PARALLEL +popd diff --git a/numbuf/cmake/Modules/FindNumPy.cmake b/numbuf/cmake/Modules/FindNumPy.cmake new file mode 100644 index 000000000..0b9fb3e5c --- /dev/null +++ b/numbuf/cmake/Modules/FindNumPy.cmake @@ -0,0 +1,54 @@ +# - Find the NumPy libraries +# This module finds if NumPy is installed, and sets the following variables +# indicating where it is. +# +# +# NUMPY_FOUND - was NumPy found +# NUMPY_VERSION - the version of NumPy found as a string +# NUMPY_VERSION_MAJOR - the major version number of NumPy +# NUMPY_VERSION_MINOR - the minor version number of NumPy +# NUMPY_VERSION_PATCH - the patch version number of NumPy +# NUMPY_VERSION_DECIMAL - e.g. version 1.6.1 is 10601 +# NUMPY_INCLUDE_DIR - path to the NumPy include files + +unset(NUMPY_VERSION) +unset(NUMPY_INCLUDE_DIR) + +if(NOT "${CUSTOM_PYTHON_EXECUTABLE}" STREQUAL "CUSTOM_PYTHON_EXECUTABLE-NOTFOUND") + execute_process(COMMAND "${CUSTOM_PYTHON_EXECUTABLE}" "-c" + "import numpy as n; print(n.__version__); print(n.get_include());" + RESULT_VARIABLE __result + OUTPUT_VARIABLE __output + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(__result MATCHES 0) + string(REGEX REPLACE ";" "\\\\;" __values ${__output}) + string(REGEX REPLACE "\r?\n" ";" __values ${__values}) + list(GET __values 0 NUMPY_VERSION) + list(GET __values 1 NUMPY_INCLUDE_DIR) + + string(REGEX MATCH "^([0-9])+\\.([0-9])+\\.([0-9])+" __ver_check "${NUMPY_VERSION}") + if(NOT "${__ver_check}" STREQUAL "") + set(NUMPY_VERSION_MAJOR ${CMAKE_MATCH_1}) + set(NUMPY_VERSION_MINOR ${CMAKE_MATCH_2}) + set(NUMPY_VERSION_PATCH ${CMAKE_MATCH_3}) + math(EXPR NUMPY_VERSION_DECIMAL + "(${NUMPY_VERSION_MAJOR} * 10000) + (${NUMPY_VERSION_MINOR} * 100) + ${NUMPY_VERSION_PATCH}") + string(REGEX REPLACE "\\\\" "/" NUMPY_INCLUDE_DIR ${NUMPY_INCLUDE_DIR}) + else() + unset(NUMPY_VERSION) + unset(NUMPY_INCLUDE_DIR) + message(STATUS "Requested NumPy version and include path, but got instead:\n${__output}\n") + endif() + endif() +else() + message(STATUS "To find NumPy Python executable is required to be found.") +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(NumPy REQUIRED_VARS NUMPY_INCLUDE_DIR NUMPY_VERSION + VERSION_VAR NUMPY_VERSION) + +if(NUMPY_FOUND) + message(STATUS "NumPy ver. ${NUMPY_VERSION} found (include: ${NUMPY_INCLUDE_DIR})") +endif() diff --git a/numbuf/cpp/src/numbuf/dict.cc b/numbuf/cpp/src/numbuf/dict.cc new file mode 100644 index 000000000..047191d8c --- /dev/null +++ b/numbuf/cpp/src/numbuf/dict.cc @@ -0,0 +1,24 @@ +#include "dict.h" + +using namespace arrow; + +namespace numbuf { + +Status DictBuilder::Finish(std::shared_ptr key_tuple_data, + std::shared_ptr val_list_data, std::shared_ptr val_tuple_data, + std::shared_ptr val_dict_data, std::shared_ptr* out) { + // lists and dicts can't be keys of dicts in Python, that is why for + // the keys we do not need to collect sublists + std::shared_ptr keys, vals; + RETURN_NOT_OK(keys_.Finish(nullptr, key_tuple_data, nullptr, &keys)); + RETURN_NOT_OK(vals_.Finish(val_list_data, val_tuple_data, val_dict_data, &vals)); + auto keys_field = std::make_shared("keys", keys->type()); + auto vals_field = std::make_shared("vals", vals->type()); + auto type = + std::make_shared(std::vector({keys_field, vals_field})); + std::vector field_arrays({keys, vals}); + DCHECK(keys->length() == vals->length()); + out->reset(new StructArray(type, keys->length(), field_arrays)); + return Status::OK(); +} +} diff --git a/numbuf/cpp/src/numbuf/dict.h b/numbuf/cpp/src/numbuf/dict.h new file mode 100644 index 000000000..c8f5925a7 --- /dev/null +++ b/numbuf/cpp/src/numbuf/dict.h @@ -0,0 +1,46 @@ +#ifndef NUMBUF_DICT_H +#define NUMBUF_DICT_H + +#include + +#include "sequence.h" + +namespace numbuf { + +/*! Constructing dictionaries of key/value pairs. Sequences of + keys and values are built separately using a pair of + SequenceBuilders. The resulting Arrow representation + can be obtained via the Finish method. +*/ +class DictBuilder { + public: + DictBuilder(arrow::MemoryPool* pool = nullptr) : keys_(pool), vals_(pool) {} + + //! Builder for the keys of the dictionary + SequenceBuilder& keys() { return keys_; } + //! Builder for the values of the dictionary + SequenceBuilder& vals() { return vals_; } + + /*! Construct an Arrow StructArray representing the dictionary. + Contains a field "keys" for the keys and "vals" for the values. + + \param list_data + List containing the data from nested lists in the value + list of the dictionary + + \param dict_data + List containing the data from nested dictionaries in the + value list of the dictionary + */ + arrow::Status Finish(std::shared_ptr key_tuple_data, + std::shared_ptr val_list_data, + std::shared_ptr val_tuple_data, + std::shared_ptr val_dict_data, std::shared_ptr* out); + + private: + SequenceBuilder keys_; + SequenceBuilder vals_; +}; +} + +#endif diff --git a/numbuf/cpp/src/numbuf/sequence.cc b/numbuf/cpp/src/numbuf/sequence.cc new file mode 100644 index 000000000..b1b5fe2e6 --- /dev/null +++ b/numbuf/cpp/src/numbuf/sequence.cc @@ -0,0 +1,178 @@ +#include "sequence.h" + +using namespace arrow; + +namespace numbuf { + +SequenceBuilder::SequenceBuilder(MemoryPool* pool) + : pool_(pool), + types_(pool, std::make_shared()), + offsets_(pool, std::make_shared()), + nones_(pool, std::make_shared()), + bools_(pool, std::make_shared()), + ints_(pool, std::make_shared()), + bytes_(pool, std::make_shared()), + strings_(pool, std::make_shared()), + floats_(pool, std::make_shared()), + doubles_(pool, std::make_shared()), + uint8_tensors_(std::make_shared(), pool), + int8_tensors_(std::make_shared(), pool), + uint16_tensors_(std::make_shared(), pool), + int16_tensors_(std::make_shared(), pool), + uint32_tensors_(std::make_shared(), pool), + int32_tensors_(std::make_shared(), pool), + uint64_tensors_(std::make_shared(), pool), + int64_tensors_(std::make_shared(), pool), + float_tensors_(std::make_shared(), pool), + double_tensors_(std::make_shared(), pool), + list_offsets_({0}), + tuple_offsets_({0}), + dict_offsets_({0}) {} + +#define UPDATE(OFFSET, TAG) \ + if (TAG == -1) { \ + TAG = num_tags; \ + num_tags += 1; \ + } \ + RETURN_NOT_OK(offsets_.Append(OFFSET)); \ + RETURN_NOT_OK(types_.Append(TAG)); \ + RETURN_NOT_OK(nones_.AppendToBitmap(true)); + +Status SequenceBuilder::AppendNone() { + RETURN_NOT_OK(offsets_.Append(0)); + RETURN_NOT_OK(types_.Append(0)); + return nones_.AppendToBitmap(false); +} + +Status SequenceBuilder::AppendBool(bool data) { + UPDATE(bools_.length(), bool_tag); + return bools_.Append(data); +} + +Status SequenceBuilder::AppendInt64(int64_t data) { + UPDATE(ints_.length(), int_tag); + return ints_.Append(data); +} + +Status SequenceBuilder::AppendUInt64(uint64_t data) { + UPDATE(ints_.length(), int_tag); + return ints_.Append(data); +} + +Status SequenceBuilder::AppendBytes(const uint8_t* data, int32_t length) { + UPDATE(bytes_.length(), bytes_tag); + return bytes_.Append(data, length); +} + +Status SequenceBuilder::AppendString(const char* data, int32_t length) { + UPDATE(strings_.length(), string_tag); + return strings_.Append(data, length); +} + +Status SequenceBuilder::AppendFloat(float data) { + UPDATE(floats_.length(), float_tag); + return floats_.Append(data); +} + +Status SequenceBuilder::AppendDouble(double data) { + UPDATE(doubles_.length(), double_tag); + return doubles_.Append(data); +} + +#define DEF_TENSOR_APPEND(NAME, TYPE, TAG) \ + Status SequenceBuilder::AppendTensor(const std::vector& dims, TYPE* data) { \ + if (TAG == -1) { NAME.Start(); } \ + UPDATE(NAME.length(), TAG); \ + return NAME.Append(dims, data); \ + } + +DEF_TENSOR_APPEND(uint8_tensors_, uint8_t, uint8_tensor_tag); +DEF_TENSOR_APPEND(int8_tensors_, int8_t, int8_tensor_tag); +DEF_TENSOR_APPEND(uint16_tensors_, uint16_t, uint16_tensor_tag); +DEF_TENSOR_APPEND(int16_tensors_, int16_t, int16_tensor_tag); +DEF_TENSOR_APPEND(uint32_tensors_, uint32_t, uint32_tensor_tag); +DEF_TENSOR_APPEND(int32_tensors_, int32_t, int32_tensor_tag); +DEF_TENSOR_APPEND(uint64_tensors_, uint64_t, uint64_tensor_tag); +DEF_TENSOR_APPEND(int64_tensors_, int64_t, int64_tensor_tag); +DEF_TENSOR_APPEND(float_tensors_, float, float_tensor_tag); +DEF_TENSOR_APPEND(double_tensors_, double, double_tensor_tag); + +Status SequenceBuilder::AppendList(int32_t size) { + UPDATE(list_offsets_.size() - 1, list_tag); + list_offsets_.push_back(list_offsets_.back() + size); + return Status::OK(); +} + +Status SequenceBuilder::AppendTuple(int32_t size) { + UPDATE(tuple_offsets_.size() - 1, tuple_tag); + tuple_offsets_.push_back(tuple_offsets_.back() + size); + return Status::OK(); +} + +Status SequenceBuilder::AppendDict(int32_t size) { + UPDATE(dict_offsets_.size() - 1, dict_tag); + dict_offsets_.push_back(dict_offsets_.back() + size); + return Status::OK(); +} + +#define ADD_ELEMENT(VARNAME, TAG) \ + if (TAG != -1) { \ + types[TAG] = std::make_shared("", VARNAME.type()); \ + RETURN_NOT_OK(VARNAME.Finish(&children[TAG])); \ + RETURN_NOT_OK(nones_.AppendToBitmap(true)); \ + } + +#define ADD_SUBSEQUENCE(DATA, OFFSETS, BUILDER, TAG, NAME) \ + if (DATA) { \ + DCHECK(DATA->length() == OFFSETS.back()); \ + auto list_builder = std::make_shared(pool_, DATA); \ + auto field = std::make_shared(NAME, list_builder->type()); \ + auto type = std::make_shared(std::vector({field})); \ + auto lists = std::vector>({list_builder}); \ + StructBuilder builder(pool_, type, lists); \ + OFFSETS.pop_back(); \ + ARROW_CHECK_OK(list_builder->Append(OFFSETS.data(), OFFSETS.size())); \ + builder.Append(); \ + ADD_ELEMENT(builder, TAG); \ + } else { \ + DCHECK(OFFSETS.size() == 1); \ + } + +Status SequenceBuilder::Finish(std::shared_ptr list_data, + std::shared_ptr tuple_data, std::shared_ptr dict_data, + std::shared_ptr* out) { + std::vector> types(num_tags); + std::vector children(num_tags); + + ADD_ELEMENT(bools_, bool_tag); + ADD_ELEMENT(ints_, int_tag); + ADD_ELEMENT(strings_, string_tag); + ADD_ELEMENT(bytes_, bytes_tag); + ADD_ELEMENT(floats_, float_tag); + ADD_ELEMENT(doubles_, double_tag); + + ADD_ELEMENT(uint8_tensors_, uint8_tensor_tag); + + ADD_ELEMENT(int8_tensors_, int8_tensor_tag); + ADD_ELEMENT(uint16_tensors_, uint16_tensor_tag); + ADD_ELEMENT(int16_tensors_, int16_tensor_tag); + ADD_ELEMENT(uint32_tensors_, uint32_tensor_tag); + + ADD_ELEMENT(int32_tensors_, int32_tensor_tag); + ADD_ELEMENT(uint64_tensors_, uint64_tensor_tag); + ADD_ELEMENT(int64_tensors_, int64_tensor_tag); + + ADD_ELEMENT(float_tensors_, float_tensor_tag); + ADD_ELEMENT(double_tensors_, double_tensor_tag); + + ADD_SUBSEQUENCE(list_data, list_offsets_, list_builder, list_tag, "list"); + ADD_SUBSEQUENCE(tuple_data, tuple_offsets_, tuple_builder, tuple_tag, "tuple"); + ADD_SUBSEQUENCE(dict_data, dict_offsets_, dict_builder, dict_tag, "dict"); + + std::vector type_ids = {}; + TypePtr type = TypePtr(new UnionType(types, type_ids, UnionMode::DENSE)); + out->reset(new UnionArray(type, types_.length(), children, types_.data(), + offsets_.data(), nones_.null_count(), nones_.null_bitmap())); + return Status::OK(); +} +} diff --git a/numbuf/cpp/src/numbuf/sequence.h b/numbuf/cpp/src/numbuf/sequence.h new file mode 100644 index 000000000..4c87d81a6 --- /dev/null +++ b/numbuf/cpp/src/numbuf/sequence.h @@ -0,0 +1,142 @@ +#ifndef NUMBUF_LIST_H +#define NUMBUF_LIST_H + +#include "tensor.h" +#include +#include + +namespace numbuf { + +/*! A Sequence is a heterogeneous collections of elements. It can contain + scalar Python types, lists, tuples, dictionaries and tensors. +*/ +class SequenceBuilder { + public: + SequenceBuilder(arrow::MemoryPool* pool = nullptr); + + //! Appending a none to the sequence + arrow::Status AppendNone(); + + //! Appending a boolean to the sequence + arrow::Status AppendBool(bool data); + + //! Appending an int64_t to the sequence + arrow::Status AppendInt64(int64_t data); + + //! Appending an uint64_t to the sequence + arrow::Status AppendUInt64(uint64_t data); + + //! Append a list of bytes to the sequence + arrow::Status AppendBytes(const uint8_t* data, int32_t length); + + //! Appending a string to the sequence + arrow::Status AppendString(const char* data, int32_t length); + + //! Appending a float to the sequence + arrow::Status AppendFloat(float data); + + //! Appending a double to the sequence + arrow::Status AppendDouble(double data); + + /*! Appending a tensor to the sequence + + \param dims + A vector of dimensions + + \param data + A pointer to the start of the data block. The length of the data block + will be the product of the dimensions + */ + arrow::Status AppendTensor(const std::vector& dims, uint8_t* data); + arrow::Status AppendTensor(const std::vector& dims, int8_t* data); + arrow::Status AppendTensor(const std::vector& dims, uint16_t* data); + arrow::Status AppendTensor(const std::vector& dims, int16_t* data); + arrow::Status AppendTensor(const std::vector& dims, uint32_t* data); + arrow::Status AppendTensor(const std::vector& dims, int32_t* data); + arrow::Status AppendTensor(const std::vector& dims, uint64_t* data); + arrow::Status AppendTensor(const std::vector& dims, int64_t* data); + arrow::Status AppendTensor(const std::vector& dims, float* data); + arrow::Status AppendTensor(const std::vector& dims, double* data); + + /*! Add a sublist to the sequenc. The data contained in the sublist will be + specified in the "Finish" method. + + To construct l = [[11, 22], 33, [44, 55]] you would for example run + list = ListBuilder(); + list.AppendList(2); + list.Append(33); + list.AppendList(2); + list.Finish([11, 22, 44, 55]); + list.Finish(); + + \param size + The size of the sublist + */ + arrow::Status AppendList(int32_t size); + + arrow::Status AppendTuple(int32_t size); + + arrow::Status AppendDict(int32_t size); + + //! Finish building the sequence and return the result + arrow::Status Finish(std::shared_ptr list_data, + std::shared_ptr tuple_data, std::shared_ptr dict_data, + std::shared_ptr* out); + + private: + arrow::MemoryPool* pool_; + + arrow::Int8Builder types_; + arrow::Int32Builder offsets_; + + arrow::NullArrayBuilder nones_; + arrow::BooleanBuilder bools_; + arrow::Int64Builder ints_; + arrow::BinaryBuilder bytes_; + arrow::StringBuilder strings_; + arrow::FloatBuilder floats_; + arrow::DoubleBuilder doubles_; + + UInt8TensorBuilder uint8_tensors_; + Int8TensorBuilder int8_tensors_; + UInt16TensorBuilder uint16_tensors_; + Int16TensorBuilder int16_tensors_; + UInt32TensorBuilder uint32_tensors_; + Int32TensorBuilder int32_tensors_; + UInt64TensorBuilder uint64_tensors_; + Int64TensorBuilder int64_tensors_; + FloatTensorBuilder float_tensors_; + DoubleTensorBuilder double_tensors_; + + std::vector list_offsets_; + std::vector tuple_offsets_; + std::vector dict_offsets_; + + int8_t bool_tag = -1; + int8_t int_tag = -1; + int8_t string_tag = -1; + int8_t bytes_tag = -1; + int8_t float_tag = -1; + int8_t double_tag = -1; + + int8_t uint8_tensor_tag = -1; + int8_t int8_tensor_tag = -1; + int8_t uint16_tensor_tag = -1; + int8_t int16_tensor_tag = -1; + int8_t uint32_tensor_tag = -1; + int8_t int32_tensor_tag = -1; + int8_t uint64_tensor_tag = -1; + int8_t int64_tensor_tag = -1; + int8_t float_tensor_tag = -1; + int8_t double_tensor_tag = -1; + + int8_t list_tag = -1; + int8_t tuple_tag = -1; + int8_t dict_tag = -1; + + int8_t num_tags = 0; +}; + +} // namespace numbuf + +#endif // NUMBUF_LIST_H diff --git a/numbuf/cpp/src/numbuf/tensor.cc b/numbuf/cpp/src/numbuf/tensor.cc new file mode 100644 index 000000000..db9325563 --- /dev/null +++ b/numbuf/cpp/src/numbuf/tensor.cc @@ -0,0 +1,56 @@ +#include "tensor.h" + +using namespace arrow; + +namespace numbuf { + +template +TensorBuilder::TensorBuilder(const TypePtr& dtype, MemoryPool* pool) + : dtype_(dtype), pool_(pool) {} + +template +Status TensorBuilder::Start() { + dim_data_ = std::make_shared(pool_, std::make_shared()); + dims_ = std::make_shared(pool_, dim_data_); + value_data_ = std::make_shared>(pool_, dtype_); + values_ = std::make_shared(pool_, value_data_); + auto dims_field = std::make_shared("dims", dims_->type()); + auto values_field = std::make_shared("data", values_->type()); + auto type = + std::make_shared(std::vector({dims_field, values_field})); + tensors_ = std::make_shared( + pool_, type, std::vector>({dims_, values_})); + return Status::OK(); +} + +template +Status TensorBuilder::Append(const std::vector& dims, const elem_type* data) { + DCHECK(tensors_); + RETURN_NOT_OK(tensors_->Append()); + RETURN_NOT_OK(dims_->Append()); + RETURN_NOT_OK(values_->Append()); + int32_t size = 1; + for (auto dim : dims) { + size *= dim; + RETURN_NOT_OK(dim_data_->Append(dim)); + } + RETURN_NOT_OK(value_data_->Append(data, size)); + return Status::OK(); // tensors_->Append(); +} + +template +Status TensorBuilder::Finish(std::shared_ptr* out) { + return tensors_->Finish(out); +} + +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +} diff --git a/numbuf/cpp/src/numbuf/tensor.h b/numbuf/cpp/src/numbuf/tensor.h new file mode 100644 index 000000000..2725f4615 --- /dev/null +++ b/numbuf/cpp/src/numbuf/tensor.h @@ -0,0 +1,65 @@ +#ifndef NUMBUF_TENSOR_H +#define NUMBUF_TENSOR_H + +#include +#include +#include + +namespace numbuf { + +/*! This is a class for building a dataframe where each row corresponds to + a Tensor (= multidimensional array) of numerical data. There are two + columns, "dims" which contains an array of dimensions for each Tensor + and "data" which contains data buffer of the Tensor as a flattened array. +*/ +template +class TensorBuilder { + public: + typedef typename T::c_type elem_type; + + TensorBuilder(const arrow::TypePtr& dtype, arrow::MemoryPool* pool = nullptr); + + arrow::Status Start(); + + /*! Append a new tensor. + + \param dims + The dimensions of the Tensor + + \param data + Pointer to the beginning of the data buffer of the Tensor. The + total length of the buffer is sizeof(elem_type) * product of dims[i] over i + */ + arrow::Status Append(const std::vector& dims, const elem_type* data); + + //! Convert the tensors to an Arrow StructArray + arrow::Status Finish(std::shared_ptr* out); + + //! Number of tensors in the column + int32_t length() { return tensors_->length(); } + + const arrow::TypePtr& type() { return tensors_->type(); } + + private: + arrow::TypePtr dtype_; + arrow::MemoryPool* pool_; + std::shared_ptr dim_data_; + std::shared_ptr dims_; + std::shared_ptr> value_data_; + std::shared_ptr values_; + std::shared_ptr tensors_; +}; + +typedef TensorBuilder UInt8TensorBuilder; +typedef TensorBuilder Int8TensorBuilder; +typedef TensorBuilder UInt16TensorBuilder; +typedef TensorBuilder Int16TensorBuilder; +typedef TensorBuilder UInt32TensorBuilder; +typedef TensorBuilder Int32TensorBuilder; +typedef TensorBuilder UInt64TensorBuilder; +typedef TensorBuilder Int64TensorBuilder; +typedef TensorBuilder FloatTensorBuilder; +typedef TensorBuilder DoubleTensorBuilder; +} + +#endif // NUMBUF_TENSOR_H diff --git a/numbuf/numbuf/__init__.py b/numbuf/numbuf/__init__.py new file mode 100644 index 000000000..21cb259ba --- /dev/null +++ b/numbuf/numbuf/__init__.py @@ -0,0 +1 @@ +from libnumbuf import * diff --git a/numbuf/python/src/pynumbuf/adapters/numpy.cc b/numbuf/python/src/pynumbuf/adapters/numpy.cc new file mode 100644 index 000000000..238eb0092 --- /dev/null +++ b/numbuf/python/src/pynumbuf/adapters/numpy.cc @@ -0,0 +1,127 @@ +#include "numpy.h" +#include "python.h" + +#include + +#include + +using namespace arrow; + +extern "C" { +extern PyObject* numbuf_serialize_callback; +extern PyObject* numbuf_deserialize_callback; +} + +namespace numbuf { + +#define ARROW_TYPE_TO_NUMPY_CASE(TYPE) \ + case Type::TYPE: \ + return NPY_##TYPE; + +#define DESERIALIZE_ARRAY_CASE(TYPE, ArrayType, type) \ + case Type::TYPE: { \ + auto values = std::dynamic_pointer_cast(content->values()); \ + DCHECK(values); \ + type* data = const_cast(values->raw_data()) + content->offset(offset); \ + *out = PyArray_SimpleNewFromData( \ + num_dims, dim.data(), NPY_##TYPE, reinterpret_cast(data)); \ + if (base != Py_None) { PyArray_SetBaseObject((PyArrayObject*)*out, base); } \ + Py_XINCREF(base); \ + } \ + return Status::OK(); + +Status DeserializeArray( + std::shared_ptr array, int32_t offset, PyObject* base, PyObject** out) { + DCHECK(array); + auto tensor = std::dynamic_pointer_cast(array); + DCHECK(tensor); + auto dims = std::dynamic_pointer_cast(tensor->field(0)); + auto content = std::dynamic_pointer_cast(tensor->field(1)); + npy_intp num_dims = dims->value_length(offset); + std::vector dim(num_dims); + for (int i = dims->offset(offset); i < dims->offset(offset + 1); ++i) { + dim[i - dims->offset(offset)] = + std::dynamic_pointer_cast(dims->values())->Value(i); + } + switch (content->value_type()->type) { + DESERIALIZE_ARRAY_CASE(INT8, Int8Array, int8_t) + DESERIALIZE_ARRAY_CASE(INT16, Int16Array, int16_t) + DESERIALIZE_ARRAY_CASE(INT32, Int32Array, int32_t) + DESERIALIZE_ARRAY_CASE(INT64, Int64Array, int64_t) + DESERIALIZE_ARRAY_CASE(UINT8, UInt8Array, uint8_t) + DESERIALIZE_ARRAY_CASE(UINT16, UInt16Array, uint16_t) + DESERIALIZE_ARRAY_CASE(UINT32, UInt32Array, uint32_t) + DESERIALIZE_ARRAY_CASE(UINT64, UInt64Array, uint64_t) + DESERIALIZE_ARRAY_CASE(FLOAT, FloatArray, float) + DESERIALIZE_ARRAY_CASE(DOUBLE, DoubleArray, double) + default: + DCHECK(false) << "arrow type not recognized: " << content->value_type()->type; + } + return Status::OK(); +} + +Status SerializeArray( + PyArrayObject* array, SequenceBuilder& builder, std::vector& subdicts) { + size_t ndim = PyArray_NDIM(array); + int dtype = PyArray_TYPE(array); + std::vector dims(ndim); + for (int i = 0; i < ndim; ++i) { + dims[i] = PyArray_DIM(array, i); + } + // TODO(pcm): Once we don't use builders any more below and directly share + // the memory buffer, we need to be more careful about this and not + // decrease the reference count of "contiguous" before the serialization + // is finished + auto contiguous = PyArray_GETCONTIGUOUS(array); + auto data = PyArray_DATA(contiguous); + switch (dtype) { + case NPY_UINT8: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + case NPY_INT8: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + case NPY_UINT16: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + case NPY_INT16: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + case NPY_UINT32: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + case NPY_INT32: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + case NPY_UINT64: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + case NPY_INT64: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + case NPY_FLOAT: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + case NPY_DOUBLE: + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); + break; + default: + if (!numbuf_serialize_callback) { + std::stringstream stream; + stream << "numpy data type not recognized: " << dtype; + return Status::NotImplemented(stream.str()); + } else { + PyObject* arglist = Py_BuildValue("(O)", array); + // The reference count of the result of the call to PyObject_CallObject + // must be decremented. This is done in SerializeDict in python.cc. + PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); + Py_XDECREF(arglist); + if (!result) { return Status::NotImplemented("python error"); } + builder.AppendDict(PyDict_Size(result)); + subdicts.push_back(result); + } + } + Py_XDECREF(contiguous); + return Status::OK(); +} +} diff --git a/numbuf/python/src/pynumbuf/adapters/numpy.h b/numbuf/python/src/pynumbuf/adapters/numpy.h new file mode 100644 index 000000000..91f0473da --- /dev/null +++ b/numbuf/python/src/pynumbuf/adapters/numpy.h @@ -0,0 +1,23 @@ +#ifndef PYNUMBUF_NUMPY_H +#define PYNUMBUF_NUMPY_H + +#include +#include + +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API +#include + +#include +#include + +namespace numbuf { + +arrow::Status SerializeArray( + PyArrayObject* array, SequenceBuilder& builder, std::vector& subdicts); +arrow::Status DeserializeArray( + std::shared_ptr array, int32_t offset, PyObject* base, PyObject** out); +} + +#endif diff --git a/numbuf/python/src/pynumbuf/adapters/python.cc b/numbuf/python/src/pynumbuf/adapters/python.cc new file mode 100644 index 000000000..8bebe2810 --- /dev/null +++ b/numbuf/python/src/pynumbuf/adapters/python.cc @@ -0,0 +1,290 @@ +#include "python.h" + +#include + +#include "scalars.h" + +using namespace arrow; + +int32_t MAX_RECURSION_DEPTH = 100; + +extern "C" { + +extern PyObject* numbuf_serialize_callback; +extern PyObject* numbuf_deserialize_callback; +} + +namespace numbuf { + +Status get_value( + ArrayPtr arr, int32_t index, int32_t type, PyObject* base, PyObject** result) { + switch (arr->type()->type) { + case Type::BOOL: + *result = + PyBool_FromLong(std::static_pointer_cast(arr)->Value(index)); + return Status::OK(); + case Type::INT64: + *result = PyInt_FromLong(std::static_pointer_cast(arr)->Value(index)); + return Status::OK(); + case Type::BINARY: { + int32_t nchars; + const uint8_t* str = + std::static_pointer_cast(arr)->GetValue(index, &nchars); + *result = PyString_FromStringAndSize(reinterpret_cast(str), nchars); + return Status::OK(); + } + case Type::STRING: { + int32_t nchars; + const uint8_t* str = + std::static_pointer_cast(arr)->GetValue(index, &nchars); + *result = PyUnicode_FromStringAndSize(reinterpret_cast(str), nchars); + return Status::OK(); + } + case Type::FLOAT: + *result = + PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + return Status::OK(); + case Type::DOUBLE: + *result = + PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + return Status::OK(); + case Type::STRUCT: { + auto s = std::static_pointer_cast(arr); + auto l = std::static_pointer_cast(s->field(0)); + if (s->type()->child(0)->name == "list") { + return DeserializeList(l->values(), l->value_offset(index), + l->value_offset(index + 1), base, result); + } else if (s->type()->child(0)->name == "tuple") { + return DeserializeTuple(l->values(), l->value_offset(index), + l->value_offset(index + 1), base, result); + } else if (s->type()->child(0)->name == "dict") { + return DeserializeDict(l->values(), l->value_offset(index), + l->value_offset(index + 1), base, result); + } else { + return DeserializeArray(arr, index, base, result); + } + } + default: + DCHECK(false) << "union tag not recognized " << type; + } + return Status::OK(); +} + +Status append(PyObject* elem, SequenceBuilder& builder, std::vector& sublists, + std::vector& subtuples, std::vector& subdicts) { + // The bool case must precede the int case (PyInt_Check passes for bools) + if (PyBool_Check(elem)) { + RETURN_NOT_OK(builder.AppendBool(elem == Py_True)); + } else if (PyFloat_Check(elem)) { + RETURN_NOT_OK(builder.AppendDouble(PyFloat_AS_DOUBLE(elem))); + } else if (PyLong_Check(elem)) { + int overflow = 0; + int64_t data = PyLong_AsLongLongAndOverflow(elem, &overflow); + RETURN_NOT_OK(builder.AppendInt64(data)); + if (overflow) { return Status::NotImplemented("long overflow"); } + } else if (PyInt_Check(elem)) { + RETURN_NOT_OK(builder.AppendInt64(static_cast(PyInt_AS_LONG(elem)))); + } else if (PyString_Check(elem)) { + auto data = reinterpret_cast(PyString_AS_STRING(elem)); + auto size = PyString_GET_SIZE(elem); + RETURN_NOT_OK(builder.AppendBytes(data, size)); + } else if (PyUnicode_Check(elem)) { + Py_ssize_t size; +#if PY_MAJOR_VERSION >= 3 + char* data = + PyUnicode_AsUTF8AndSize(elem, &size); // TODO(pcm): Check if this is correct +#else + PyObject* str = PyUnicode_AsUTF8String(elem); + char* data = PyString_AS_STRING(str); + size = PyString_GET_SIZE(str); +#endif + Status s = builder.AppendString(data, size); + Py_XDECREF(str); + RETURN_NOT_OK(s); + } else if (PyList_Check(elem)) { + builder.AppendList(PyList_Size(elem)); + sublists.push_back(elem); + } else if (PyDict_Check(elem)) { + builder.AppendDict(PyDict_Size(elem)); + subdicts.push_back(elem); + } else if (PyTuple_CheckExact(elem)) { + builder.AppendTuple(PyTuple_Size(elem)); + subtuples.push_back(elem); + } else if (PyArray_IsScalar(elem, Generic)) { + RETURN_NOT_OK(AppendScalar(elem, builder)); + } else if (PyArray_Check(elem)) { + RETURN_NOT_OK(SerializeArray((PyArrayObject*)elem, builder, subdicts)); + } else if (elem == Py_None) { + RETURN_NOT_OK(builder.AppendNone()); + } else { + if (!numbuf_serialize_callback) { + std::stringstream ss; + ss << "data type of " << PyString_AS_STRING(PyObject_Repr(elem)) + << " not recognized and custom serialization handler not registered"; + return Status::NotImplemented(ss.str()); + } else { + PyObject* arglist = Py_BuildValue("(O)", elem); + // The reference count of the result of the call to PyObject_CallObject + // must be decremented. This is done in SerializeDict in this file. + PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); + Py_XDECREF(arglist); + if (!result) { return Status::NotImplemented("python error"); } + builder.AppendDict(PyDict_Size(result)); + subdicts.push_back(result); + } + } + return Status::OK(); +} + +Status SerializeSequences(std::vector sequences, int32_t recursion_depth, + std::shared_ptr* out) { + DCHECK(out); + if (recursion_depth >= MAX_RECURSION_DEPTH) { + return Status::NotImplemented( + "This object exceeds the maximum recursion depth. It may contain itself " + "recursively."); + } + SequenceBuilder builder(nullptr); + std::vector sublists, subtuples, subdicts; + for (const auto& sequence : sequences) { + PyObject* item; + PyObject* iterator = PyObject_GetIter(sequence); + while ((item = PyIter_Next(iterator))) { + Status s = append(item, builder, sublists, subtuples, subdicts); + Py_DECREF(item); + // if an error occurs, we need to decrement the reference counts before returning + if (!s.ok()) { + Py_DECREF(iterator); + return s; + } + } + Py_DECREF(iterator); + } + std::shared_ptr list; + if (sublists.size() > 0) { + RETURN_NOT_OK(SerializeSequences(sublists, recursion_depth + 1, &list)); + } + std::shared_ptr tuple; + if (subtuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(subtuples, recursion_depth + 1, &tuple)); + } + std::shared_ptr dict; + if (subdicts.size() > 0) { + RETURN_NOT_OK(SerializeDict(subdicts, recursion_depth + 1, &dict)); + } + return builder.Finish(list, tuple, dict, out); +} + +#define DESERIALIZE_SEQUENCE(CREATE, SET_ITEM) \ + auto data = std::dynamic_pointer_cast(array); \ + int32_t size = array->length(); \ + PyObject* result = CREATE(stop_idx - start_idx); \ + auto types = std::make_shared(size, data->types()); \ + auto offsets = std::make_shared(size, data->offset_buf()); \ + for (size_t i = start_idx; i < stop_idx; ++i) { \ + if (data->IsNull(i)) { \ + Py_INCREF(Py_None); \ + SET_ITEM(result, i - start_idx, Py_None); \ + } else { \ + int32_t offset = offsets->Value(i); \ + int8_t type = types->Value(i); \ + ArrayPtr arr = data->child(type); \ + PyObject* value; \ + RETURN_NOT_OK(get_value(arr, offset, type, base, &value)); \ + SET_ITEM(result, i - start_idx, value); \ + } \ + } \ + *out = result; \ + return Status::OK(); + +Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, + PyObject* base, PyObject** out) { + DESERIALIZE_SEQUENCE(PyList_New, PyList_SetItem) +} + +Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, + PyObject* base, PyObject** out) { + DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SetItem) +} + +Status SerializeDict( + std::vector dicts, int32_t recursion_depth, std::shared_ptr* out) { + DictBuilder result; + if (recursion_depth >= MAX_RECURSION_DEPTH) { + return Status::NotImplemented( + "This object exceeds the maximum recursion depth. It may contain itself " + "recursively."); + } + std::vector key_tuples, val_lists, val_tuples, val_dicts, dummy; + for (const auto& dict : dicts) { + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(dict, &pos, &key, &value)) { + RETURN_NOT_OK(append(key, result.keys(), dummy, key_tuples, dummy)); + DCHECK(dummy.size() == 0); + RETURN_NOT_OK(append(value, result.vals(), val_lists, val_tuples, val_dicts)); + } + } + std::shared_ptr key_tuples_arr; + if (key_tuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(key_tuples, recursion_depth + 1, &key_tuples_arr)); + } + std::shared_ptr val_list_arr; + if (val_lists.size() > 0) { + RETURN_NOT_OK(SerializeSequences(val_lists, recursion_depth + 1, &val_list_arr)); + } + std::shared_ptr val_tuples_arr; + if (val_tuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(val_tuples, recursion_depth + 1, &val_tuples_arr)); + } + std::shared_ptr val_dict_arr; + if (val_dicts.size() > 0) { + RETURN_NOT_OK(SerializeDict(val_dicts, recursion_depth + 1, &val_dict_arr)); + } + result.Finish(key_tuples_arr, val_list_arr, val_tuples_arr, val_dict_arr, out); + + // This block is used to decrement the reference counts of the results + // returned by the serialization callback, which is called in SerializeArray + // in numpy.cc as well as in DeserializeDict and in append in this file. + static PyObject* py_type = PyString_FromString("_pytype_"); + for (const auto& dict : dicts) { + if (PyDict_Contains(dict, py_type)) { + // If the dictionary contains the key "_pytype_", then the user has to + // have registered a callback. + ARROW_CHECK(numbuf_serialize_callback); + Py_XDECREF(dict); + } + } + + return Status::OK(); +} + +Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, + PyObject* base, PyObject** out) { + auto data = std::dynamic_pointer_cast(array); + // TODO(pcm): error handling, get rid of the temporary copy of the list + PyObject *keys, *vals; + PyObject* result = PyDict_New(); + ARROW_RETURN_NOT_OK(DeserializeList(data->field(0), start_idx, stop_idx, base, &keys)); + ARROW_RETURN_NOT_OK(DeserializeList(data->field(1), start_idx, stop_idx, base, &vals)); + for (size_t i = start_idx; i < stop_idx; ++i) { + PyDict_SetItem( + result, PyList_GetItem(keys, i - start_idx), PyList_GetItem(vals, i - start_idx)); + } + Py_XDECREF(keys); // PyList_GetItem(keys, ...) incremented the reference count + Py_XDECREF(vals); // PyList_GetItem(vals, ...) incremented the reference count + static PyObject* py_type = PyString_FromString("_pytype_"); + if (PyDict_Contains(result, py_type) && numbuf_deserialize_callback) { + PyObject* arglist = Py_BuildValue("(O)", result); + // The result of the call to PyObject_CallObject will be passed to Python + // and its reference count will be decremented by the interpreter. + PyObject* callback_result = PyObject_CallObject(numbuf_deserialize_callback, arglist); + Py_XDECREF(arglist); + Py_XDECREF(result); + result = callback_result; + if (!callback_result) { return Status::NotImplemented("python error"); } + } + *out = result; + return Status::OK(); +} +} diff --git a/numbuf/python/src/pynumbuf/adapters/python.h b/numbuf/python/src/pynumbuf/adapters/python.h new file mode 100644 index 000000000..dc25ad018 --- /dev/null +++ b/numbuf/python/src/pynumbuf/adapters/python.h @@ -0,0 +1,26 @@ +#ifndef PYNUMBUF_PYTHON_H +#define PYNUMBUF_PYTHON_H + +#include + +#include +#include +#include + +#include "numpy.h" + +namespace numbuf { + +arrow::Status SerializeSequences(std::vector sequences, + int32_t recursion_depth, std::shared_ptr* out); +arrow::Status SerializeDict(std::vector dicts, int32_t recursion_depth, + std::shared_ptr* out); +arrow::Status DeserializeList(std::shared_ptr array, int32_t start_idx, + int32_t stop_idx, PyObject* base, PyObject** out); +arrow::Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, + int32_t stop_idx, PyObject* base, PyObject** out); +arrow::Status DeserializeDict(std::shared_ptr array, int32_t start_idx, + int32_t stop_idx, PyObject* base, PyObject** out); +} + +#endif diff --git a/numbuf/python/src/pynumbuf/adapters/scalars.h b/numbuf/python/src/pynumbuf/adapters/scalars.h new file mode 100644 index 000000000..e64027c9e --- /dev/null +++ b/numbuf/python/src/pynumbuf/adapters/scalars.h @@ -0,0 +1,54 @@ +#ifndef PYNUMBUF_SCALARS_H +#define PYNUMBUF_SCALARS_H + +#include + +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API +#include +#include + +#include + +namespace numbuf { + +arrow::Status AppendScalar(PyObject* obj, SequenceBuilder& builder) { + if (PyArray_IsScalar(obj, Bool)) { + return builder.AppendBool(((PyBoolScalarObject*)obj)->obval != 0); + } else if (PyArray_IsScalar(obj, Float)) { + return builder.AppendFloat(((PyFloatScalarObject*)obj)->obval); + } else if (PyArray_IsScalar(obj, Double)) { + return builder.AppendDouble(((PyDoubleScalarObject*)obj)->obval); + } + int64_t value = 0; + if (PyArray_IsScalar(obj, Byte)) { + value = ((PyByteScalarObject*)obj)->obval; + } else if (PyArray_IsScalar(obj, UByte)) { + value = ((PyUByteScalarObject*)obj)->obval; + } else if (PyArray_IsScalar(obj, Short)) { + value = ((PyShortScalarObject*)obj)->obval; + } else if (PyArray_IsScalar(obj, UShort)) { + value = ((PyUShortScalarObject*)obj)->obval; + } else if (PyArray_IsScalar(obj, Int)) { + value = ((PyIntScalarObject*)obj)->obval; + } else if (PyArray_IsScalar(obj, UInt)) { + value = ((PyUIntScalarObject*)obj)->obval; + } else if (PyArray_IsScalar(obj, Long)) { + value = ((PyLongScalarObject*)obj)->obval; + } else if (PyArray_IsScalar(obj, ULong)) { + value = ((PyULongScalarObject*)obj)->obval; + } else if (PyArray_IsScalar(obj, LongLong)) { + value = ((PyLongLongScalarObject*)obj)->obval; + } else if (PyArray_IsScalar(obj, ULongLong)) { + value = ((PyULongLongScalarObject*)obj)->obval; + } else { + DCHECK(false) << "scalar type not recognized"; + } + return builder.AppendInt64(value); +} + +} // namespace + +#endif // PYNUMBUF_SCALARS_H diff --git a/numbuf/python/src/pynumbuf/memory.h b/numbuf/python/src/pynumbuf/memory.h new file mode 100644 index 000000000..acd62f523 --- /dev/null +++ b/numbuf/python/src/pynumbuf/memory.h @@ -0,0 +1,67 @@ +#ifndef PYNUMBUF_MEMORY_H +#define PYNUMBUF_MEMORY_H + +#include + +namespace numbuf { + +class FixedBufferStream : public arrow::io::OutputStream, + public arrow::io::ReadableFileInterface { + public: + virtual ~FixedBufferStream() {} + + explicit FixedBufferStream(uint8_t* data, int64_t nbytes) + : data_(data), position_(0), size_(nbytes) {} + + arrow::Status Read(int64_t nbytes, std::shared_ptr* out) override { + DCHECK(out); + DCHECK(position_ + nbytes <= size_) << "position: " << position_ + << " nbytes: " << nbytes << "size: " << size_; + *out = std::make_shared(data_ + position_, nbytes); + position_ += nbytes; + return arrow::Status::OK(); + } + + arrow::Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) { + assert(0); + return arrow::Status::OK(); + } + + arrow::Status Seek(int64_t position) override { + position_ = position; + return arrow::Status::OK(); + } + + arrow::Status Close() override { return arrow::Status::OK(); } + + arrow::Status Tell(int64_t* position) override { + *position = position_; + return arrow::Status::OK(); + } + + arrow::Status Write(const uint8_t* data, int64_t nbytes) override { + DCHECK(position_ >= 0 && position_ < size_); + DCHECK(position_ + nbytes <= size_) << "position: " << position_ + << " nbytes: " << nbytes << "size: " << size_; + uint8_t* dst = data_ + position_; + memcpy(dst, data, nbytes); + position_ += nbytes; + return arrow::Status::OK(); + } + + arrow::Status GetSize(int64_t* size) override { + *size = size_; + return arrow::Status::OK(); + } + + bool supports_zero_copy() const override { return true; } + + private: + uint8_t* data_; + int64_t position_; + int64_t size_; +}; + +} // namespace numbuf + +#endif // PYNUMBUF_MEMORY_H diff --git a/numbuf/python/src/pynumbuf/numbuf.cc b/numbuf/python/src/pynumbuf/numbuf.cc new file mode 100644 index 000000000..70be81b26 --- /dev/null +++ b/numbuf/python/src/pynumbuf/numbuf.cc @@ -0,0 +1,192 @@ +#include +#include +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API +#include + +#include + +#include + +#include "adapters/python.h" +#include "memory.h" + +using namespace arrow; +using namespace numbuf; + +std::shared_ptr make_row_batch(std::shared_ptr data) { + auto field = std::make_shared("list", data->type()); + std::shared_ptr schema(new Schema({field})); + return std::shared_ptr(new RecordBatch(schema, data->length(), {data})); +} + +extern "C" { + +static PyObject* NumbufError; + +PyObject* numbuf_serialize_callback = NULL; +PyObject* numbuf_deserialize_callback = NULL; + +int PyObjectToArrow(PyObject* object, std::shared_ptr** result) { + if (PyCapsule_IsValid(object, "arrow")) { + *result = reinterpret_cast*>( + PyCapsule_GetPointer(object, "arrow")); + return 1; + } else { + PyErr_SetString(PyExc_TypeError, "must be an 'arrow' capsule"); + return 0; + } +} + +static void ArrowCapsule_Destructor(PyObject* capsule) { + delete reinterpret_cast*>( + PyCapsule_GetPointer(capsule, "arrow")); +} + +/* Documented in doc/numbuf.rst in ray-core */ +static PyObject* serialize_list(PyObject* self, PyObject* args) { + PyObject* value; + if (!PyArg_ParseTuple(args, "O", &value)) { return NULL; } + std::shared_ptr array; + if (PyList_Check(value)) { + int32_t recursion_depth = 0; + Status s = + SerializeSequences(std::vector({value}), recursion_depth, &array); + if (!s.ok()) { + // If this condition is true, there was an error in the callback that + // needs to be passed through + if (!PyErr_Occurred()) { PyErr_SetString(NumbufError, s.ToString().c_str()); } + return NULL; + } + + auto batch = new std::shared_ptr(); + *batch = make_row_batch(array); + + int64_t size = 0; + ARROW_CHECK_OK(arrow::ipc::GetRecordBatchSize(batch->get(), &size)); + + std::shared_ptr buffer; + ARROW_CHECK_OK(ipc::WriteSchema((*batch)->schema().get(), &buffer)); + auto ptr = reinterpret_cast(buffer->data()); + + PyObject* r = PyTuple_New(3); + PyTuple_SetItem(r, 0, PyByteArray_FromStringAndSize(ptr, buffer->size())); + PyTuple_SetItem(r, 1, PyInt_FromLong(size)); + PyTuple_SetItem(r, 2, + PyCapsule_New(reinterpret_cast(batch), "arrow", &ArrowCapsule_Destructor)); + return r; + } + return NULL; +} + +/* Documented in doc/numbuf.rst in ray-core */ +static PyObject* write_to_buffer(PyObject* self, PyObject* args) { + std::shared_ptr* batch; + PyObject* memoryview; + if (!PyArg_ParseTuple(args, "O&O", &PyObjectToArrow, &batch, &memoryview)) { + return NULL; + } + if (!PyMemoryView_Check(memoryview)) { return NULL; } + Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview); + auto target = std::make_shared( + reinterpret_cast(buffer->buf), buffer->len); + int64_t body_end_offset; + int64_t header_end_offset; + ARROW_CHECK_OK(ipc::WriteRecordBatch((*batch)->columns(), (*batch)->num_rows(), + target.get(), &body_end_offset, &header_end_offset)); + return PyInt_FromLong(header_end_offset); +} + +/* Documented in doc/numbuf.rst in ray-core */ +static PyObject* read_from_buffer(PyObject* self, PyObject* args) { + PyObject* memoryview; + PyObject* metadata; + int64_t metadata_offset; + if (!PyArg_ParseTuple(args, "OOL", &memoryview, &metadata, &metadata_offset)) { + return NULL; + } + + auto ptr = reinterpret_cast(PyByteArray_AsString(metadata)); + auto schema_buffer = std::make_shared(ptr, PyByteArray_Size(metadata)); + std::shared_ptr message; + ARROW_CHECK_OK(ipc::Message::Open(schema_buffer, &message)); + DCHECK_EQ(ipc::Message::SCHEMA, message->type()); + std::shared_ptr schema_msg = message->GetSchema(); + std::shared_ptr schema; + ARROW_CHECK_OK(schema_msg->GetSchema(&schema)); + + Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview); + auto source = std::make_shared( + reinterpret_cast(buffer->buf), buffer->len); + std::shared_ptr reader; + ARROW_CHECK_OK( + arrow::ipc::RecordBatchReader::Open(source.get(), metadata_offset, &reader)); + auto batch = new std::shared_ptr(); + ARROW_CHECK_OK(reader->GetRecordBatch(schema, batch)); + + return PyCapsule_New(reinterpret_cast(batch), "arrow", &ArrowCapsule_Destructor); +} + +/* Documented in doc/numbuf.rst in ray-core */ +static PyObject* deserialize_list(PyObject* self, PyObject* args) { + std::shared_ptr* data; + PyObject* base = Py_None; + if (!PyArg_ParseTuple(args, "O&|O", &PyObjectToArrow, &data, &base)) { return NULL; } + PyObject* result; + Status s = DeserializeList((*data)->column(0), 0, (*data)->num_rows(), base, &result); + if (!s.ok()) { + // If this condition is true, there was an error in the callback that + // needs to be passed through + if (!PyErr_Occurred()) { PyErr_SetString(NumbufError, s.ToString().c_str()); } + return NULL; + } + return result; +} + +static PyObject* register_callbacks(PyObject* self, PyObject* args) { + PyObject* result = NULL; + PyObject* serialize_callback; + PyObject* deserialize_callback; + if (PyArg_ParseTuple( + args, "OO:register_callbacks", &serialize_callback, &deserialize_callback)) { + if (!PyCallable_Check(serialize_callback)) { + PyErr_SetString(PyExc_TypeError, "serialize_callback must be callable"); + return NULL; + } + if (!PyCallable_Check(deserialize_callback)) { + PyErr_SetString(PyExc_TypeError, "deserialize_callback must be callable"); + return NULL; + } + Py_XINCREF(serialize_callback); // Add a reference to new serialization callback + Py_XINCREF(deserialize_callback); // Add a reference to new deserialization callback + Py_XDECREF(numbuf_serialize_callback); // Dispose of old serialization callback + Py_XDECREF(numbuf_deserialize_callback); // Dispose of old deserialization callback + numbuf_serialize_callback = serialize_callback; + numbuf_deserialize_callback = deserialize_callback; + Py_INCREF(Py_None); + result = Py_None; + } + return result; +} + +static PyMethodDef NumbufMethods[] = { + {"serialize_list", serialize_list, METH_VARARGS, "serialize a Python list"}, + {"deserialize_list", deserialize_list, METH_VARARGS, "deserialize a Python list"}, + {"write_to_buffer", write_to_buffer, METH_VARARGS, "write serialized data to buffer"}, + {"read_from_buffer", read_from_buffer, METH_VARARGS, + "read serialized data from buffer"}, + {"register_callbacks", register_callbacks, METH_VARARGS, + "set serialization and deserialization callbacks"}, + {NULL, NULL, 0, NULL}}; + +PyMODINIT_FUNC initlibnumbuf(void) { + PyObject* m; + m = Py_InitModule3("libnumbuf", NumbufMethods, "Python C Extension for Numbuf"); + char numbuf_error[] = "numbuf.error"; + NumbufError = PyErr_NewException(numbuf_error, NULL, NULL); + Py_INCREF(NumbufError); + PyModule_AddObject(m, "numbuf_error", NumbufError); + import_array(); +} +} diff --git a/numbuf/python/test/runtest.py b/numbuf/python/test/runtest.py new file mode 100644 index 000000000..56a1f0996 --- /dev/null +++ b/numbuf/python/test/runtest.py @@ -0,0 +1,114 @@ +import unittest +import numbuf +import numpy as np +from numpy.testing import assert_equal + +TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world", + u"x", u"\u262F", 42.0, + 1L << 62, (1.0, "hi"), + None, (None, None), ("hello", None), + True, False, (True, False), "hello", + {True: "hello", False: "world"}, + {"hello" : "world", 1: 42, 1.0: 45}, {}, + np.int8(3), np.int32(4), np.int64(5), + np.uint8(3), np.uint32(4), np.uint64(5), + np.float32(1.0), np.float64(1.0)] + +class SerializationTests(unittest.TestCase): + + def roundTripTest(self, data): + schema, size, serialized = numbuf.serialize_list(data) + result = numbuf.deserialize_list(serialized) + assert_equal(data, result) + + def testSimple(self): + self.roundTripTest([1, 2, 3]) + self.roundTripTest([1.0, 2.0, 3.0]) + self.roundTripTest(['hello', 'world']) + self.roundTripTest([1, 'hello', 1.0]) + self.roundTripTest([{'hello': 1.0, 'world': 42}]) + self.roundTripTest([True, False]) + + def testNone(self): + self.roundTripTest([1, 2, None, 3]) + + def testNested(self): + self.roundTripTest([{"hello": {"world": (1, 2, 3)}}]) + self.roundTripTest([((1,), (1, 2, 3, (4, 5, 6), "string"))]) + self.roundTripTest([{"hello": [1, 2, 3]}]) + self.roundTripTest([{"hello": [1, [2, 3]]}]) + self.roundTripTest([{"hello": (None, 2, [3, 4])}]) + self.roundTripTest([{"hello": (None, 2, [3, 4], np.array([1.0, 2.0, 3.0]))}]) + + def numpyTest(self, t): + a = np.random.randint(0, 10, size=(100, 100)).astype(t) + self.roundTripTest([a]) + + def testArrays(self): + for t in ["int8", "uint8", "int16", "uint16", "int32", "uint32", "float32", "float64"]: + self.numpyTest(t) + + def testRay(self): + for obj in TEST_OBJECTS: + self.roundTripTest([obj]) + + def testCallback(self): + + class Foo(object): + def __init__(self): + self.x = 1 + + class Bar(object): + def __init__(self): + self.foo = Foo() + + def serialize(obj): + return dict(obj.__dict__, **{"_pytype_": type(obj).__name__}) + + def deserialize(obj): + if obj["_pytype_"] == "Foo": + result = Foo() + elif obj["_pytype_"] == "Bar": + result = Bar() + + obj.pop("_pytype_", None) + result.__dict__ = obj + return result + + bar = Bar() + bar.foo.x = 42 + + numbuf.register_callbacks(serialize, deserialize) + + metadata, size, serialized = numbuf.serialize_list([bar]) + self.assertEqual(numbuf.deserialize_list(serialized)[0].foo.x, 42) + + def testObjectArray(self): + x = np.array([1, 2, "hello"], dtype=object) + y = np.array([[1, 2], [3, 4]], dtype=object) + + def myserialize(obj): + return {"_pytype_": "numpy.array", "data": obj.tolist()} + + def mydeserialize(obj): + if obj["_pytype_"] == "numpy.array": + return np.array(obj["data"], dtype=object) + + numbuf.register_callbacks(myserialize, mydeserialize) + + metadata, size, serialized = numbuf.serialize_list([x, y]) + + assert_equal(numbuf.deserialize_list(serialized), [x, y]) + + def testBuffer(self): + for (i, obj) in enumerate(TEST_OBJECTS): + schema, size, batch = numbuf.serialize_list([obj]) + size = size + 4096 # INITIAL_METADATA_SIZE in arrow + buff = np.zeros(size, dtype="uint8") + metadata_offset = numbuf.write_to_buffer(batch, memoryview(buff)) + array = numbuf.read_from_buffer(memoryview(buff), schema, metadata_offset) + result = numbuf.deserialize_list(array) + assert_equal(result[0], obj) + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/numbuf/setup.py b/numbuf/setup.py new file mode 100644 index 000000000..26b4f208a --- /dev/null +++ b/numbuf/setup.py @@ -0,0 +1,33 @@ +import subprocess +from setuptools import setup, find_packages +import setuptools.command.install as _install +from sys import platform + +extension = "" +if platform == "linux" or platform == "linux2": + extension = ".so" +elif platform == "darwin": + extension = ".dylib" + +# Because of relative paths, this must be run from inside numbuf/. + +class install(_install.install): + def run(self): + subprocess.check_call(["./setup.sh"]) + subprocess.check_call(["./build.sh"]) + # Calling _install.install.run(self) does not fetch required packages and + # instead performs an old-style install. See command/install.py in + # setuptools. So, calling do_egg_install() manually here. + self.do_egg_install() + +setup(name="numbuf", + version="0.0.1", + packages=find_packages(), + package_data={"numbuf": ["libnumbuf.so", + "libarrow" + extension, + "libarrow_io" + extension, + "libarrow_ipc" + extension]}, + cmdclass={"install": install}, + setup_requires=["numpy"], + include_package_data=True, + zip_safe=False) diff --git a/numbuf/setup.sh b/numbuf/setup.sh new file mode 100755 index 000000000..02a5acd73 --- /dev/null +++ b/numbuf/setup.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +platform="unknown" +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + echo "Platform is linux." + platform="linux" +elif [[ "$unamestr" == "Darwin" ]]; then + echo "Platform is macosx." + platform="macosx" +else + echo "Unrecognized platform." + exit 1 +fi + +pushd "$ROOT_DIR" + ./thirdparty/download_thirdparty.sh + ./thirdparty/build_thirdparty.sh +popd diff --git a/numbuf/thirdparty/build_thirdparty.sh b/numbuf/thirdparty/build_thirdparty.sh new file mode 100755 index 000000000..e5c4b5b45 --- /dev/null +++ b/numbuf/thirdparty/build_thirdparty.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -x +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +PREFIX=$TP_DIR/installed + +# Determine how many parallel jobs to use for make based on the number of cores +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + PARALLEL=$(nproc) +elif [[ "$unamestr" == "Darwin" ]]; then + PARALLEL=$(sysctl -n hw.ncpu) + echo "Platform is macosx." +else + echo "Unrecognized platform." + exit 1 +fi + +echo "building arrow" +cd $TP_DIR/arrow/cpp +mkdir -p $TP_DIR/arrow/cpp/build +cd $TP_DIR/arrow/cpp/build +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g" -DCMAKE_CXX_FLAGS="-g" .. +make VERBOSE=1 -j$PARALLEL diff --git a/numbuf/thirdparty/download_thirdparty.sh b/numbuf/thirdparty/download_thirdparty.sh new file mode 100755 index 000000000..c6a9ea19b --- /dev/null +++ b/numbuf/thirdparty/download_thirdparty.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -x +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +git clone https://github.com/pcmoritz/arrow.git "$TP_DIR/arrow" +cd "$TP_DIR/arrow" +git checkout 58bd7bedc63d66d5898297bab25b54dfb67665db diff --git a/numbuf/vsprojects/numbuf.vcxproj b/numbuf/vsprojects/numbuf.vcxproj new file mode 100644 index 000000000..b762afa79 --- /dev/null +++ b/numbuf/vsprojects/numbuf.vcxproj @@ -0,0 +1,75 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + {609D1438-D42D-4CBA-80A5-A1398C3BCC85} + Win32Proj + + + + v140 + DynamicLibrary + + + + + + + + + + + + + lib$(MSBuildProjectName) + .pyd + + + + $(THIRD_PARTY)arrow\cpp\src;%(AdditionalIncludeDirectories) + + + Console + + + + + + + + + + + + + + + + + + + + + + {10e7d8e8-0eeb-46ea-a58d-f9236b5960ad} + + + + \ No newline at end of file diff --git a/numbuf/vsprojects/numbuf.vcxproj.filters b/numbuf/vsprojects/numbuf.vcxproj.filters new file mode 100644 index 000000000..c20d75f12 --- /dev/null +++ b/numbuf/vsprojects/numbuf.vcxproj.filters @@ -0,0 +1,60 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + \ No newline at end of file