From b143b3679281085680c818a4652bf9c2b39872f6 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sat, 16 Jul 2016 17:05:21 -0700 Subject: [PATCH 01/32] Initial commit --- .gitignore | 29 ++++++++ LICENSE | 201 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + 3 files changed, 232 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..4581ef2ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 000000000..b94616b43 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# numbuf +Serializing primitive Python types in Arrow From 9bc7e7843ff20f46cc5f3816cc80ba62ee15ed40 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Wed, 20 Jul 2016 21:47:37 -0700 Subject: [PATCH 02/32] Capability to serialize most primitive Python types --- CMakeLists.txt | 40 +++++ cmake/Modules/FindNumPy.cmake | 54 +++++++ cpp/src/numbuf/dict.cc | 23 +++ cpp/src/numbuf/dict.h | 48 ++++++ cpp/src/numbuf/sequence.cc | 167 ++++++++++++++++++++ cpp/src/numbuf/sequence.h | 138 +++++++++++++++++ cpp/src/numbuf/tensor.cc | 50 ++++++ cpp/src/numbuf/tensor.h | 67 ++++++++ python/src/pynumbuf/adapters/numpy.cc | 88 +++++++++++ python/src/pynumbuf/adapters/numpy.h | 22 +++ python/src/pynumbuf/adapters/python.cc | 193 +++++++++++++++++++++++ python/src/pynumbuf/adapters/python.h | 22 +++ python/src/pynumbuf/adapters/scalars.h | 54 +++++++ python/src/pynumbuf/memory.h | 47 ++++++ python/src/pynumbuf/numbuf.cc | 205 +++++++++++++++++++++++++ python/test/runtest.py | 63 ++++++++ 16 files changed, 1281 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 cmake/Modules/FindNumPy.cmake create mode 100644 cpp/src/numbuf/dict.cc create mode 100644 cpp/src/numbuf/dict.h create mode 100644 cpp/src/numbuf/sequence.cc create mode 100644 cpp/src/numbuf/sequence.h create mode 100644 cpp/src/numbuf/tensor.cc create mode 100644 cpp/src/numbuf/tensor.h create mode 100644 python/src/pynumbuf/adapters/numpy.cc create mode 100644 python/src/pynumbuf/adapters/numpy.h create mode 100644 python/src/pynumbuf/adapters/python.cc create mode 100644 python/src/pynumbuf/adapters/python.h create mode 100644 python/src/pynumbuf/adapters/scalars.h create mode 100644 python/src/pynumbuf/memory.h create mode 100644 python/src/pynumbuf/numbuf.cc create mode 100644 python/test/runtest.py diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..bda3778bb --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,40 @@ +cmake_minimum_required(VERSION 2.8) + +project(numbuf) + +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) + +find_package(PythonInterp REQUIRED) +find_package(PythonLibs REQUIRED) +find_package(NumPy REQUIRED) + +include_directories("${PYTHON_INCLUDE_DIRS}") +include_directories("${NUMPY_INCLUDE_DIR}") + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + +if (UNIX AND NOT APPLE) + link_libraries(rt) +endif() + +set(ARROW_DIR "${CMAKE_SOURCE_DIR}/../arrow/" CACHE STRING + "Path of the arrow source directory") + +set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/../arrow/cpp/build/debug/libarrow.so" CACHE STRING + "Path to libarrow.a (needs to be changed if arrow is build in debug mode)") + +include_directories("${ARROW_DIR}/cpp/src/") +include_directories("cpp/src/") +include_directories("python/src/") + +add_definitions(-fPIC) + +add_library(numbuf SHARED + cpp/src/numbuf/tensor.cc + cpp/src/numbuf/dict.cc + cpp/src/numbuf/sequence.cc + python/src/pynumbuf/numbuf.cc + python/src/pynumbuf/adapters/numpy.cc + python/src/pynumbuf/adapters/python.cc) + +target_link_libraries(numbuf ${ARROW_STATIC_LIB} ${PYTHON_LIBRARIES}) diff --git a/cmake/Modules/FindNumPy.cmake b/cmake/Modules/FindNumPy.cmake new file mode 100644 index 000000000..6e1f3c415 --- /dev/null +++ b/cmake/Modules/FindNumPy.cmake @@ -0,0 +1,54 @@ +# - Find the NumPy libraries +# This module finds if NumPy is installed, and sets the following variables +# indicating where it is. +# +# +# NUMPY_FOUND - was NumPy found +# NUMPY_VERSION - the version of NumPy found as a string +# NUMPY_VERSION_MAJOR - the major version number of NumPy +# NUMPY_VERSION_MINOR - the minor version number of NumPy +# NUMPY_VERSION_PATCH - the patch version number of NumPy +# NUMPY_VERSION_DECIMAL - e.g. version 1.6.1 is 10601 +# NUMPY_INCLUDE_DIR - path to the NumPy include files + +unset(NUMPY_VERSION) +unset(NUMPY_INCLUDE_DIR) + +if(PYTHONINTERP_FOUND) + execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" + "import numpy as n; print(n.__version__); print(n.get_include());" + RESULT_VARIABLE __result + OUTPUT_VARIABLE __output + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(__result MATCHES 0) + string(REGEX REPLACE ";" "\\\\;" __values ${__output}) + string(REGEX REPLACE "\r?\n" ";" __values ${__values}) + list(GET __values 0 NUMPY_VERSION) + list(GET __values 1 NUMPY_INCLUDE_DIR) + + string(REGEX MATCH "^([0-9])+\\.([0-9])+\\.([0-9])+" __ver_check "${NUMPY_VERSION}") + if(NOT "${__ver_check}" STREQUAL "") + set(NUMPY_VERSION_MAJOR ${CMAKE_MATCH_1}) + set(NUMPY_VERSION_MINOR ${CMAKE_MATCH_2}) + set(NUMPY_VERSION_PATCH ${CMAKE_MATCH_3}) + math(EXPR NUMPY_VERSION_DECIMAL + "(${NUMPY_VERSION_MAJOR} * 10000) + (${NUMPY_VERSION_MINOR} * 100) + ${NUMPY_VERSION_PATCH}") + string(REGEX REPLACE "\\\\" "/" NUMPY_INCLUDE_DIR ${NUMPY_INCLUDE_DIR}) + else() + unset(NUMPY_VERSION) + unset(NUMPY_INCLUDE_DIR) + message(STATUS "Requested NumPy version and include path, but got instead:\n${__output}\n") + endif() + endif() +else() + message(STATUS "To find NumPy Python interpretator is required to be found.") +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(NumPy REQUIRED_VARS NUMPY_INCLUDE_DIR NUMPY_VERSION + VERSION_VAR NUMPY_VERSION) + +if(NUMPY_FOUND) + message(STATUS "NumPy ver. ${NUMPY_VERSION} found (include: ${NUMPY_INCLUDE_DIR})") +endif() diff --git a/cpp/src/numbuf/dict.cc b/cpp/src/numbuf/dict.cc new file mode 100644 index 000000000..ff5f7239f --- /dev/null +++ b/cpp/src/numbuf/dict.cc @@ -0,0 +1,23 @@ +#include "dict.h" + +using namespace arrow; + +namespace numbuf { + +std::shared_ptr DictBuilder::Finish( + std::shared_ptr list_data, + std::shared_ptr tuple_data, + std::shared_ptr dict_data) { + // lists and dicts can't be keys of dicts in Python, that is why for + // the keys we do not need to collect sublists + auto keys = keys_.Finish(nullptr, nullptr, nullptr); + auto vals = vals_.Finish(list_data, tuple_data, dict_data); + auto keys_field = std::make_shared("keys", keys->type()); + auto vals_field = std::make_shared("vals", vals->type()); + auto type = std::make_shared(std::vector({keys_field, vals_field})); + std::vector field_arrays({keys, vals}); + DCHECK(keys->length() == vals->length()); + return std::make_shared(type, keys->length(), field_arrays); +} + +} diff --git a/cpp/src/numbuf/dict.h b/cpp/src/numbuf/dict.h new file mode 100644 index 000000000..5c2abeacd --- /dev/null +++ b/cpp/src/numbuf/dict.h @@ -0,0 +1,48 @@ +#ifndef NUMBUF_DICT_H +#define NUMBUF_DICT_H + +#include + +#include "sequence.h" + +namespace numbuf { + +/*! Constructing dictionaries of key/value pairs. Sequences of + keys and values are built separately using a pair of + SequenceBuilders. The resulting Arrow representation + can be obtained via the Finish method. +*/ +class DictBuilder { +public: + DictBuilder(arrow::MemoryPool* pool = nullptr) + : keys_(pool), vals_(pool) {} + + //! Builder for the keys of the dictionary + SequenceBuilder& keys() { return keys_; } + //! Builder for the values of the dictionary + SequenceBuilder& vals() { return vals_; } + + /*! Construct an Arrow StructArray representing the dictionary. + Contains a field "keys" for the keys and "vals" for the values. + + \param list_data + List containing the data from nested lists in the value + list of the dictionary + + \param dict_data + List containing the data from nested dictionaries in the + value list of the dictionary + */ + std::shared_ptr Finish( + std::shared_ptr list_data, + std::shared_ptr tuple_data, + std::shared_ptr dict_data); + +private: + SequenceBuilder keys_; + SequenceBuilder vals_; +}; + +} + +#endif diff --git a/cpp/src/numbuf/sequence.cc b/cpp/src/numbuf/sequence.cc new file mode 100644 index 000000000..ae250180f --- /dev/null +++ b/cpp/src/numbuf/sequence.cc @@ -0,0 +1,167 @@ +#include "sequence.h" + +using namespace arrow; + +namespace numbuf { + +SequenceBuilder::SequenceBuilder(MemoryPool* pool) + : pool_(pool), types_(pool), offsets_(pool), + nones_(pool, std::make_shared()), + bools_(pool, std::make_shared()), + ints_(pool), strings_(pool, std::make_shared()), + floats_(pool), doubles_(pool), + uint8_tensors_(std::make_shared(), pool), + int8_tensors_(std::make_shared(), pool), + uint16_tensors_(std::make_shared(), pool), + int16_tensors_(std::make_shared(), pool), + uint32_tensors_(std::make_shared(), pool), + int32_tensors_(std::make_shared(), pool), + uint64_tensors_(std::make_shared(), pool), + int64_tensors_(std::make_shared(), pool), + float_tensors_(std::make_shared(), pool), + double_tensors_(std::make_shared(), pool), + list_offsets_({0}), tuple_offsets_({0}), dict_offsets_({0}) {} + +#define UPDATE(OFFSET, TAG) \ + if (TAG == -1) { \ + TAG = num_tags; \ + num_tags += 1; \ + } \ + RETURN_NOT_OK(offsets_.Append(OFFSET)); \ + RETURN_NOT_OK(types_.Append(TAG)); \ + RETURN_NOT_OK(nones_.AppendToBitmap(true)); + +Status SequenceBuilder::Append() { + RETURN_NOT_OK(offsets_.Append(0)); + RETURN_NOT_OK(types_.Append(0)); + return nones_.AppendToBitmap(false); +} + +Status SequenceBuilder::Append(bool data) { + UPDATE(bools_.length(), bool_tag); + return bools_.Append(data); +} + +Status SequenceBuilder::Append(int64_t data) { + UPDATE(ints_.length(), int_tag); + return ints_.Append(data); +} + +Status SequenceBuilder::Append(uint64_t data) { + UPDATE(ints_.length(), int_tag); + return ints_.Append(data); +} + +Status SequenceBuilder::Append(const char* data, int32_t length) { + UPDATE(strings_.length(), string_tag); + return strings_.Append(data, length); +} + +Status SequenceBuilder::Append(float data) { + UPDATE(floats_.length(), float_tag); + return floats_.Append(data); +} + +Status SequenceBuilder::Append(double data) { + UPDATE(doubles_.length(), double_tag); + return doubles_.Append(data); +} + +#define DEF_TENSOR_APPEND(NAME, TYPE, TAG) \ + Status SequenceBuilder::Append(const std::vector& dims, TYPE* data) { \ + UPDATE(NAME.length(), TAG); \ + return NAME.Append(dims, data); \ + } + +DEF_TENSOR_APPEND(uint8_tensors_, uint8_t, uint8_tensor_tag); +DEF_TENSOR_APPEND(int8_tensors_, int8_t, int8_tensor_tag); +DEF_TENSOR_APPEND(uint16_tensors_, uint16_t, uint16_tensor_tag); +DEF_TENSOR_APPEND(int16_tensors_, int16_t, int16_tensor_tag); +DEF_TENSOR_APPEND(uint32_tensors_, uint32_t, uint32_tensor_tag); +DEF_TENSOR_APPEND(int32_tensors_, int32_t, int32_tensor_tag); +DEF_TENSOR_APPEND(uint64_tensors_, uint64_t, uint64_tensor_tag); +DEF_TENSOR_APPEND(int64_tensors_, int64_t, int64_tensor_tag); +DEF_TENSOR_APPEND(float_tensors_, float, float_tensor_tag); +DEF_TENSOR_APPEND(double_tensors_, double, double_tensor_tag); + +Status SequenceBuilder::AppendList(int32_t size) { + UPDATE(list_offsets_.size() - 1, list_tag); + list_offsets_.push_back(list_offsets_.back() + size); + return Status::OK(); +} + +Status SequenceBuilder::AppendTuple(int32_t size) { + UPDATE(tuple_offsets_.size() - 1, tuple_tag); + tuple_offsets_.push_back(tuple_offsets_.back() + size); + return Status::OK(); +} + +Status SequenceBuilder::AppendDict(int32_t size) { + UPDATE(dict_offsets_.size() - 1, dict_tag); + dict_offsets_.push_back(dict_offsets_.back() + size); + return Status::OK(); +} + +#define ADD_ELEMENT(VARNAME, TAG) \ + if (TAG != -1) { \ + types[TAG] = VARNAME.type(); \ + children[TAG] = VARNAME.Finish(); \ + ARROW_CHECK_OK(nones_.AppendToBitmap(true)); \ + } + +#define ADD_SUBSEQUENCE(DATA, OFFSETS, BUILDER, TAG, NAME) \ + if (DATA) { \ + DCHECK(DATA->length() == OFFSETS.back()); \ + auto list_builder = std::make_shared(pool_, DATA); \ + auto field = std::make_shared(NAME, list_builder->type()); \ + auto type = std::make_shared(std::vector({field})); \ + auto lists = std::vector>({list_builder}); \ + StructBuilder builder(pool_, type, lists); \ + OFFSETS.pop_back(); \ + ARROW_CHECK_OK(list_builder->Append(OFFSETS.data(), OFFSETS.size())); \ + builder.Append(); \ + ADD_ELEMENT(builder, TAG); \ + } else { \ + DCHECK(OFFSETS.size() == 1); \ + } + +std::shared_ptr SequenceBuilder::Finish( + std::shared_ptr list_data, + std::shared_ptr tuple_data, + std::shared_ptr dict_data) { + + std::vector types(num_tags); + std::vector children(num_tags); + + ADD_ELEMENT(bools_, bool_tag); + ADD_ELEMENT(ints_, int_tag); + ADD_ELEMENT(strings_, string_tag); + ADD_ELEMENT(floats_, float_tag); + ADD_ELEMENT(doubles_, double_tag); + + ADD_ELEMENT(uint8_tensors_, uint8_tensor_tag); + + ADD_ELEMENT(int8_tensors_, int8_tensor_tag); + ADD_ELEMENT(uint16_tensors_, uint16_tensor_tag); + ADD_ELEMENT(int16_tensors_, int16_tensor_tag); + ADD_ELEMENT(uint32_tensors_, uint32_tensor_tag); + + ADD_ELEMENT(int32_tensors_, int32_tensor_tag); + ADD_ELEMENT(uint64_tensors_, uint64_tensor_tag); + ADD_ELEMENT(int64_tensors_, int64_tensor_tag); + + ADD_ELEMENT(float_tensors_, float_tensor_tag); + ADD_ELEMENT(double_tensors_, double_tensor_tag); + + ADD_SUBSEQUENCE(list_data, list_offsets_, list_builder, list_tag, "list"); + ADD_SUBSEQUENCE(tuple_data, tuple_offsets_, tuple_builder, tuple_tag, "tuple"); + ADD_SUBSEQUENCE(dict_data, dict_offsets_, dict_builder, dict_tag, "dict"); + + TypePtr type = TypePtr(new DenseUnionType(types)); + + return std::make_shared(type, types_.length(), + children, types_.data(), offsets_.data(), + nones_.null_count(), nones_.null_bitmap()); +} + +} diff --git a/cpp/src/numbuf/sequence.h b/cpp/src/numbuf/sequence.h new file mode 100644 index 000000000..f30ea00fe --- /dev/null +++ b/cpp/src/numbuf/sequence.h @@ -0,0 +1,138 @@ +#ifndef NUMBUF_LIST_H +#define NUMBUF_LIST_H + +#include +#include +#include "tensor.h" + +namespace numbuf { + +/*! A Sequence is a heterogeneous collections of elements. It can contain + scalar Python types, lists, tuples, dictionaries and tensors. +*/ +class SequenceBuilder { + public: + SequenceBuilder(arrow::MemoryPool* pool = nullptr); + + //! Appending a none to the sequence + arrow::Status Append(); + + //! Appending a boolean to the sequence + arrow::Status Append(bool data); + + //! Appending an int64_t to the sequence + arrow::Status Append(int64_t data); + + //! Appending an uint64_t to the sequence + arrow::Status Append(uint64_t data); + + //! Appending a string to the sequence + arrow::Status Append(const char* data, int32_t length); + + //! Appending a float to the sequence + arrow::Status Append(float data); + + //! Appending a double to the sequence + arrow::Status Append(double data); + + /*! Appending a tensor to the sequence + + \param dims + A vector of dimensions + + \param data + A pointer to the start of the data block. The length of the data block + will be the product of the dimensions + */ + arrow::Status Append(const std::vector& dims, uint8_t* data); + arrow::Status Append(const std::vector& dims, int8_t* data); + arrow::Status Append(const std::vector& dims, uint16_t* data); + arrow::Status Append(const std::vector& dims, int16_t* data); + arrow::Status Append(const std::vector& dims, uint32_t* data); + arrow::Status Append(const std::vector& dims, int32_t* data); + arrow::Status Append(const std::vector& dims, uint64_t* data); + arrow::Status Append(const std::vector& dims, int64_t* data); + arrow::Status Append(const std::vector& dims, float* data); + arrow::Status Append(const std::vector& dims, double* data); + + /*! Add a sublist to the sequenc. The data contained in the sublist will be + specified in the "Finish" method. + + To construct l = [[11, 22], 33, [44, 55]] you would for example run + list = ListBuilder(); + list.AppendList(2); + list.Append(33); + list.AppendList(2); + list.Finish([11, 22, 44, 55]); + list.Finish(); + + \param size + The size of the sublist + */ + arrow::Status AppendList(int32_t size); + + arrow::Status AppendTuple(int32_t size); + + arrow::Status AppendDict(int32_t size); + + //! Finish building the sequence and return the result + std::shared_ptr Finish( + std::shared_ptr list_data, + std::shared_ptr tuple_data, + std::shared_ptr dict_data); + + private: + arrow::MemoryPool* pool_; + + arrow::Int8Builder types_; + arrow::Int32Builder offsets_; + + arrow::NullArrayBuilder nones_; + arrow::BooleanBuilder bools_; + arrow::Int64Builder ints_; + arrow::StringBuilder strings_; + arrow::FloatBuilder floats_; + arrow::DoubleBuilder doubles_; + + UInt8TensorBuilder uint8_tensors_; + Int8TensorBuilder int8_tensors_; + UInt16TensorBuilder uint16_tensors_; + Int16TensorBuilder int16_tensors_; + UInt32TensorBuilder uint32_tensors_; + Int32TensorBuilder int32_tensors_; + UInt64TensorBuilder uint64_tensors_; + Int64TensorBuilder int64_tensors_; + FloatTensorBuilder float_tensors_; + DoubleTensorBuilder double_tensors_; + + std::vector list_offsets_; + std::vector tuple_offsets_; + std::vector dict_offsets_; + + int8_t bool_tag = -1; + int8_t int_tag = -1; + int8_t string_tag = -1; + int8_t float_tag = -1; + int8_t double_tag = -1; + + int8_t uint8_tensor_tag = -1; + int8_t int8_tensor_tag = -1; + int8_t uint16_tensor_tag = -1; + int8_t int16_tensor_tag = -1; + int8_t uint32_tensor_tag = -1; + int8_t int32_tensor_tag = -1; + int8_t uint64_tensor_tag = -1; + int8_t int64_tensor_tag = -1; + int8_t float_tensor_tag = -1; + int8_t double_tensor_tag = -1; + + int8_t list_tag = -1; + int8_t tuple_tag = -1; + int8_t dict_tag = -1; + + int8_t num_tags = 0; +}; + +} // namespace numbuf + +#endif // NUMBUF_LIST_H diff --git a/cpp/src/numbuf/tensor.cc b/cpp/src/numbuf/tensor.cc new file mode 100644 index 000000000..f0f7647de --- /dev/null +++ b/cpp/src/numbuf/tensor.cc @@ -0,0 +1,50 @@ +#include "tensor.h" + +using namespace arrow; + +namespace numbuf { + +template +TensorBuilder::TensorBuilder(const TypePtr& dtype, MemoryPool* pool) + : dtype_(dtype) { + dim_data_ = std::make_shared(pool); + dims_ = std::make_shared(pool, dim_data_); + value_data_ = std::make_shared>(pool, dtype); + values_ = std::make_shared(pool, value_data_); + auto dims_field = std::make_shared("dims", dims_->type()); + auto values_field = std::make_shared("data", values_->type()); + auto type = std::make_shared(std::vector({dims_field, values_field})); + tensors_ = std::make_shared(pool, type, std::vector>({dims_, values_})); +}; + +template +Status TensorBuilder::Append(const std::vector& dims, const elem_type* data) { + RETURN_NOT_OK(tensors_->Append()); + RETURN_NOT_OK(dims_->Append()); + RETURN_NOT_OK(values_->Append()); + int32_t size = 1; + for (auto dim : dims) { + size *= dim; + RETURN_NOT_OK(dim_data_->Append(dim)); + } + RETURN_NOT_OK(value_data_->Append(data, size)); + return Status::OK(); // tensors_->Append(); +} + +template +std::shared_ptr TensorBuilder::Finish() { + return tensors_->Finish(); +} + +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; +template class TensorBuilder; + +} diff --git a/cpp/src/numbuf/tensor.h b/cpp/src/numbuf/tensor.h new file mode 100644 index 000000000..351892758 --- /dev/null +++ b/cpp/src/numbuf/tensor.h @@ -0,0 +1,67 @@ +#ifndef NUMBUF_TENSOR_H +#define NUMBUF_TENSOR_H + +#include +#include +#include + +namespace numbuf { + +/*! This is a class for building a dataframe where each row corresponds to + a Tensor (= multidimensional array) of numerical data. There are two + columns, "dims" which contains an array of dimensions for each Tensor + and "data" which contains data buffer of the Tensor as a flattened array. +*/ +template +class TensorBuilder { +public: + typedef typename T::c_type elem_type; + + TensorBuilder(const arrow::TypePtr& dtype, arrow::MemoryPool* pool = nullptr); + + /*! Append a new tensor. + + \param dims + The dimensions of the Tensor + + \param data + Pointer to the beginning of the data buffer of the Tensor. The + total length of the buffer is sizeof(elem_type) * product of dims[i] over i + */ + arrow::Status Append(const std::vector& dims, const elem_type* data); + + //! Convert the tensors to an Arrow StructArray + std::shared_ptr Finish(); + + //! Number of tensors in the column + int32_t length() { + return tensors_->length(); + } + + const arrow::TypePtr& type() { + return tensors_->type(); + } + +private: + arrow::TypePtr dtype_; + std::shared_ptr dim_data_; + std::shared_ptr dims_; + std::shared_ptr> value_data_; + std::shared_ptr values_; + std::shared_ptr tensors_; +}; + +typedef TensorBuilder UInt8TensorBuilder; +typedef TensorBuilder Int8TensorBuilder; +typedef TensorBuilder UInt16TensorBuilder; +typedef TensorBuilder Int16TensorBuilder; +typedef TensorBuilder UInt32TensorBuilder; +typedef TensorBuilder Int32TensorBuilder; +typedef TensorBuilder UInt64TensorBuilder; +typedef TensorBuilder Int64TensorBuilder; +typedef TensorBuilder FloatTensorBuilder; +typedef TensorBuilder DoubleTensorBuilder; + +} + +#endif // NUMBUF_TENSOR_H diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc new file mode 100644 index 000000000..6ca6753ca --- /dev/null +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -0,0 +1,88 @@ +#include "numpy.h" + +#include + +using namespace arrow; + +namespace numbuf { + +#define ARROW_TYPE_TO_NUMPY_CASE(TYPE) \ + case Type::TYPE: \ + return NPY_##TYPE; + +#define DESERIALIZE_ARRAY_CASE(TYPE, ArrayType, type) \ + case Type::TYPE: { \ + auto values = std::dynamic_pointer_cast(content->values()); \ + DCHECK(values); \ + type* data = const_cast(values->raw_data()) \ + + content->offset(offset); \ + *out = PyArray_SimpleNewFromData(num_dims, dim.data(), NPY_##TYPE, \ + reinterpret_cast(data)); \ + } \ + return Status::OK(); + +Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject** out) { + DCHECK(array); + auto tensor = std::dynamic_pointer_cast(array); + DCHECK(tensor); + auto dims = std::dynamic_pointer_cast(tensor->field(0)); + auto content = std::dynamic_pointer_cast(tensor->field(1)); + npy_intp num_dims = dims->value_length(offset); + std::vector dim(num_dims); + for (int i = dims->offset(offset); i < dims->offset(offset+1); ++i) { + dim[i - dims->offset(offset)] = + std::dynamic_pointer_cast(dims->values())->Value(i); + } + switch (content->value_type()->type) { + DESERIALIZE_ARRAY_CASE(INT8, Int8Array, int8_t) + DESERIALIZE_ARRAY_CASE(INT16, Int16Array, int16_t) + DESERIALIZE_ARRAY_CASE(INT32, Int32Array, int32_t) + DESERIALIZE_ARRAY_CASE(INT64, Int64Array, int64_t) + DESERIALIZE_ARRAY_CASE(UINT8, UInt8Array, uint8_t) + DESERIALIZE_ARRAY_CASE(UINT16, UInt16Array, uint16_t) + DESERIALIZE_ARRAY_CASE(UINT32, UInt32Array, uint32_t) + DESERIALIZE_ARRAY_CASE(UINT64, UInt64Array, uint64_t) + DESERIALIZE_ARRAY_CASE(FLOAT, FloatArray, float) + DESERIALIZE_ARRAY_CASE(DOUBLE, DoubleArray, double) + default: + DCHECK(false) << "arrow type not recognized: " << content->value_type()->type; + } + return Status::OK(); +} + +Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder) { + size_t ndim = PyArray_NDIM(array); + int dtype = PyArray_TYPE(array); + std::vector dims(ndim); + for (int i = 0; i < ndim; ++i) { + dims[i] = PyArray_DIM(array, i); + } + auto data = PyArray_DATA(array); + switch (dtype) { + case NPY_UINT8: + return builder.Append(dims, reinterpret_cast(data)); + case NPY_INT8: + return builder.Append(dims, reinterpret_cast(data)); + case NPY_UINT16: + return builder.Append(dims, reinterpret_cast(data)); + case NPY_INT16: + return builder.Append(dims, reinterpret_cast(data)); + case NPY_UINT32: + return builder.Append(dims, reinterpret_cast(data)); + case NPY_INT32: + return builder.Append(dims, reinterpret_cast(data)); + case NPY_UINT64: + return builder.Append(dims, reinterpret_cast(data)); + case NPY_INT64: + return builder.Append(dims, reinterpret_cast(data)); + case NPY_FLOAT: + return builder.Append(dims, reinterpret_cast(data)); + case NPY_DOUBLE: + return builder.Append(dims, reinterpret_cast(data)); + default: + DCHECK(false) << "numpy data type not recognized: " << dtype; + } + return Status::OK(); +} + +} diff --git a/python/src/pynumbuf/adapters/numpy.h b/python/src/pynumbuf/adapters/numpy.h new file mode 100644 index 000000000..8b8b3859a --- /dev/null +++ b/python/src/pynumbuf/adapters/numpy.h @@ -0,0 +1,22 @@ +#ifndef PYNUMBUF_NUMPY_H +#define PYNUMBUF_NUMPY_H + +#include +#include + +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API +#include + +#include +#include + +namespace numbuf { + +arrow::Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder); +arrow::Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject** out); + +} + +#endif diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc new file mode 100644 index 000000000..751ebf91b --- /dev/null +++ b/python/src/pynumbuf/adapters/python.cc @@ -0,0 +1,193 @@ +#include "python.h" + +#include + +#include "scalars.h" + +using namespace arrow; + +namespace numbuf { + +PyObject* get_value(ArrayPtr arr, int32_t index, int32_t type) { + PyObject* result; + switch (arr->type()->type) { + case Type::BOOL: + return PyBool_FromLong(std::static_pointer_cast(arr)->Value(index)); + case Type::INT64: + return PyInt_FromLong(std::static_pointer_cast(arr)->Value(index)); + case Type::STRING: { + int32_t nchars; + const uint8_t* str = std::static_pointer_cast(arr)->GetValue(index, &nchars); + return PyString_FromStringAndSize(reinterpret_cast(str), nchars); + } + case Type::FLOAT: + return PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + case Type::DOUBLE: + return PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + case Type::STRUCT: { + auto s = std::static_pointer_cast(arr); + auto l = std::static_pointer_cast(s->field(0)); + if (s->type()->child(0)->name == "list") { + ARROW_CHECK_OK(DeserializeList(l->values(), l->value_offset(index), l->value_offset(index+1), &result)); + } else if (s->type()->child(0)->name == "tuple") { + ARROW_CHECK_OK(DeserializeTuple(l->values(), l->value_offset(index), l->value_offset(index+1), &result)); + } else if (s->type()->child(0)->name == "dict") { + ARROW_CHECK_OK(DeserializeDict(l->values(), l->value_offset(index), l->value_offset(index+1), &result)); + } else { + ARROW_CHECK_OK(DeserializeArray(arr, index, &result)); + } + return result; + } + default: + DCHECK(false) << "union tag not recognized " << type; + } + return NULL; +} + +Status append(PyObject* elem, SequenceBuilder& builder, + std::vector& sublists, + std::vector& subtuples, + std::vector& subdicts) { + // The bool case must precede the int case (PyInt_Check passes for bools) + if (PyBool_Check(elem)) { + RETURN_NOT_OK(builder.Append(elem == Py_True)); + } else if (PyFloat_Check(elem)) { + RETURN_NOT_OK(builder.Append(PyFloat_AS_DOUBLE(elem))); + } else if (PyLong_Check(elem)) { + int overflow = 0; + int64_t data = PyLong_AsLongLongAndOverflow(elem, &overflow); + RETURN_NOT_OK(builder.Append(data)); + if(overflow) { + return Status::NotImplemented("long overflow"); + } + } else if (PyInt_Check(elem)) { + RETURN_NOT_OK(builder.Append(PyInt_AS_LONG(elem))); + } else if (PyString_Check(elem)) { + RETURN_NOT_OK(builder.Append(PyString_AS_STRING(elem), PyString_GET_SIZE(elem))); + } else if (PyList_Check(elem)) { + builder.AppendList(PyList_Size(elem)); + sublists.push_back(elem); + } else if (PyDict_Check(elem)) { + builder.AppendDict(PyDict_Size(elem)); + subdicts.push_back(elem); + } else if (PyTuple_Check(elem)) { + builder.AppendTuple(PyTuple_Size(elem)); + subtuples.push_back(elem); + } else if (PyArray_IsScalar(elem, Generic)) { + RETURN_NOT_OK(AppendScalar(elem, builder)); + } else if (PyArray_Check(elem)) { + RETURN_NOT_OK(SerializeArray((PyArrayObject*) elem, builder)); + } else if (elem == Py_None) { + RETURN_NOT_OK(builder.Append()); + } else { + std::stringstream ss; + ss << "data type of " << PyString_AS_STRING(PyObject_Repr(elem)) + << " not recognized"; + return Status::NotImplemented(ss.str()); + } + return Status::OK(); +} + +Status SerializeSequences(std::vector sequences, std::shared_ptr* out) { + DCHECK(out); + SequenceBuilder builder(nullptr); + std::vector sublists, subtuples, subdicts; + for (const auto& sequence : sequences) { + PyObject* item; + PyObject* iterator = PyObject_GetIter(sequence); + while (item = PyIter_Next(iterator)) { + RETURN_NOT_OK(append(item, builder, sublists, subtuples, subdicts)); + Py_DECREF(item); + } + Py_DECREF(iterator); + } + std::shared_ptr list; + if (sublists.size() > 0) { + RETURN_NOT_OK(SerializeSequences(sublists, &list)); + } + std::shared_ptr tuple; + if (subtuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(subtuples, &tuple)); + } + std::shared_ptr dict; + if (subdicts.size() > 0) { + RETURN_NOT_OK(SerializeDict(subdicts, &dict)); + } + *out = builder.Finish(list, tuple, dict); + return Status::OK(); +} + +#define DESERIALIZE_SEQUENCE(CREATE, SET_ITEM) \ + auto data = std::dynamic_pointer_cast(array); \ + int32_t size = array->length(); \ + PyObject* result = CREATE(stop_idx - start_idx); \ + auto types = std::make_shared(size, data->types()); \ + auto offsets = std::make_shared(size, data->offset_buf()); \ + for (size_t i = start_idx; i < stop_idx; ++i) { \ + if (data->IsNull(i)) { \ + Py_INCREF(Py_None); \ + SET_ITEM(result, i-start_idx, Py_None); \ + } else { \ + int32_t offset = offsets->Value(i); \ + int8_t type = types->Value(i); \ + ArrayPtr arr = data->child(type); \ + SET_ITEM(result, i-start_idx, get_value(arr, offset, type)); \ + } \ + } \ + *out = result; \ + return Status::OK(); + +Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out) { + DESERIALIZE_SEQUENCE(PyList_New, PyList_SetItem) +} + +Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out) { + DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SetItem) +} + +Status SerializeDict(std::vector dicts, std::shared_ptr* out) { + DictBuilder result; + std::vector sublists, subtuples, subdicts, dummy; + for (const auto& dict : dicts) { + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(dict, &pos, &key, &value)) { + RETURN_NOT_OK(append(key, result.keys(), dummy, dummy, dummy)); + DCHECK(dummy.size() == 0); + RETURN_NOT_OK(append(value, result.vals(), sublists, subtuples, subdicts)); + } + } + std::shared_ptr val_list; + if (sublists.size() > 0) { + RETURN_NOT_OK(SerializeSequences(sublists, &val_list)); + } + std::shared_ptr val_tuples; + if (subtuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(subtuples, &val_tuples)); + } + std::shared_ptr val_dict; + if (subdicts.size() > 0) { + RETURN_NOT_OK(SerializeDict(subdicts, &val_dict)); + } + *out = result.Finish(val_list, val_tuples, val_dict); + return Status::OK(); +} + +Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out) { + auto data = std::dynamic_pointer_cast(array); + // TODO(pcm): error handling, get rid of the temporary copy of the list + PyObject *keys, *vals; + PyObject* result = PyDict_New(); + ARROW_RETURN_NOT_OK(DeserializeList(data->field(0), start_idx, stop_idx, &keys)); + ARROW_RETURN_NOT_OK(DeserializeList(data->field(1), start_idx, stop_idx, &vals)); + for (size_t i = start_idx; i < stop_idx; ++i) { + PyDict_SetItem(result, PyList_GetItem(keys, i - start_idx), PyList_GetItem(vals, i - start_idx)); + } + Py_XDECREF(keys); // PyList_GetItem(keys, ...) incremented the reference count + Py_XDECREF(vals); // PyList_GetItem(vals, ...) incremented the reference count + *out = result; + return Status::OK(); +} + + +} diff --git a/python/src/pynumbuf/adapters/python.h b/python/src/pynumbuf/adapters/python.h new file mode 100644 index 000000000..1fa07f920 --- /dev/null +++ b/python/src/pynumbuf/adapters/python.h @@ -0,0 +1,22 @@ +#ifndef PYNUMBUF_PYTHON_H +#define PYNUMBUF_PYTHON_H + +#include + +#include +#include +#include + +#include "numpy.h" + +namespace numbuf { + +arrow::Status SerializeSequences(std::vector sequences, std::shared_ptr* out); +arrow::Status SerializeDict(std::vector dicts, std::shared_ptr* out); +arrow::Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out); +arrow::Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out); +arrow::Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out); + +} + +#endif diff --git a/python/src/pynumbuf/adapters/scalars.h b/python/src/pynumbuf/adapters/scalars.h new file mode 100644 index 000000000..928289252 --- /dev/null +++ b/python/src/pynumbuf/adapters/scalars.h @@ -0,0 +1,54 @@ +#ifndef PYNUMBUF_SCALARS_H +#define PYNUMBUF_SCALARS_H + +#include + +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API +#include +#include + +#include + +namespace numbuf { + +arrow::Status AppendScalar(PyObject* obj, SequenceBuilder& builder) { + if (PyArray_IsScalar(obj, Bool)) { + return builder.Append(((PyBoolScalarObject *)obj)->obval != 0); + } else if (PyArray_IsScalar(obj, Float)) { + return builder.Append(((PyFloatScalarObject *)obj)->obval); + } else if (PyArray_IsScalar(obj, Double)) { + return builder.Append(((PyDoubleScalarObject *)obj)->obval); + } + int64_t value = 0; + if (PyArray_IsScalar(obj, Byte)) { + value = ((PyByteScalarObject *)obj)->obval; + } else if (PyArray_IsScalar(obj, UByte)) { + value = ((PyUByteScalarObject *)obj)->obval; + } else if (PyArray_IsScalar(obj, Short)) { + value = ((PyShortScalarObject *)obj)->obval; + } else if (PyArray_IsScalar(obj, UShort)) { + value = ((PyUShortScalarObject *)obj)->obval; + } else if (PyArray_IsScalar(obj, Int)) { + value = ((PyIntScalarObject *)obj)->obval; + } else if (PyArray_IsScalar(obj, UInt)) { + value = ((PyUIntScalarObject *)obj)->obval; + } else if (PyArray_IsScalar(obj, Long)) { + value = ((PyLongScalarObject *)obj)->obval; + } else if (PyArray_IsScalar(obj, ULong)) { + value = ((PyULongScalarObject *)obj)->obval; + } else if (PyArray_IsScalar(obj, LongLong)) { + value = ((PyLongLongScalarObject *)obj)->obval; + } else if (PyArray_IsScalar(obj, ULongLong)) { + value = ((PyULongLongScalarObject *)obj)->obval; + } else { + DCHECK(false) << "scalar type not recognized"; + } + return builder.Append(value); +} + +} // namespace + +#endif // PYNUMBUF_SCALARS_H diff --git a/python/src/pynumbuf/memory.h b/python/src/pynumbuf/memory.h new file mode 100644 index 000000000..ebaa22c51 --- /dev/null +++ b/python/src/pynumbuf/memory.h @@ -0,0 +1,47 @@ +#ifndef PYNUMBUF_MEMORY_H +#define PYNUMBUF_MEMORY_H + +#include + +namespace numbuf { + +class BufferSource : public arrow::ipc::MemorySource { + public: + virtual ~BufferSource() {} + + explicit BufferSource(uint8_t* data, int64_t nbytes) + : data_(data), size_(nbytes) {} + + arrow::Status ReadAt(int64_t position, int64_t nbytes, + std::shared_ptr* out) override { + DCHECK(out); + DCHECK(position + nbytes <= size_) << "position: " << position << " nbytes: " << nbytes << "size: " << size_; + *out = std::make_shared(data_ + position, nbytes); + return arrow::Status::OK(); + } + + arrow::Status Close() override { + return arrow::Status::OK(); + } + + arrow::Status Write(int64_t position, const uint8_t* data, + int64_t nbytes) override { + DCHECK(position >= 0 && position < size_); + DCHECK(position + nbytes <= size_) << "position: " << position << " nbytes: " << nbytes << "size: " << size_; + uint8_t* dst = data_ + position; + memcpy(dst, data, nbytes); + return arrow::Status::OK(); + } + + int64_t Size() const override { + return size_; + } + +private: + uint8_t* data_; + int64_t size_; +}; + +} // namespace + +#endif // PYNUMBUF_MEMORY_H diff --git a/python/src/pynumbuf/numbuf.cc b/python/src/pynumbuf/numbuf.cc new file mode 100644 index 000000000..6b4c524dd --- /dev/null +++ b/python/src/pynumbuf/numbuf.cc @@ -0,0 +1,205 @@ +#include +#include +#include +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API +#include + +#include + +#include + +#include "adapters/python.h" +#include "memory.h" + +using namespace arrow; +using namespace numbuf; + +extern "C" { + +// Error handling + +static PyObject *NumbufError; + +int PyObjectToArrow(PyObject* object, std::shared_ptr **result) { + if (PyCapsule_IsValid(object, "arrow")) { + *result = reinterpret_cast*>(PyCapsule_GetPointer(object, "arrow")); + return 1; + } else { + PyErr_SetString(PyExc_TypeError, "must be an 'arrow' capsule"); + return 0; + } +} + +static void ArrowCapsule_Destructor(PyObject* capsule) { + delete reinterpret_cast*>(PyCapsule_GetPointer(capsule, "arrow")); +} + +std::shared_ptr make_row_batch(std::shared_ptr data) { + auto field = std::make_shared("list", data->type()); + std::shared_ptr schema(new Schema({field})); + return std::shared_ptr(new RowBatch(schema, data->length(), {data})); +} + +/*! Serializes a Python list into an Arrow array. + + \param args + The argument must be a Python list + + \returns + A Python "arrow" capsule containing the arrow::Array +*/ +PyObject* serialize_list(PyObject* self, PyObject* args) { + PyObject* value; + if (!PyArg_ParseTuple(args, "O", &value)) { + return NULL; + } + std::shared_ptr* result = new std::shared_ptr(); + if (PyList_Check(value)) { + Status s = SerializeSequences(std::vector({value}), result); + if (!s.ok()) { + PyErr_SetString(NumbufError, s.ToString().c_str()); + return NULL; + } + return PyCapsule_New(reinterpret_cast(result), "arrow", &ArrowCapsule_Destructor); + } + return NULL; +} + +/*! Number of bytes the serialized version of the object will take. + + \param args + A Python "arrow" capsule containing the arrow::Array + + \returns + Size of the object in memory once it is serialized +*/ +PyObject* get_serialized_size(PyObject* self, PyObject* args) { + std::shared_ptr* data; + if (!PyArg_ParseTuple(args, "O&", &PyObjectToArrow, &data)) { + return NULL; + } + auto batch = make_row_batch(*data); + int64_t size = 0; + ARROW_CHECK_OK(arrow::ipc::GetRowBatchSize(batch.get(), &size)); + return PyInt_FromLong(size); +} + +/*! Serialize an arrow::Array into a buffer. + + \param args + A Python "arrow" capsule containing the arrow::Array and + a python memoryview object the data will be written to + + \return + The arrow metadata offset for the arrow metadata +*/ +PyObject* write_to_buffer(PyObject* self, PyObject* args) { + std::shared_ptr* data; + PyObject* memoryview; + if (!PyArg_ParseTuple(args, "O&O", &PyObjectToArrow, &data, &memoryview)) { + return NULL; + } + if (!PyMemoryView_Check(memoryview)) { + return NULL; + } + auto batch = make_row_batch(*data); + Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview); + auto target = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); + int64_t metadata_offset; + ARROW_CHECK_OK(ipc::WriteRowBatch(target.get(), batch.get(), 0, &metadata_offset)); + return PyInt_FromLong(metadata_offset); +} + +/*! Serialize schema metadata associated to and arrow::Array + + \param args + A Python "arrow" capsule containing the arrow::Array + + \return + A bytearray object containing the schema metadata +*/ +PyObject* get_schema_metadata(PyObject* self, PyObject* args) { + std::shared_ptr* data; + if (!PyArg_ParseTuple(args, "O&", &PyObjectToArrow, &data)) { + return NULL; + } + auto batch = make_row_batch(*data); + std::shared_ptr buffer; + ARROW_CHECK_OK(ipc::WriteSchema(batch->schema().get(), &buffer)); + auto ptr = reinterpret_cast(buffer->data()); + return PyByteArray_FromStringAndSize(ptr, buffer->size()); +} + +/*! Read serialized data from buffer and produce an arrow capsule + + \param args + A Python memoryview from which data will be loaded, + a Python bytearray containing the metadata and the metadata_offset + + \return + A Python "arrow" capsule containing the arrow data +*/ +PyObject* read_from_buffer(PyObject* self, PyObject* args) { + PyObject* memoryview; + PyObject* metadata; + int64_t metadata_offset; + if (!PyArg_ParseTuple(args, "OOl", &memoryview, &metadata, &metadata_offset)) { + return NULL; + } + + auto ptr = reinterpret_cast(PyByteArray_AsString(metadata)); + auto schema_buffer = std::make_shared(ptr, PyByteArray_Size(metadata)); + std::shared_ptr message; + ARROW_CHECK_OK(ipc::Message::Open(schema_buffer, &message)); + DCHECK_EQ(ipc::Message::SCHEMA, message->type()); + std::shared_ptr schema_msg = message->GetSchema(); + std::shared_ptr schema; + ARROW_CHECK_OK(schema_msg->GetSchema(&schema)); + + Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview); + auto source = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); + std::shared_ptr reader; + ARROW_CHECK_OK(arrow::ipc::RowBatchReader::Open(source.get(), metadata_offset, &reader)); + std::shared_ptr data; + ARROW_CHECK_OK(reader->GetRowBatch(schema, &data)); + + std::shared_ptr* result = new std::shared_ptr(); + *result = data->column(0); + return PyCapsule_New(reinterpret_cast(result), "arrow", &ArrowCapsule_Destructor); +} + +/*! +*/ +PyObject* deserialize_list(PyObject* self, PyObject* args) { + std::shared_ptr* data; + if (!PyArg_ParseTuple(args, "O&", &PyObjectToArrow, &data)) { + return NULL; + } + PyObject* result; + ARROW_CHECK_OK(DeserializeList(*data, 0, (*data)->length(), &result)); + return result; +} + +static PyMethodDef NumbufMethods[] = { + { "serialize_list", serialize_list, METH_VARARGS, "serialize a Python list" }, + { "deserialize_list", deserialize_list, METH_VARARGS, "deserialize a Python list" }, + { "get_serialized_size", get_serialized_size, METH_VARARGS, "get the number of bytes the object will occupy once serialized" }, + { "write_to_buffer", write_to_buffer, METH_VARARGS, "write serialized data to buffer"}, + { "read_from_buffer", read_from_buffer, METH_VARARGS, "read serialized data from buffer"}, + { "get_schema_metadata", get_schema_metadata, METH_VARARGS, "return the schema of an arrow object"}, + { NULL, NULL, 0, NULL } +}; + +PyMODINIT_FUNC initlibnumbuf(void) { + PyObject* m; + m = Py_InitModule3("libnumbuf", NumbufMethods, "Python C Extension for Numbuf"); + char numbuf_error[] = "numbuf.error"; + NumbufError = PyErr_NewException(numbuf_error, NULL, NULL); + Py_INCREF(NumbufError); + PyModule_AddObject(m, "numbuf_error", NumbufError); + import_array(); +} + +} diff --git a/python/test/runtest.py b/python/test/runtest.py new file mode 100644 index 000000000..f3697d51f --- /dev/null +++ b/python/test/runtest.py @@ -0,0 +1,63 @@ +import unittest +import libnumbuf +import numpy as np +from numpy.testing import assert_equal + +TEST_OBJECTS = [[1, "hello", 3.0], 42, 43L, "hello world", 42.0, 1L << 62, + (1.0, "hi"), None, (None, None), ("hello", None), + True, False, (True, False), + {True: "hello", False: "world"}, + {"hello" : "world", 1: 42, 1.0: 45}, {}, + np.int8(3), np.int32(4), np.int64(5), + np.uint8(3), np.uint32(4), np.uint64(5), + np.float32(1.0), np.float64(1.0)] + +class SerializationTests(unittest.TestCase): + + def roundTripTest(self, data): + serialized = libnumbuf.serialize_list(data) + result = libnumbuf.deserialize_list(serialized) + assert_equal(data, result) + + def testSimple(self): + self.roundTripTest([1, 2, 3]) + self.roundTripTest([1.0, 2.0, 3.0]) + self.roundTripTest(['hello', 'world']) + self.roundTripTest([1, 'hello', 1.0]) + self.roundTripTest([{'hello': 1.0, 'world': 42}]) + self.roundTripTest([True, False]) + + def testNone(self): + self.roundTripTest([1, 2, None, 3]) + + def testNested(self): + self.roundTripTest([{"hello": {"world": (1, 2, 3)}}]) + self.roundTripTest([((1,), (1, 2, 3, (4, 5, 6), "string"))]) + self.roundTripTest([{"hello": [1, 2, 3]}]) + self.roundTripTest([{"hello": [1, [2, 3]]}]) + self.roundTripTest([{"hello": (None, 2, [3, 4])}]) + self.roundTripTest([{"hello": (None, 2, [3, 4], np.ndarray([1.0, 2.0, 3.0]))}]) + + def numpyTest(self, t): + a = np.random.randint(0, 10, size=(100, 100)).astype(t) + self.roundTripTest([a]) + + def testArrays(self): + for t in ["int8", "uint8", "int16", "uint16", "int32", "uint32", "float32", "float64"]: + self.numpyTest(t) + + def testRay(self): + for obj in TEST_OBJECTS: + self.roundTripTest([obj]) + + def testBuffer(self): + for (i, obj) in enumerate(TEST_OBJECTS): + x = libnumbuf.serialize_list([1, 2, 3]) + schema = libnumbuf.get_schema_metadata(x) + size = libnumbuf.get_serialized_size(x) + 4096 # INITIAL_METADATA_SIZE in arrow + buff = np.zeros(size, dtype="uint8") + metadata_offset = libnumbuf.write_to_buffer(x, memoryview(buff)) + array = libnumbuf.read_from_buffer(memoryview(buff), schema, metadata_offset) + +if __name__ == "__main__": + unittest.main() From c9730599440eaee26e1b65868f6a6b28c8fde98c Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 21 Jul 2016 15:40:42 -0700 Subject: [PATCH 03/32] switch to release mode --- CMakeLists.txt | 2 +- python/src/pynumbuf/adapters/numpy.cc | 38 +++++++--- python/src/pynumbuf/adapters/python.cc | 7 +- python/src/pynumbuf/numbuf.cc | 99 ++++++++++---------------- python/test/runtest.py | 11 +-- 5 files changed, 77 insertions(+), 80 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bda3778bb..efcc8b70d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ endif() set(ARROW_DIR "${CMAKE_SOURCE_DIR}/../arrow/" CACHE STRING "Path of the arrow source directory") -set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/../arrow/cpp/build/debug/libarrow.so" CACHE STRING +set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/../arrow/cpp/build/release/libarrow.so" CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)") include_directories("${ARROW_DIR}/cpp/src/") diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc index 6ca6753ca..67537fd25 100644 --- a/python/src/pynumbuf/adapters/numpy.cc +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -57,31 +57,47 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder) { for (int i = 0; i < ndim; ++i) { dims[i] = PyArray_DIM(array, i); } - auto data = PyArray_DATA(array); + // TODO(pcm): Once we don't use builders any more below and directly share + // the memory buffer, we need to be more careful about this and not + // decrease the reference count of "contiguous" before the serialization + // is finished + auto contiguous = PyArray_GETCONTIGUOUS(array); + auto data = PyArray_DATA(contiguous); switch (dtype) { case NPY_UINT8: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; case NPY_INT8: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; case NPY_UINT16: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; case NPY_INT16: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; case NPY_UINT32: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; case NPY_INT32: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; case NPY_UINT64: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; case NPY_INT64: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; case NPY_FLOAT: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; case NPY_DOUBLE: - return builder.Append(dims, reinterpret_cast(data)); + RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + break; default: DCHECK(false) << "numpy data type not recognized: " << dtype; } + Py_XDECREF(contiguous); return Status::OK(); } diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 751ebf91b..580c02f1d 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -96,8 +96,13 @@ Status SerializeSequences(std::vector sequences, std::shared_ptr **result) { +int PyObjectToArrow(PyObject* object, std::shared_ptr **result) { if (PyCapsule_IsValid(object, "arrow")) { - *result = reinterpret_cast*>(PyCapsule_GetPointer(object, "arrow")); + *result = reinterpret_cast*>(PyCapsule_GetPointer(object, "arrow")); return 1; } else { PyErr_SetString(PyExc_TypeError, "must be an 'arrow' capsule"); @@ -33,7 +31,7 @@ int PyObjectToArrow(PyObject* object, std::shared_ptr **result) { } static void ArrowCapsule_Destructor(PyObject* capsule) { - delete reinterpret_cast*>(PyCapsule_GetPointer(capsule, "arrow")); + delete reinterpret_cast*>(PyCapsule_GetPointer(capsule, "arrow")); } std::shared_ptr make_row_batch(std::shared_ptr data) { @@ -48,44 +46,45 @@ std::shared_ptr make_row_batch(std::shared_ptr data) { The argument must be a Python list \returns - A Python "arrow" capsule containing the arrow::Array + A bytearray object containing the schema metadata + + The size in bytes the serialized object will occupy in memory + + A Python "arrow" capsule containing the RowBatch */ PyObject* serialize_list(PyObject* self, PyObject* args) { PyObject* value; if (!PyArg_ParseTuple(args, "O", &value)) { return NULL; } - std::shared_ptr* result = new std::shared_ptr(); + std::shared_ptr array; if (PyList_Check(value)) { - Status s = SerializeSequences(std::vector({value}), result); + Status s = SerializeSequences(std::vector({value}), &array); if (!s.ok()) { PyErr_SetString(NumbufError, s.ToString().c_str()); return NULL; } - return PyCapsule_New(reinterpret_cast(result), "arrow", &ArrowCapsule_Destructor); + + auto batch = new std::shared_ptr(); + *batch = make_row_batch(array); + + int64_t size = 0; + ARROW_CHECK_OK(arrow::ipc::GetRowBatchSize(batch->get(), &size)); + + std::shared_ptr buffer; + ARROW_CHECK_OK(ipc::WriteSchema((*batch)->schema().get(), &buffer)); + auto ptr = reinterpret_cast(buffer->data()); + + PyObject* r = PyTuple_New(3); + PyTuple_SetItem(r, 0, PyByteArray_FromStringAndSize(ptr, buffer->size())); + PyTuple_SetItem(r, 1, PyInt_FromLong(size)); + PyTuple_SetItem(r, 2, PyCapsule_New(reinterpret_cast(batch), + "arrow", &ArrowCapsule_Destructor)); + return r; } return NULL; } -/*! Number of bytes the serialized version of the object will take. - - \param args - A Python "arrow" capsule containing the arrow::Array - - \returns - Size of the object in memory once it is serialized -*/ -PyObject* get_serialized_size(PyObject* self, PyObject* args) { - std::shared_ptr* data; - if (!PyArg_ParseTuple(args, "O&", &PyObjectToArrow, &data)) { - return NULL; - } - auto batch = make_row_batch(*data); - int64_t size = 0; - ARROW_CHECK_OK(arrow::ipc::GetRowBatchSize(batch.get(), &size)); - return PyInt_FromLong(size); -} - /*! Serialize an arrow::Array into a buffer. \param args @@ -96,42 +95,21 @@ PyObject* get_serialized_size(PyObject* self, PyObject* args) { The arrow metadata offset for the arrow metadata */ PyObject* write_to_buffer(PyObject* self, PyObject* args) { - std::shared_ptr* data; + std::shared_ptr* batch; PyObject* memoryview; - if (!PyArg_ParseTuple(args, "O&O", &PyObjectToArrow, &data, &memoryview)) { + if (!PyArg_ParseTuple(args, "O&O", &PyObjectToArrow, &batch, &memoryview)) { return NULL; } if (!PyMemoryView_Check(memoryview)) { return NULL; } - auto batch = make_row_batch(*data); Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview); auto target = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); int64_t metadata_offset; - ARROW_CHECK_OK(ipc::WriteRowBatch(target.get(), batch.get(), 0, &metadata_offset)); + ARROW_CHECK_OK(ipc::WriteRowBatch(target.get(), batch->get(), 0, &metadata_offset)); return PyInt_FromLong(metadata_offset); } -/*! Serialize schema metadata associated to and arrow::Array - - \param args - A Python "arrow" capsule containing the arrow::Array - - \return - A bytearray object containing the schema metadata -*/ -PyObject* get_schema_metadata(PyObject* self, PyObject* args) { - std::shared_ptr* data; - if (!PyArg_ParseTuple(args, "O&", &PyObjectToArrow, &data)) { - return NULL; - } - auto batch = make_row_batch(*data); - std::shared_ptr buffer; - ARROW_CHECK_OK(ipc::WriteSchema(batch->schema().get(), &buffer)); - auto ptr = reinterpret_cast(buffer->data()); - return PyByteArray_FromStringAndSize(ptr, buffer->size()); -} - /*! Read serialized data from buffer and produce an arrow capsule \param args @@ -139,7 +117,7 @@ PyObject* get_schema_metadata(PyObject* self, PyObject* args) { a Python bytearray containing the metadata and the metadata_offset \return - A Python "arrow" capsule containing the arrow data + A Python "arrow" capsule containing the arrow RowBatch */ PyObject* read_from_buffer(PyObject* self, PyObject* args) { PyObject* memoryview; @@ -162,33 +140,30 @@ PyObject* read_from_buffer(PyObject* self, PyObject* args) { auto source = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); std::shared_ptr reader; ARROW_CHECK_OK(arrow::ipc::RowBatchReader::Open(source.get(), metadata_offset, &reader)); - std::shared_ptr data; - ARROW_CHECK_OK(reader->GetRowBatch(schema, &data)); + auto batch = new std::shared_ptr(); + ARROW_CHECK_OK(reader->GetRowBatch(schema, batch)); - std::shared_ptr* result = new std::shared_ptr(); - *result = data->column(0); - return PyCapsule_New(reinterpret_cast(result), "arrow", &ArrowCapsule_Destructor); + return PyCapsule_New(reinterpret_cast(batch), + "arrow", &ArrowCapsule_Destructor); } /*! */ PyObject* deserialize_list(PyObject* self, PyObject* args) { - std::shared_ptr* data; + std::shared_ptr* data; if (!PyArg_ParseTuple(args, "O&", &PyObjectToArrow, &data)) { return NULL; } PyObject* result; - ARROW_CHECK_OK(DeserializeList(*data, 0, (*data)->length(), &result)); + ARROW_CHECK_OK(DeserializeList((*data)->column(0), 0, (*data)->num_rows(), &result)); return result; } static PyMethodDef NumbufMethods[] = { { "serialize_list", serialize_list, METH_VARARGS, "serialize a Python list" }, { "deserialize_list", deserialize_list, METH_VARARGS, "deserialize a Python list" }, - { "get_serialized_size", get_serialized_size, METH_VARARGS, "get the number of bytes the object will occupy once serialized" }, { "write_to_buffer", write_to_buffer, METH_VARARGS, "write serialized data to buffer"}, { "read_from_buffer", read_from_buffer, METH_VARARGS, "read serialized data from buffer"}, - { "get_schema_metadata", get_schema_metadata, METH_VARARGS, "return the schema of an arrow object"}, { NULL, NULL, 0, NULL } }; diff --git a/python/test/runtest.py b/python/test/runtest.py index f3697d51f..9cfa62c92 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -15,7 +15,7 @@ TEST_OBJECTS = [[1, "hello", 3.0], 42, 43L, "hello world", 42.0, 1L << 62, class SerializationTests(unittest.TestCase): def roundTripTest(self, data): - serialized = libnumbuf.serialize_list(data) + schema, size, serialized = libnumbuf.serialize_list(data) result = libnumbuf.deserialize_list(serialized) assert_equal(data, result) @@ -52,12 +52,13 @@ class SerializationTests(unittest.TestCase): def testBuffer(self): for (i, obj) in enumerate(TEST_OBJECTS): - x = libnumbuf.serialize_list([1, 2, 3]) - schema = libnumbuf.get_schema_metadata(x) - size = libnumbuf.get_serialized_size(x) + 4096 # INITIAL_METADATA_SIZE in arrow + schema, size, batch = libnumbuf.serialize_list([obj]) + size = size + 4096 # INITIAL_METADATA_SIZE in arrow buff = np.zeros(size, dtype="uint8") - metadata_offset = libnumbuf.write_to_buffer(x, memoryview(buff)) + metadata_offset = libnumbuf.write_to_buffer(batch, memoryview(buff)) array = libnumbuf.read_from_buffer(memoryview(buff), schema, metadata_offset) + result = libnumbuf.deserialize_list(array) + assert_equal(result[0], obj) if __name__ == "__main__": unittest.main() From 0a26135821914a61beecd62365e193a1ad0d7e09 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sat, 23 Jul 2016 18:34:22 -0700 Subject: [PATCH 04/32] fix mac build --- CMakeLists.txt | 6 +++++- python/src/pynumbuf/adapters/python.cc | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index efcc8b70d..3194ac097 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,10 @@ find_package(PythonInterp REQUIRED) find_package(PythonLibs REQUIRED) find_package(NumPy REQUIRED) +if(APPLE) + SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif(APPLE) + include_directories("${PYTHON_INCLUDE_DIRS}") include_directories("${NUMPY_INCLUDE_DIR}") @@ -20,7 +24,7 @@ endif() set(ARROW_DIR "${CMAKE_SOURCE_DIR}/../arrow/" CACHE STRING "Path of the arrow source directory") -set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/../arrow/cpp/build/release/libarrow.so" CACHE STRING +set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/../arrow/cpp/build/release/libarrow.a" CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)") include_directories("${ARROW_DIR}/cpp/src/") diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 580c02f1d..7c747c4d9 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -61,7 +61,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, return Status::NotImplemented("long overflow"); } } else if (PyInt_Check(elem)) { - RETURN_NOT_OK(builder.Append(PyInt_AS_LONG(elem))); + RETURN_NOT_OK(builder.Append(static_cast(PyInt_AS_LONG(elem)))); } else if (PyString_Check(elem)) { RETURN_NOT_OK(builder.Append(PyString_AS_STRING(elem), PyString_GET_SIZE(elem))); } else if (PyList_Check(elem)) { @@ -95,7 +95,7 @@ Status SerializeSequences(std::vector sequences, std::shared_ptr Date: Sun, 24 Jul 2016 21:24:26 -0700 Subject: [PATCH 05/32] fix tests on macOS --- CMakeLists.txt | 39 +++++++++++++++++++++++++++++++++-- cmake/Modules/FindNumPy.cmake | 6 +++--- python/src/pynumbuf/numbuf.cc | 20 +++++++++--------- python/test/runtest.py | 2 +- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3194ac097..5019a9e62 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,37 @@ project(numbuf) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) -find_package(PythonInterp REQUIRED) -find_package(PythonLibs REQUIRED) +if(NOT APPLE) + find_package(PythonInterp REQUIRED) + find_package(PythonLibs REQUIRED) + set(CUSTOM_PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}) +else() + find_program(CUSTOM_PYTHON_EXECUTABLE python) + message("-- Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}") + execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c + "import sys; print 'python' + sys.version[0:3]" + OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c + "import sys; print sys.exec_prefix" + OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) + FIND_LIBRARY(PYTHON_LIBRARIES + NAMES ${PYTHON_LIBRARY_NAME} + HINTS "${PYTHON_PREFIX}" + PATH_SUFFIXES "lib" "libs" + NO_DEFAULT_PATH) + execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c + "from distutils.sysconfig import *; print get_python_inc()" + OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE) + if(PYTHON_LIBRARIES AND PYTHON_INCLUDE_DIRS) + SET(PYTHONLIBS_FOUND TRUE) + message("-- Found PythonLibs: " ${PYTHON_LIBRARIES}) + message("-- -- Used custom search path") + else() + find_package(PythonLibs REQUIRED) + message("-- -- Used find_package(PythonLibs)") + endif() +endif() + find_package(NumPy REQUIRED) if(APPLE) @@ -41,4 +70,10 @@ add_library(numbuf SHARED python/src/pynumbuf/adapters/numpy.cc python/src/pynumbuf/adapters/python.cc) +if(APPLE) + add_custom_command(TARGET numbuf + POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libnumbuf.so) +endif(APPLE) + target_link_libraries(numbuf ${ARROW_STATIC_LIB} ${PYTHON_LIBRARIES}) diff --git a/cmake/Modules/FindNumPy.cmake b/cmake/Modules/FindNumPy.cmake index 6e1f3c415..0b9fb3e5c 100644 --- a/cmake/Modules/FindNumPy.cmake +++ b/cmake/Modules/FindNumPy.cmake @@ -14,8 +14,8 @@ unset(NUMPY_VERSION) unset(NUMPY_INCLUDE_DIR) -if(PYTHONINTERP_FOUND) - execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" +if(NOT "${CUSTOM_PYTHON_EXECUTABLE}" STREQUAL "CUSTOM_PYTHON_EXECUTABLE-NOTFOUND") + execute_process(COMMAND "${CUSTOM_PYTHON_EXECUTABLE}" "-c" "import numpy as n; print(n.__version__); print(n.get_include());" RESULT_VARIABLE __result OUTPUT_VARIABLE __output @@ -42,7 +42,7 @@ if(PYTHONINTERP_FOUND) endif() endif() else() - message(STATUS "To find NumPy Python interpretator is required to be found.") + message(STATUS "To find NumPy Python executable is required to be found.") endif() include(FindPackageHandleStandardArgs) diff --git a/python/src/pynumbuf/numbuf.cc b/python/src/pynumbuf/numbuf.cc index d13d92371..d95feaf84 100644 --- a/python/src/pynumbuf/numbuf.cc +++ b/python/src/pynumbuf/numbuf.cc @@ -16,6 +16,12 @@ using namespace arrow; using namespace numbuf; +std::shared_ptr make_row_batch(std::shared_ptr data) { + auto field = std::make_shared("list", data->type()); + std::shared_ptr schema(new Schema({field})); + return std::shared_ptr(new RowBatch(schema, data->length(), {data})); +} + extern "C" { static PyObject *NumbufError; @@ -34,12 +40,6 @@ static void ArrowCapsule_Destructor(PyObject* capsule) { delete reinterpret_cast*>(PyCapsule_GetPointer(capsule, "arrow")); } -std::shared_ptr make_row_batch(std::shared_ptr data) { - auto field = std::make_shared("list", data->type()); - std::shared_ptr schema(new Schema({field})); - return std::shared_ptr(new RowBatch(schema, data->length(), {data})); -} - /*! Serializes a Python list into an Arrow array. \param args @@ -52,7 +52,7 @@ std::shared_ptr make_row_batch(std::shared_ptr data) { A Python "arrow" capsule containing the RowBatch */ -PyObject* serialize_list(PyObject* self, PyObject* args) { +static PyObject* serialize_list(PyObject* self, PyObject* args) { PyObject* value; if (!PyArg_ParseTuple(args, "O", &value)) { return NULL; @@ -94,7 +94,7 @@ PyObject* serialize_list(PyObject* self, PyObject* args) { \return The arrow metadata offset for the arrow metadata */ -PyObject* write_to_buffer(PyObject* self, PyObject* args) { +static PyObject* write_to_buffer(PyObject* self, PyObject* args) { std::shared_ptr* batch; PyObject* memoryview; if (!PyArg_ParseTuple(args, "O&O", &PyObjectToArrow, &batch, &memoryview)) { @@ -119,7 +119,7 @@ PyObject* write_to_buffer(PyObject* self, PyObject* args) { \return A Python "arrow" capsule containing the arrow RowBatch */ -PyObject* read_from_buffer(PyObject* self, PyObject* args) { +static PyObject* read_from_buffer(PyObject* self, PyObject* args) { PyObject* memoryview; PyObject* metadata; int64_t metadata_offset; @@ -149,7 +149,7 @@ PyObject* read_from_buffer(PyObject* self, PyObject* args) { /*! */ -PyObject* deserialize_list(PyObject* self, PyObject* args) { +static PyObject* deserialize_list(PyObject* self, PyObject* args) { std::shared_ptr* data; if (!PyArg_ParseTuple(args, "O&", &PyObjectToArrow, &data)) { return NULL; diff --git a/python/test/runtest.py b/python/test/runtest.py index 9cfa62c92..5c713cc14 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -5,7 +5,7 @@ from numpy.testing import assert_equal TEST_OBJECTS = [[1, "hello", 3.0], 42, 43L, "hello world", 42.0, 1L << 62, (1.0, "hi"), None, (None, None), ("hello", None), - True, False, (True, False), + True, False, (True, False), "hello", {True: "hello", False: "world"}, {"hello" : "world", 1: 42, 1.0: 45}, {}, np.int8(3), np.int32(4), np.int64(5), From 0dcfcfa2e912f135c7bbae577856364c5aa26afe Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Mon, 25 Jul 2016 13:21:31 -0700 Subject: [PATCH 06/32] fix compilation on macOS --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5019a9e62..a46830891 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,7 @@ add_library(numbuf SHARED python/src/pynumbuf/adapters/numpy.cc python/src/pynumbuf/adapters/python.cc) +get_filename_component(PYTHON_SHARED_LIBRARY ${PYTHON_LIBRARIES} NAME) if(APPLE) add_custom_command(TARGET numbuf POST_BUILD COMMAND From 212536f87ccfdb474a5f32dc54fb563fd6007b8d Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Mon, 25 Jul 2016 16:38:17 -0700 Subject: [PATCH 07/32] Put documentation into ray-core --- python/src/pynumbuf/numbuf.cc | 36 ++++------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/python/src/pynumbuf/numbuf.cc b/python/src/pynumbuf/numbuf.cc index d95feaf84..b0e4457d3 100644 --- a/python/src/pynumbuf/numbuf.cc +++ b/python/src/pynumbuf/numbuf.cc @@ -40,18 +40,7 @@ static void ArrowCapsule_Destructor(PyObject* capsule) { delete reinterpret_cast*>(PyCapsule_GetPointer(capsule, "arrow")); } -/*! Serializes a Python list into an Arrow array. - - \param args - The argument must be a Python list - - \returns - A bytearray object containing the schema metadata - - The size in bytes the serialized object will occupy in memory - - A Python "arrow" capsule containing the RowBatch -*/ +/* Documented in doc/numbuf.rst in ray-core */ static PyObject* serialize_list(PyObject* self, PyObject* args) { PyObject* value; if (!PyArg_ParseTuple(args, "O", &value)) { @@ -85,15 +74,7 @@ static PyObject* serialize_list(PyObject* self, PyObject* args) { return NULL; } -/*! Serialize an arrow::Array into a buffer. - - \param args - A Python "arrow" capsule containing the arrow::Array and - a python memoryview object the data will be written to - - \return - The arrow metadata offset for the arrow metadata -*/ +/* Documented in doc/numbuf.rst in ray-core */ static PyObject* write_to_buffer(PyObject* self, PyObject* args) { std::shared_ptr* batch; PyObject* memoryview; @@ -110,15 +91,7 @@ static PyObject* write_to_buffer(PyObject* self, PyObject* args) { return PyInt_FromLong(metadata_offset); } -/*! Read serialized data from buffer and produce an arrow capsule - - \param args - A Python memoryview from which data will be loaded, - a Python bytearray containing the metadata and the metadata_offset - - \return - A Python "arrow" capsule containing the arrow RowBatch -*/ +/* Documented in doc/numbuf.rst in ray-core */ static PyObject* read_from_buffer(PyObject* self, PyObject* args) { PyObject* memoryview; PyObject* metadata; @@ -147,8 +120,7 @@ static PyObject* read_from_buffer(PyObject* self, PyObject* args) { "arrow", &ArrowCapsule_Destructor); } -/*! -*/ +/* Documented in doc/numbuf.rst in ray-core */ static PyObject* deserialize_list(PyObject* self, PyObject* args) { std::shared_ptr* data; if (!PyArg_ParseTuple(args, "O&", &PyObjectToArrow, &data)) { From de8bbaca61def7d5fe6a06c1a66c858f7bbaf13c Mon Sep 17 00:00:00 2001 From: Mehrdad Date: Wed, 27 Jul 2016 21:08:09 -0700 Subject: [PATCH 08/32] long long bug fix --- python/src/pynumbuf/numbuf.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/pynumbuf/numbuf.cc b/python/src/pynumbuf/numbuf.cc index b0e4457d3..33b4ac078 100644 --- a/python/src/pynumbuf/numbuf.cc +++ b/python/src/pynumbuf/numbuf.cc @@ -96,7 +96,7 @@ static PyObject* read_from_buffer(PyObject* self, PyObject* args) { PyObject* memoryview; PyObject* metadata; int64_t metadata_offset; - if (!PyArg_ParseTuple(args, "OOl", &memoryview, &metadata, &metadata_offset)) { + if (!PyArg_ParseTuple(args, "OOL", &memoryview, &metadata, &metadata_offset)) { return NULL; } From a81dc0c54197116dd236f4476ed13bdc6ee5e55b Mon Sep 17 00:00:00 2001 From: Mehrdad Date: Wed, 27 Jul 2016 21:08:32 -0700 Subject: [PATCH 09/32] Visual Studio projects --- vsprojects/numbuf.vcxproj | 75 +++++++++++++++++++++++++++++++ vsprojects/numbuf.vcxproj.filters | 60 +++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 vsprojects/numbuf.vcxproj create mode 100644 vsprojects/numbuf.vcxproj.filters diff --git a/vsprojects/numbuf.vcxproj b/vsprojects/numbuf.vcxproj new file mode 100644 index 000000000..b762afa79 --- /dev/null +++ b/vsprojects/numbuf.vcxproj @@ -0,0 +1,75 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + {609D1438-D42D-4CBA-80A5-A1398C3BCC85} + Win32Proj + + + + v140 + DynamicLibrary + + + + + + + + + + + + + lib$(MSBuildProjectName) + .pyd + + + + $(THIRD_PARTY)arrow\cpp\src;%(AdditionalIncludeDirectories) + + + Console + + + + + + + + + + + + + + + + + + + + + + {10e7d8e8-0eeb-46ea-a58d-f9236b5960ad} + + + + \ No newline at end of file diff --git a/vsprojects/numbuf.vcxproj.filters b/vsprojects/numbuf.vcxproj.filters new file mode 100644 index 000000000..c20d75f12 --- /dev/null +++ b/vsprojects/numbuf.vcxproj.filters @@ -0,0 +1,60 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + \ No newline at end of file From 8e165d43d455925732d5361b79f818b6cead59ca Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 4 Aug 2016 15:34:47 -0700 Subject: [PATCH 10/32] Implement unicode serialization --- cpp/src/numbuf/sequence.cc | 32 +++++++++++++-------- cpp/src/numbuf/sequence.h | 39 +++++++++++++++----------- python/src/pynumbuf/adapters/numpy.cc | 20 ++++++------- python/src/pynumbuf/adapters/python.cc | 33 +++++++++++++++++----- python/src/pynumbuf/adapters/scalars.h | 8 +++--- python/test/runtest.py | 5 ++-- 6 files changed, 85 insertions(+), 52 deletions(-) diff --git a/cpp/src/numbuf/sequence.cc b/cpp/src/numbuf/sequence.cc index ae250180f..a5e1cf8ca 100644 --- a/cpp/src/numbuf/sequence.cc +++ b/cpp/src/numbuf/sequence.cc @@ -8,7 +8,9 @@ SequenceBuilder::SequenceBuilder(MemoryPool* pool) : pool_(pool), types_(pool), offsets_(pool), nones_(pool, std::make_shared()), bools_(pool, std::make_shared()), - ints_(pool), strings_(pool, std::make_shared()), + ints_(pool), + bytes_(pool, std::make_shared()), + strings_(pool, std::make_shared()), floats_(pool), doubles_(pool), uint8_tensors_(std::make_shared(), pool), int8_tensors_(std::make_shared(), pool), @@ -31,46 +33,51 @@ SequenceBuilder::SequenceBuilder(MemoryPool* pool) RETURN_NOT_OK(types_.Append(TAG)); \ RETURN_NOT_OK(nones_.AppendToBitmap(true)); -Status SequenceBuilder::Append() { +Status SequenceBuilder::AppendNone() { RETURN_NOT_OK(offsets_.Append(0)); RETURN_NOT_OK(types_.Append(0)); return nones_.AppendToBitmap(false); } -Status SequenceBuilder::Append(bool data) { +Status SequenceBuilder::AppendBool(bool data) { UPDATE(bools_.length(), bool_tag); return bools_.Append(data); } -Status SequenceBuilder::Append(int64_t data) { +Status SequenceBuilder::AppendInt64(int64_t data) { UPDATE(ints_.length(), int_tag); return ints_.Append(data); } -Status SequenceBuilder::Append(uint64_t data) { +Status SequenceBuilder::AppendUInt64(uint64_t data) { UPDATE(ints_.length(), int_tag); return ints_.Append(data); } -Status SequenceBuilder::Append(const char* data, int32_t length) { +Status SequenceBuilder::AppendBytes(const uint8_t* data, int32_t length) { + UPDATE(bytes_.length(), bytes_tag); + return bytes_.Append(data, length); +} + +Status SequenceBuilder::AppendString(const char* data, int32_t length) { UPDATE(strings_.length(), string_tag); return strings_.Append(data, length); } -Status SequenceBuilder::Append(float data) { +Status SequenceBuilder::AppendFloat(float data) { UPDATE(floats_.length(), float_tag); return floats_.Append(data); } -Status SequenceBuilder::Append(double data) { +Status SequenceBuilder::AppendDouble(double data) { UPDATE(doubles_.length(), double_tag); return doubles_.Append(data); } -#define DEF_TENSOR_APPEND(NAME, TYPE, TAG) \ - Status SequenceBuilder::Append(const std::vector& dims, TYPE* data) { \ - UPDATE(NAME.length(), TAG); \ - return NAME.Append(dims, data); \ +#define DEF_TENSOR_APPEND(NAME, TYPE, TAG) \ + Status SequenceBuilder::AppendTensor(const std::vector& dims, TYPE* data) { \ + UPDATE(NAME.length(), TAG); \ + return NAME.Append(dims, data); \ } DEF_TENSOR_APPEND(uint8_tensors_, uint8_t, uint8_tensor_tag); @@ -136,6 +143,7 @@ std::shared_ptr SequenceBuilder::Finish( ADD_ELEMENT(bools_, bool_tag); ADD_ELEMENT(ints_, int_tag); ADD_ELEMENT(strings_, string_tag); + ADD_ELEMENT(bytes_, bytes_tag); ADD_ELEMENT(floats_, float_tag); ADD_ELEMENT(doubles_, double_tag); diff --git a/cpp/src/numbuf/sequence.h b/cpp/src/numbuf/sequence.h index f30ea00fe..7237dbd83 100644 --- a/cpp/src/numbuf/sequence.h +++ b/cpp/src/numbuf/sequence.h @@ -15,25 +15,28 @@ class SequenceBuilder { SequenceBuilder(arrow::MemoryPool* pool = nullptr); //! Appending a none to the sequence - arrow::Status Append(); + arrow::Status AppendNone(); //! Appending a boolean to the sequence - arrow::Status Append(bool data); + arrow::Status AppendBool(bool data); //! Appending an int64_t to the sequence - arrow::Status Append(int64_t data); + arrow::Status AppendInt64(int64_t data); //! Appending an uint64_t to the sequence - arrow::Status Append(uint64_t data); + arrow::Status AppendUInt64(uint64_t data); + + //! Append a list of bytes to the sequence + arrow::Status AppendBytes(const uint8_t* data, int32_t length); //! Appending a string to the sequence - arrow::Status Append(const char* data, int32_t length); + arrow::Status AppendString(const char* data, int32_t length); //! Appending a float to the sequence - arrow::Status Append(float data); + arrow::Status AppendFloat(float data); //! Appending a double to the sequence - arrow::Status Append(double data); + arrow::Status AppendDouble(double data); /*! Appending a tensor to the sequence @@ -44,16 +47,16 @@ class SequenceBuilder { A pointer to the start of the data block. The length of the data block will be the product of the dimensions */ - arrow::Status Append(const std::vector& dims, uint8_t* data); - arrow::Status Append(const std::vector& dims, int8_t* data); - arrow::Status Append(const std::vector& dims, uint16_t* data); - arrow::Status Append(const std::vector& dims, int16_t* data); - arrow::Status Append(const std::vector& dims, uint32_t* data); - arrow::Status Append(const std::vector& dims, int32_t* data); - arrow::Status Append(const std::vector& dims, uint64_t* data); - arrow::Status Append(const std::vector& dims, int64_t* data); - arrow::Status Append(const std::vector& dims, float* data); - arrow::Status Append(const std::vector& dims, double* data); + arrow::Status AppendTensor(const std::vector& dims, uint8_t* data); + arrow::Status AppendTensor(const std::vector& dims, int8_t* data); + arrow::Status AppendTensor(const std::vector& dims, uint16_t* data); + arrow::Status AppendTensor(const std::vector& dims, int16_t* data); + arrow::Status AppendTensor(const std::vector& dims, uint32_t* data); + arrow::Status AppendTensor(const std::vector& dims, int32_t* data); + arrow::Status AppendTensor(const std::vector& dims, uint64_t* data); + arrow::Status AppendTensor(const std::vector& dims, int64_t* data); + arrow::Status AppendTensor(const std::vector& dims, float* data); + arrow::Status AppendTensor(const std::vector& dims, double* data); /*! Add a sublist to the sequenc. The data contained in the sublist will be specified in the "Finish" method. @@ -90,6 +93,7 @@ class SequenceBuilder { arrow::NullArrayBuilder nones_; arrow::BooleanBuilder bools_; arrow::Int64Builder ints_; + arrow::BinaryBuilder bytes_; arrow::StringBuilder strings_; arrow::FloatBuilder floats_; arrow::DoubleBuilder doubles_; @@ -112,6 +116,7 @@ class SequenceBuilder { int8_t bool_tag = -1; int8_t int_tag = -1; int8_t string_tag = -1; + int8_t bytes_tag = -1; int8_t float_tag = -1; int8_t double_tag = -1; diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc index 67537fd25..45c4149a4 100644 --- a/python/src/pynumbuf/adapters/numpy.cc +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -65,34 +65,34 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder) { auto data = PyArray_DATA(contiguous); switch (dtype) { case NPY_UINT8: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; case NPY_INT8: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; case NPY_UINT16: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; case NPY_INT16: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; case NPY_UINT32: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; case NPY_INT32: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; case NPY_UINT64: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; case NPY_INT64: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; case NPY_FLOAT: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; case NPY_DOUBLE: - RETURN_NOT_OK(builder.Append(dims, reinterpret_cast(data))); + RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; default: DCHECK(false) << "numpy data type not recognized: " << dtype; diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 7c747c4d9..790fd5497 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -15,10 +15,15 @@ PyObject* get_value(ArrayPtr arr, int32_t index, int32_t type) { return PyBool_FromLong(std::static_pointer_cast(arr)->Value(index)); case Type::INT64: return PyInt_FromLong(std::static_pointer_cast(arr)->Value(index)); + case Type::BINARY: { + int32_t nchars; + const uint8_t* str = std::static_pointer_cast(arr)->GetValue(index, &nchars); + return PyString_FromStringAndSize(reinterpret_cast(str), nchars); + } case Type::STRING: { int32_t nchars; const uint8_t* str = std::static_pointer_cast(arr)->GetValue(index, &nchars); - return PyString_FromStringAndSize(reinterpret_cast(str), nchars); + return PyUnicode_FromStringAndSize(reinterpret_cast(str), nchars); } case Type::FLOAT: return PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); @@ -50,20 +55,34 @@ Status append(PyObject* elem, SequenceBuilder& builder, std::vector& subdicts) { // The bool case must precede the int case (PyInt_Check passes for bools) if (PyBool_Check(elem)) { - RETURN_NOT_OK(builder.Append(elem == Py_True)); + RETURN_NOT_OK(builder.AppendBool(elem == Py_True)); } else if (PyFloat_Check(elem)) { - RETURN_NOT_OK(builder.Append(PyFloat_AS_DOUBLE(elem))); + RETURN_NOT_OK(builder.AppendFloat(PyFloat_AS_DOUBLE(elem))); } else if (PyLong_Check(elem)) { int overflow = 0; int64_t data = PyLong_AsLongLongAndOverflow(elem, &overflow); - RETURN_NOT_OK(builder.Append(data)); + RETURN_NOT_OK(builder.AppendInt64(data)); if(overflow) { return Status::NotImplemented("long overflow"); } } else if (PyInt_Check(elem)) { - RETURN_NOT_OK(builder.Append(static_cast(PyInt_AS_LONG(elem)))); + RETURN_NOT_OK(builder.AppendInt64(static_cast(PyInt_AS_LONG(elem)))); } else if (PyString_Check(elem)) { - RETURN_NOT_OK(builder.Append(PyString_AS_STRING(elem), PyString_GET_SIZE(elem))); + auto data = reinterpret_cast(PyString_AS_STRING(elem)); + auto size = PyString_GET_SIZE(elem); + RETURN_NOT_OK(builder.AppendBytes(data, size)); + } else if (PyUnicode_Check(elem)) { + Py_ssize_t size; + #if PY_MAJOR_VERSION >= 3 + char* data = PyUnicode_AsUTF8AndSize(elem, &size); // TODO(pcm): Check if this is correct + #else + PyObject* str = PyUnicode_AsUTF8String(elem); + char* data = PyString_AS_STRING(str); + size = PyString_GET_SIZE(str); + #endif + Status s = builder.AppendString(data, size); + Py_XDECREF(str); + RETURN_NOT_OK(s); } else if (PyList_Check(elem)) { builder.AppendList(PyList_Size(elem)); sublists.push_back(elem); @@ -78,7 +97,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, } else if (PyArray_Check(elem)) { RETURN_NOT_OK(SerializeArray((PyArrayObject*) elem, builder)); } else if (elem == Py_None) { - RETURN_NOT_OK(builder.Append()); + RETURN_NOT_OK(builder.AppendNone()); } else { std::stringstream ss; ss << "data type of " << PyString_AS_STRING(PyObject_Repr(elem)) diff --git a/python/src/pynumbuf/adapters/scalars.h b/python/src/pynumbuf/adapters/scalars.h index 928289252..e680b446f 100644 --- a/python/src/pynumbuf/adapters/scalars.h +++ b/python/src/pynumbuf/adapters/scalars.h @@ -16,11 +16,11 @@ namespace numbuf { arrow::Status AppendScalar(PyObject* obj, SequenceBuilder& builder) { if (PyArray_IsScalar(obj, Bool)) { - return builder.Append(((PyBoolScalarObject *)obj)->obval != 0); + return builder.AppendBool(((PyBoolScalarObject *)obj)->obval != 0); } else if (PyArray_IsScalar(obj, Float)) { - return builder.Append(((PyFloatScalarObject *)obj)->obval); + return builder.AppendFloat(((PyFloatScalarObject *)obj)->obval); } else if (PyArray_IsScalar(obj, Double)) { - return builder.Append(((PyDoubleScalarObject *)obj)->obval); + return builder.AppendDouble(((PyDoubleScalarObject *)obj)->obval); } int64_t value = 0; if (PyArray_IsScalar(obj, Byte)) { @@ -46,7 +46,7 @@ arrow::Status AppendScalar(PyObject* obj, SequenceBuilder& builder) { } else { DCHECK(false) << "scalar type not recognized"; } - return builder.Append(value); + return builder.AppendInt64(value); } } // namespace diff --git a/python/test/runtest.py b/python/test/runtest.py index 5c713cc14..14545117d 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -3,8 +3,9 @@ import libnumbuf import numpy as np from numpy.testing import assert_equal -TEST_OBJECTS = [[1, "hello", 3.0], 42, 43L, "hello world", 42.0, 1L << 62, - (1.0, "hi"), None, (None, None), ("hello", None), +TEST_OBJECTS = [[1, "hello", 3.0], 42, 43L, "hello world", u"x", u"\u262F", + 42.0, 1L << 62, (1.0, "hi"), + None, (None, None), ("hello", None), True, False, (True, False), "hello", {True: "hello", False: "world"}, {"hello" : "world", 1: 42, 1.0: 45}, {}, From b6b2b03950b077c1d75fe5be710f18ea88b688dd Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Mon, 8 Aug 2016 16:29:59 -0700 Subject: [PATCH 11/32] throw proper error if numpy array that contains object is serialized --- python/src/pynumbuf/adapters/numpy.cc | 6 +++++- python/test/runtest.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc index 45c4149a4..fb74db28f 100644 --- a/python/src/pynumbuf/adapters/numpy.cc +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -1,5 +1,7 @@ #include "numpy.h" +#include + #include using namespace arrow; @@ -95,7 +97,9 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder) { RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; default: - DCHECK(false) << "numpy data type not recognized: " << dtype; + std::stringstream stream; + stream << "numpy data type not recognized: " << dtype; + return Status::NotImplemented(stream.str()); } Py_XDECREF(contiguous); return Status::OK(); diff --git a/python/test/runtest.py b/python/test/runtest.py index 14545117d..1bb4addba 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -47,6 +47,13 @@ class SerializationTests(unittest.TestCase): for t in ["int8", "uint8", "int16", "uint16", "int32", "uint32", "float32", "float64"]: self.numpyTest(t) + def testNumpyObject(self): + a = np.array([np.zeros((2,2))], dtype=object) + try: + x = self.roundTripTest([a]) + except: + pass + def testRay(self): for obj in TEST_OBJECTS: self.roundTripTest([obj]) From 7b548bddd337f2358fdf4a042f12c917a29da40d Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Mon, 8 Aug 2016 17:58:44 -0700 Subject: [PATCH 12/32] gitignore build directory --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4581ef2ee..2a83b5467 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ *.exe *.out *.app + +# Build directory +build From f468871b5d3f35de633a01e5aad1a61748e5ef9d Mon Sep 17 00:00:00 2001 From: William Paul Date: Wed, 24 Aug 2016 23:31:09 +0000 Subject: [PATCH 13/32] Tuples as keys in dicts can be serialized --- cpp/src/numbuf/dict.cc | 3 ++- cpp/src/numbuf/dict.h | 1 + python/src/pynumbuf/adapters/python.cc | 10 +++++++--- python/test/runtest.py | 5 +++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cpp/src/numbuf/dict.cc b/cpp/src/numbuf/dict.cc index ff5f7239f..14bf7f2f3 100644 --- a/cpp/src/numbuf/dict.cc +++ b/cpp/src/numbuf/dict.cc @@ -5,12 +5,13 @@ using namespace arrow; namespace numbuf { std::shared_ptr DictBuilder::Finish( + std::shared_ptr key_tuple_data, std::shared_ptr list_data, std::shared_ptr tuple_data, std::shared_ptr dict_data) { // lists and dicts can't be keys of dicts in Python, that is why for // the keys we do not need to collect sublists - auto keys = keys_.Finish(nullptr, nullptr, nullptr); + auto keys = keys_.Finish(nullptr, key_tuple_data, nullptr); auto vals = vals_.Finish(list_data, tuple_data, dict_data); auto keys_field = std::make_shared("keys", keys->type()); auto vals_field = std::make_shared("vals", vals->type()); diff --git a/cpp/src/numbuf/dict.h b/cpp/src/numbuf/dict.h index 5c2abeacd..02eb6035d 100644 --- a/cpp/src/numbuf/dict.h +++ b/cpp/src/numbuf/dict.h @@ -34,6 +34,7 @@ public: value list of the dictionary */ std::shared_ptr Finish( + std::shared_ptr key_tuple_data, std::shared_ptr list_data, std::shared_ptr tuple_data, std::shared_ptr dict_data); diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 790fd5497..871e0e097 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -171,16 +171,20 @@ Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t Status SerializeDict(std::vector dicts, std::shared_ptr* out) { DictBuilder result; - std::vector sublists, subtuples, subdicts, dummy; + std::vector sublists, subtuples, subdicts, keytuples, dummy; for (const auto& dict : dicts) { PyObject *key, *value; Py_ssize_t pos = 0; while (PyDict_Next(dict, &pos, &key, &value)) { - RETURN_NOT_OK(append(key, result.keys(), dummy, dummy, dummy)); + RETURN_NOT_OK(append(key, result.keys(), dummy, keytuples, dummy)); DCHECK(dummy.size() == 0); RETURN_NOT_OK(append(value, result.vals(), sublists, subtuples, subdicts)); } } + std::shared_ptr key_val_tuples; + if (keytuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(keytuples, &key_val_tuples)); + } std::shared_ptr val_list; if (sublists.size() > 0) { RETURN_NOT_OK(SerializeSequences(sublists, &val_list)); @@ -193,7 +197,7 @@ Status SerializeDict(std::vector dicts, std::shared_ptr* out) if (subdicts.size() > 0) { RETURN_NOT_OK(SerializeDict(subdicts, &val_dict)); } - *out = result.Finish(val_list, val_tuples, val_dict); + *out = result.Finish(key_val_tuples, val_list, val_tuples, val_dict); return Status::OK(); } diff --git a/python/test/runtest.py b/python/test/runtest.py index 1bb4addba..f8966b5ef 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -3,8 +3,9 @@ import libnumbuf import numpy as np from numpy.testing import assert_equal -TEST_OBJECTS = [[1, "hello", 3.0], 42, 43L, "hello world", u"x", u"\u262F", - 42.0, 1L << 62, (1.0, "hi"), +TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world", + u"x", u"\u262F", 42.0, + 1L << 62, (1.0, "hi"), None, (None, None), ("hello", None), True, False, (True, False), "hello", {True: "hello", False: "world"}, From 5c409c391667cedf872a549dc5ba7a429e26ed17 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 25 Aug 2016 16:04:31 -0700 Subject: [PATCH 14/32] make variable names more uniform --- cpp/src/numbuf/dict.cc | 8 +++---- cpp/src/numbuf/dict.h | 6 ++--- python/src/pynumbuf/adapters/python.cc | 32 +++++++++++++------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cpp/src/numbuf/dict.cc b/cpp/src/numbuf/dict.cc index 14bf7f2f3..98b03894b 100644 --- a/cpp/src/numbuf/dict.cc +++ b/cpp/src/numbuf/dict.cc @@ -6,13 +6,13 @@ namespace numbuf { std::shared_ptr DictBuilder::Finish( std::shared_ptr key_tuple_data, - std::shared_ptr list_data, - std::shared_ptr tuple_data, - std::shared_ptr dict_data) { + std::shared_ptr val_list_data, + std::shared_ptr val_tuple_data, + std::shared_ptr val_dict_data) { // lists and dicts can't be keys of dicts in Python, that is why for // the keys we do not need to collect sublists auto keys = keys_.Finish(nullptr, key_tuple_data, nullptr); - auto vals = vals_.Finish(list_data, tuple_data, dict_data); + auto vals = vals_.Finish(val_list_data, val_tuple_data, val_dict_data); auto keys_field = std::make_shared("keys", keys->type()); auto vals_field = std::make_shared("vals", vals->type()); auto type = std::make_shared(std::vector({keys_field, vals_field})); diff --git a/cpp/src/numbuf/dict.h b/cpp/src/numbuf/dict.h index 02eb6035d..dcb38fb6b 100644 --- a/cpp/src/numbuf/dict.h +++ b/cpp/src/numbuf/dict.h @@ -35,9 +35,9 @@ public: */ std::shared_ptr Finish( std::shared_ptr key_tuple_data, - std::shared_ptr list_data, - std::shared_ptr tuple_data, - std::shared_ptr dict_data); + std::shared_ptr val_list_data, + std::shared_ptr val_tuple_data, + std::shared_ptr val_dict_data); private: SequenceBuilder keys_; diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 871e0e097..ba2c8b0e3 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -171,33 +171,33 @@ Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t Status SerializeDict(std::vector dicts, std::shared_ptr* out) { DictBuilder result; - std::vector sublists, subtuples, subdicts, keytuples, dummy; + std::vector key_tuples, val_lists, val_tuples, val_dicts, dummy; for (const auto& dict : dicts) { PyObject *key, *value; Py_ssize_t pos = 0; while (PyDict_Next(dict, &pos, &key, &value)) { - RETURN_NOT_OK(append(key, result.keys(), dummy, keytuples, dummy)); + RETURN_NOT_OK(append(key, result.keys(), dummy, key_tuples, dummy)); DCHECK(dummy.size() == 0); - RETURN_NOT_OK(append(value, result.vals(), sublists, subtuples, subdicts)); + RETURN_NOT_OK(append(value, result.vals(), val_lists, val_tuples, val_dicts)); } } - std::shared_ptr key_val_tuples; - if (keytuples.size() > 0) { - RETURN_NOT_OK(SerializeSequences(keytuples, &key_val_tuples)); + std::shared_ptr key_tuples_arr; + if (key_tuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(key_tuples, &key_tuples_arr)); } - std::shared_ptr val_list; - if (sublists.size() > 0) { - RETURN_NOT_OK(SerializeSequences(sublists, &val_list)); + std::shared_ptr val_list_arr; + if (val_lists.size() > 0) { + RETURN_NOT_OK(SerializeSequences(val_lists, &val_list_arr)); } - std::shared_ptr val_tuples; - if (subtuples.size() > 0) { - RETURN_NOT_OK(SerializeSequences(subtuples, &val_tuples)); + std::shared_ptr val_tuples_arr; + if (val_tuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(val_tuples, &val_tuples_arr)); } - std::shared_ptr val_dict; - if (subdicts.size() > 0) { - RETURN_NOT_OK(SerializeDict(subdicts, &val_dict)); + std::shared_ptr val_dict_arr; + if (val_dicts.size() > 0) { + RETURN_NOT_OK(SerializeDict(val_dicts, &val_dict_arr)); } - *out = result.Finish(key_val_tuples, val_list, val_tuples, val_dict); + *out = result.Finish(key_tuples_arr, val_list_arr, val_tuples_arr, val_dict_arr); return Status::OK(); } From 7481a02024dcbb4ee13f3b3a3cfdad986309807b Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 25 Aug 2016 22:25:45 -0700 Subject: [PATCH 15/32] add custom callbacks for serialization --- python/src/pynumbuf/adapters/python.cc | 44 +++++++++++++++++++++++--- python/src/pynumbuf/numbuf.cc | 29 +++++++++++++++++ python/test/runtest.py | 31 ++++++++++++++++++ 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index ba2c8b0e3..89177e425 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -6,6 +6,9 @@ using namespace arrow; +extern PyObject* numbuf_serialize_callback; +extern PyObject* numbuf_deserialize_callback; + namespace numbuf { PyObject* get_value(ArrayPtr arr, int32_t index, int32_t type) { @@ -49,6 +52,17 @@ PyObject* get_value(ArrayPtr arr, int32_t index, int32_t type) { return NULL; } +Status python_error_to_status() { + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + char *err_message = PyString_AsString(value); + std::stringstream ss; + if (err_message) { + ss << "Python error in callback: " << err_message; + } + return Status::NotImplemented(ss.str()); +} + Status append(PyObject* elem, SequenceBuilder& builder, std::vector& sublists, std::vector& subtuples, @@ -99,10 +113,22 @@ Status append(PyObject* elem, SequenceBuilder& builder, } else if (elem == Py_None) { RETURN_NOT_OK(builder.AppendNone()); } else { - std::stringstream ss; - ss << "data type of " << PyString_AS_STRING(PyObject_Repr(elem)) - << " not recognized"; - return Status::NotImplemented(ss.str()); + if (!numbuf_serialize_callback) { + std::stringstream ss; + ss << "data type of " << PyString_AS_STRING(PyObject_Repr(elem)) + << " not recognized and custom serialization handler not registered"; + return Status::NotImplemented(ss.str()); + } else { + PyObject* arglist = Py_BuildValue("(O)", elem); + PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); + if (!result) { + Py_XDECREF(arglist); + return python_error_to_status(); + } + builder.AppendDict(PyDict_Size(result)); + subdicts.push_back(result); + Py_XDECREF(arglist); + } } return Status::OK(); } @@ -213,6 +239,16 @@ Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t } Py_XDECREF(keys); // PyList_GetItem(keys, ...) incremented the reference count Py_XDECREF(vals); // PyList_GetItem(vals, ...) incremented the reference count + static PyObject* py_type = PyString_FromString("_pytype_"); + if (PyDict_Contains(result, py_type) && numbuf_deserialize_callback) { + PyObject* arglist = Py_BuildValue("(O)", result); + result = PyObject_CallObject(numbuf_deserialize_callback, arglist); + if (!result) { + Py_XDECREF(arglist); + return python_error_to_status(); + } + Py_XDECREF(arglist); + } *out = result; return Status::OK(); } diff --git a/python/src/pynumbuf/numbuf.cc b/python/src/pynumbuf/numbuf.cc index 33b4ac078..104747b4d 100644 --- a/python/src/pynumbuf/numbuf.cc +++ b/python/src/pynumbuf/numbuf.cc @@ -26,6 +26,9 @@ extern "C" { static PyObject *NumbufError; +PyObject *numbuf_serialize_callback = NULL; +PyObject *numbuf_deserialize_callback = NULL; + int PyObjectToArrow(PyObject* object, std::shared_ptr **result) { if (PyCapsule_IsValid(object, "arrow")) { *result = reinterpret_cast*>(PyCapsule_GetPointer(object, "arrow")); @@ -131,11 +134,37 @@ static PyObject* deserialize_list(PyObject* self, PyObject* args) { return result; } +static PyObject* register_callbacks(PyObject* self, PyObject* args) { + PyObject* result = NULL; + PyObject* serialize_callback; + PyObject* deserialize_callback; + if (PyArg_ParseTuple(args, "OO:register_callbacks", &serialize_callback, &deserialize_callback)) { + if (!PyCallable_Check(serialize_callback)) { + PyErr_SetString(PyExc_TypeError, "serialize_callback must be callable"); + return NULL; + } + if (!PyCallable_Check(deserialize_callback)) { + PyErr_SetString(PyExc_TypeError, "deserialize_callback must be callable"); + return NULL; + } + Py_XINCREF(serialize_callback); // Add a reference to new serialization callback + Py_XINCREF(deserialize_callback); // Add a reference to new deserialization callback + Py_XDECREF(numbuf_serialize_callback); // Dispose of old serialization callback + Py_XDECREF(numbuf_deserialize_callback); // Dispose of old deserialization callback + numbuf_serialize_callback = serialize_callback; + numbuf_deserialize_callback = deserialize_callback; + Py_INCREF(Py_None); + result = Py_None; + } + return result; +} + static PyMethodDef NumbufMethods[] = { { "serialize_list", serialize_list, METH_VARARGS, "serialize a Python list" }, { "deserialize_list", deserialize_list, METH_VARARGS, "deserialize a Python list" }, { "write_to_buffer", write_to_buffer, METH_VARARGS, "write serialized data to buffer"}, { "read_from_buffer", read_from_buffer, METH_VARARGS, "read serialized data from buffer"}, + { "register_callbacks", register_callbacks, METH_VARARGS, "set serialization and deserialization callbacks"}, { NULL, NULL, 0, NULL } }; diff --git a/python/test/runtest.py b/python/test/runtest.py index f8966b5ef..f4e161647 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -59,6 +59,37 @@ class SerializationTests(unittest.TestCase): for obj in TEST_OBJECTS: self.roundTripTest([obj]) + def testCallback(self): + + class Foo(object): + def __init__(self): + self.x = 1 + + class Bar(object): + def __init__(self): + self.foo = Foo() + + def serialize(obj): + return dict(obj.__dict__, **{"_pytype_": type(obj).__name__}) + + def deserialize(obj): + if obj["_pytype_"] == "Foo": + result = Foo() + elif obj["_pytype_"] == "Bar": + result = Bar() + + obj.pop("_pytype_", None) + result.__dict__ = obj + return result + + bar = Bar() + bar.foo.x = 42 + + libnumbuf.register_callbacks(serialize, deserialize) + + metadata, size, serialized = libnumbuf.serialize_list([bar]) + self.assertEqual(libnumbuf.deserialize_list(serialized)[0].foo.x, 42) + def testBuffer(self): for (i, obj) in enumerate(TEST_OBJECTS): schema, size, batch = libnumbuf.serialize_list([obj]) From 04c28001fc7ba9125aa188ec63673f916c6e5a56 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Tue, 30 Aug 2016 16:46:33 -0700 Subject: [PATCH 16/32] Serialize numpy arrays with custom objects --- python/src/pynumbuf/adapters/numpy.cc | 26 +++++++++++++++++++++++++- python/src/pynumbuf/adapters/numpy.h | 2 +- python/src/pynumbuf/adapters/python.cc | 2 +- python/src/pynumbuf/adapters/python.h | 2 ++ python/test/runtest.py | 24 +++++++++++++++++------- 5 files changed, 46 insertions(+), 10 deletions(-) diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc index fb74db28f..8b850c1c6 100644 --- a/python/src/pynumbuf/adapters/numpy.cc +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -1,4 +1,5 @@ #include "numpy.h" +#include "python.h" #include @@ -6,6 +7,11 @@ using namespace arrow; +extern "C" { + extern PyObject *numbuf_serialize_callback; + extern PyObject *numbuf_deserialize_callback; +} + namespace numbuf { #define ARROW_TYPE_TO_NUMPY_CASE(TYPE) \ @@ -52,7 +58,8 @@ Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject** return Status::OK(); } -Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder) { +Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, + std::vector& subdicts) { size_t ndim = PyArray_NDIM(array); int dtype = PyArray_TYPE(array); std::vector dims(ndim); @@ -96,6 +103,23 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder) { case NPY_DOUBLE: RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; + case NPY_OBJECT: + if (!numbuf_serialize_callback) { + std::stringstream stream; + stream << "numpy data type not recognized: " << dtype; + return Status::NotImplemented(stream.str()); + } else { + PyObject* arglist = Py_BuildValue("(O)", array); + PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); + if (!result) { + Py_XDECREF(arglist); + return python_error_to_status(); + } + builder.AppendDict(PyDict_Size(result)); + subdicts.push_back(result); + Py_XDECREF(arglist); + } + break; default: std::stringstream stream; stream << "numpy data type not recognized: " << dtype; diff --git a/python/src/pynumbuf/adapters/numpy.h b/python/src/pynumbuf/adapters/numpy.h index 8b8b3859a..a57e5c974 100644 --- a/python/src/pynumbuf/adapters/numpy.h +++ b/python/src/pynumbuf/adapters/numpy.h @@ -14,7 +14,7 @@ namespace numbuf { -arrow::Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder); +arrow::Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, std::vector& subdicts); arrow::Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject** out); } diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 89177e425..3b9b6b2c2 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -109,7 +109,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, } else if (PyArray_IsScalar(elem, Generic)) { RETURN_NOT_OK(AppendScalar(elem, builder)); } else if (PyArray_Check(elem)) { - RETURN_NOT_OK(SerializeArray((PyArrayObject*) elem, builder)); + RETURN_NOT_OK(SerializeArray((PyArrayObject*) elem, builder, subdicts)); } else if (elem == Py_None) { RETURN_NOT_OK(builder.AppendNone()); } else { diff --git a/python/src/pynumbuf/adapters/python.h b/python/src/pynumbuf/adapters/python.h index 1fa07f920..66efa3005 100644 --- a/python/src/pynumbuf/adapters/python.h +++ b/python/src/pynumbuf/adapters/python.h @@ -17,6 +17,8 @@ arrow::Status DeserializeList(std::shared_ptr array, int32_t start arrow::Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out); arrow::Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out); +arrow::Status python_error_to_status(); + } #endif diff --git a/python/test/runtest.py b/python/test/runtest.py index f4e161647..c1b7d21e8 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -48,13 +48,6 @@ class SerializationTests(unittest.TestCase): for t in ["int8", "uint8", "int16", "uint16", "int32", "uint32", "float32", "float64"]: self.numpyTest(t) - def testNumpyObject(self): - a = np.array([np.zeros((2,2))], dtype=object) - try: - x = self.roundTripTest([a]) - except: - pass - def testRay(self): for obj in TEST_OBJECTS: self.roundTripTest([obj]) @@ -90,6 +83,23 @@ class SerializationTests(unittest.TestCase): metadata, size, serialized = libnumbuf.serialize_list([bar]) self.assertEqual(libnumbuf.deserialize_list(serialized)[0].foo.x, 42) + def testObjectArray(self): + x = np.array([1, 2, "hello"], dtype=object) + y = np.array([[1, 2], [3, 4]], dtype=object) + + def myserialize(obj): + return {"_pytype_": "numpy.array", "data": obj.tolist()} + + def mydeserialize(obj): + if obj["_pytype_"] == "numpy.array": + return np.array(obj["data"], dtype=object) + + libnumbuf.register_callbacks(myserialize, mydeserialize) + + metadata, size, serialized = libnumbuf.serialize_list([x, y]) + + assert_equal(libnumbuf.deserialize_list(serialized), [x, y]) + def testBuffer(self): for (i, obj) in enumerate(TEST_OBJECTS): schema, size, batch = libnumbuf.serialize_list([obj]) From ad12d5fece78ac0a0f7d50d4043dd96e17574f1a Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Tue, 30 Aug 2016 19:31:04 -0700 Subject: [PATCH 17/32] Handle more general numpy object types. --- python/src/pynumbuf/adapters/numpy.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc index 8b850c1c6..95e90249c 100644 --- a/python/src/pynumbuf/adapters/numpy.cc +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -103,7 +103,7 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, case NPY_DOUBLE: RETURN_NOT_OK(builder.AppendTensor(dims, reinterpret_cast(data))); break; - case NPY_OBJECT: + default: if (!numbuf_serialize_callback) { std::stringstream stream; stream << "numpy data type not recognized: " << dtype; @@ -119,11 +119,6 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, subdicts.push_back(result); Py_XDECREF(arglist); } - break; - default: - std::stringstream stream; - stream << "numpy data type not recognized: " << dtype; - return Status::NotImplemented(stream.str()); } Py_XDECREF(contiguous); return Status::OK(); From 84cfce3be2cc0be36d7f4dfc6756fdd5c23930fa Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sun, 4 Sep 2016 00:15:05 -0700 Subject: [PATCH 18/32] introduce base object --- python/src/pynumbuf/adapters/numpy.cc | 8 ++++++-- python/src/pynumbuf/adapters/numpy.h | 2 +- python/src/pynumbuf/adapters/python.cc | 22 +++++++++++----------- python/src/pynumbuf/adapters/python.h | 6 +++--- python/src/pynumbuf/numbuf.cc | 5 +++-- 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc index 95e90249c..92319f181 100644 --- a/python/src/pynumbuf/adapters/numpy.cc +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -25,11 +25,15 @@ namespace numbuf { type* data = const_cast(values->raw_data()) \ + content->offset(offset); \ *out = PyArray_SimpleNewFromData(num_dims, dim.data(), NPY_##TYPE, \ - reinterpret_cast(data)); \ + reinterpret_cast(data)); \ + if (base != Py_None) { \ + PyArray_SetBaseObject((PyArrayObject*) *out, base); \ + } \ + Py_XINCREF(base); \ } \ return Status::OK(); -Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject** out) { +Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject* base, PyObject** out) { DCHECK(array); auto tensor = std::dynamic_pointer_cast(array); DCHECK(tensor); diff --git a/python/src/pynumbuf/adapters/numpy.h b/python/src/pynumbuf/adapters/numpy.h index a57e5c974..c0fcbb7f6 100644 --- a/python/src/pynumbuf/adapters/numpy.h +++ b/python/src/pynumbuf/adapters/numpy.h @@ -15,7 +15,7 @@ namespace numbuf { arrow::Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, std::vector& subdicts); -arrow::Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject** out); +arrow::Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject* base, PyObject** out); } diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 3b9b6b2c2..173767a1a 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -11,7 +11,7 @@ extern PyObject* numbuf_deserialize_callback; namespace numbuf { -PyObject* get_value(ArrayPtr arr, int32_t index, int32_t type) { +PyObject* get_value(ArrayPtr arr, int32_t index, int32_t type, PyObject* base) { PyObject* result; switch (arr->type()->type) { case Type::BOOL: @@ -36,13 +36,13 @@ PyObject* get_value(ArrayPtr arr, int32_t index, int32_t type) { auto s = std::static_pointer_cast(arr); auto l = std::static_pointer_cast(s->field(0)); if (s->type()->child(0)->name == "list") { - ARROW_CHECK_OK(DeserializeList(l->values(), l->value_offset(index), l->value_offset(index+1), &result)); + ARROW_CHECK_OK(DeserializeList(l->values(), l->value_offset(index), l->value_offset(index+1), base, &result)); } else if (s->type()->child(0)->name == "tuple") { - ARROW_CHECK_OK(DeserializeTuple(l->values(), l->value_offset(index), l->value_offset(index+1), &result)); + ARROW_CHECK_OK(DeserializeTuple(l->values(), l->value_offset(index), l->value_offset(index+1), base, &result)); } else if (s->type()->child(0)->name == "dict") { - ARROW_CHECK_OK(DeserializeDict(l->values(), l->value_offset(index), l->value_offset(index+1), &result)); + ARROW_CHECK_OK(DeserializeDict(l->values(), l->value_offset(index), l->value_offset(index+1), base, &result)); } else { - ARROW_CHECK_OK(DeserializeArray(arr, index, &result)); + ARROW_CHECK_OK(DeserializeArray(arr, index, base, &result)); } return result; } @@ -181,17 +181,17 @@ Status SerializeSequences(std::vector sequences, std::shared_ptrValue(i); \ int8_t type = types->Value(i); \ ArrayPtr arr = data->child(type); \ - SET_ITEM(result, i-start_idx, get_value(arr, offset, type)); \ + SET_ITEM(result, i-start_idx, get_value(arr, offset, type, base)); \ } \ } \ *out = result; \ return Status::OK(); -Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out) { +Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out) { DESERIALIZE_SEQUENCE(PyList_New, PyList_SetItem) } -Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out) { +Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out) { DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SetItem) } @@ -227,13 +227,13 @@ Status SerializeDict(std::vector dicts, std::shared_ptr* out) return Status::OK(); } -Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out) { +Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out) { auto data = std::dynamic_pointer_cast(array); // TODO(pcm): error handling, get rid of the temporary copy of the list PyObject *keys, *vals; PyObject* result = PyDict_New(); - ARROW_RETURN_NOT_OK(DeserializeList(data->field(0), start_idx, stop_idx, &keys)); - ARROW_RETURN_NOT_OK(DeserializeList(data->field(1), start_idx, stop_idx, &vals)); + ARROW_RETURN_NOT_OK(DeserializeList(data->field(0), start_idx, stop_idx, base, &keys)); + ARROW_RETURN_NOT_OK(DeserializeList(data->field(1), start_idx, stop_idx, base, &vals)); for (size_t i = start_idx; i < stop_idx; ++i) { PyDict_SetItem(result, PyList_GetItem(keys, i - start_idx), PyList_GetItem(vals, i - start_idx)); } diff --git a/python/src/pynumbuf/adapters/python.h b/python/src/pynumbuf/adapters/python.h index 66efa3005..13ee629d3 100644 --- a/python/src/pynumbuf/adapters/python.h +++ b/python/src/pynumbuf/adapters/python.h @@ -13,9 +13,9 @@ namespace numbuf { arrow::Status SerializeSequences(std::vector sequences, std::shared_ptr* out); arrow::Status SerializeDict(std::vector dicts, std::shared_ptr* out); -arrow::Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out); -arrow::Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out); -arrow::Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject** out); +arrow::Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); +arrow::Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); +arrow::Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); arrow::Status python_error_to_status(); diff --git a/python/src/pynumbuf/numbuf.cc b/python/src/pynumbuf/numbuf.cc index 104747b4d..d89913d6e 100644 --- a/python/src/pynumbuf/numbuf.cc +++ b/python/src/pynumbuf/numbuf.cc @@ -126,11 +126,12 @@ static PyObject* read_from_buffer(PyObject* self, PyObject* args) { /* Documented in doc/numbuf.rst in ray-core */ static PyObject* deserialize_list(PyObject* self, PyObject* args) { std::shared_ptr* data; - if (!PyArg_ParseTuple(args, "O&", &PyObjectToArrow, &data)) { + PyObject* base = Py_None; + if (!PyArg_ParseTuple(args, "O&|O", &PyObjectToArrow, &data, &base)) { return NULL; } PyObject* result; - ARROW_CHECK_OK(DeserializeList((*data)->column(0), 0, (*data)->num_rows(), &result)); + ARROW_CHECK_OK(DeserializeList((*data)->column(0), 0, (*data)->num_rows(), base, &result)); return result; } From a428806788cb8de345cc0c279dbcb59cf9e92227 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sun, 4 Sep 2016 13:08:34 -0700 Subject: [PATCH 19/32] propagate callback error --- python/src/pynumbuf/adapters/numpy.cc | 2 +- python/src/pynumbuf/adapters/python.cc | 51 ++++++++++++-------------- python/src/pynumbuf/adapters/python.h | 2 - python/src/pynumbuf/numbuf.cc | 16 +++++++- 4 files changed, 38 insertions(+), 33 deletions(-) diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc index 92319f181..e3e1fa366 100644 --- a/python/src/pynumbuf/adapters/numpy.cc +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -117,7 +117,7 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); if (!result) { Py_XDECREF(arglist); - return python_error_to_status(); + return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/pcmoritz/numbuf/issues/10 } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 173767a1a..5b71865ff 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -11,56 +11,49 @@ extern PyObject* numbuf_deserialize_callback; namespace numbuf { -PyObject* get_value(ArrayPtr arr, int32_t index, int32_t type, PyObject* base) { - PyObject* result; +Status get_value(ArrayPtr arr, int32_t index, int32_t type, PyObject* base, PyObject** result) { switch (arr->type()->type) { case Type::BOOL: - return PyBool_FromLong(std::static_pointer_cast(arr)->Value(index)); + *result = PyBool_FromLong(std::static_pointer_cast(arr)->Value(index)); + return Status::OK(); case Type::INT64: - return PyInt_FromLong(std::static_pointer_cast(arr)->Value(index)); + *result = PyInt_FromLong(std::static_pointer_cast(arr)->Value(index)); + return Status::OK(); case Type::BINARY: { int32_t nchars; const uint8_t* str = std::static_pointer_cast(arr)->GetValue(index, &nchars); - return PyString_FromStringAndSize(reinterpret_cast(str), nchars); + *result = PyString_FromStringAndSize(reinterpret_cast(str), nchars); + return Status::OK(); } case Type::STRING: { int32_t nchars; const uint8_t* str = std::static_pointer_cast(arr)->GetValue(index, &nchars); - return PyUnicode_FromStringAndSize(reinterpret_cast(str), nchars); + *result = PyUnicode_FromStringAndSize(reinterpret_cast(str), nchars); + return Status::OK(); } case Type::FLOAT: - return PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + *result = PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + return Status::OK(); case Type::DOUBLE: - return PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + *result = PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + return Status::OK(); case Type::STRUCT: { auto s = std::static_pointer_cast(arr); auto l = std::static_pointer_cast(s->field(0)); if (s->type()->child(0)->name == "list") { - ARROW_CHECK_OK(DeserializeList(l->values(), l->value_offset(index), l->value_offset(index+1), base, &result)); + return DeserializeList(l->values(), l->value_offset(index), l->value_offset(index+1), base, result); } else if (s->type()->child(0)->name == "tuple") { - ARROW_CHECK_OK(DeserializeTuple(l->values(), l->value_offset(index), l->value_offset(index+1), base, &result)); + return DeserializeTuple(l->values(), l->value_offset(index), l->value_offset(index+1), base, result); } else if (s->type()->child(0)->name == "dict") { - ARROW_CHECK_OK(DeserializeDict(l->values(), l->value_offset(index), l->value_offset(index+1), base, &result)); + return DeserializeDict(l->values(), l->value_offset(index), l->value_offset(index+1), base, result); } else { - ARROW_CHECK_OK(DeserializeArray(arr, index, base, &result)); + return DeserializeArray(arr, index, base, result); } - return result; } default: DCHECK(false) << "union tag not recognized " << type; } - return NULL; -} - -Status python_error_to_status() { - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - char *err_message = PyString_AsString(value); - std::stringstream ss; - if (err_message) { - ss << "Python error in callback: " << err_message; - } - return Status::NotImplemented(ss.str()); + return Status::OK(); } Status append(PyObject* elem, SequenceBuilder& builder, @@ -123,7 +116,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); if (!result) { Py_XDECREF(arglist); - return python_error_to_status(); + return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/pcmoritz/numbuf/issues/10 } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); @@ -181,7 +174,9 @@ Status SerializeSequences(std::vector sequences, std::shared_ptrValue(i); \ int8_t type = types->Value(i); \ ArrayPtr arr = data->child(type); \ - SET_ITEM(result, i-start_idx, get_value(arr, offset, type, base)); \ + PyObject* value; \ + RETURN_NOT_OK(get_value(arr, offset, type, base, &value)); \ + SET_ITEM(result, i-start_idx, value); \ } \ } \ *out = result; \ @@ -245,7 +240,7 @@ Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t result = PyObject_CallObject(numbuf_deserialize_callback, arglist); if (!result) { Py_XDECREF(arglist); - return python_error_to_status(); + return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/pcmoritz/numbuf/issues/10 } Py_XDECREF(arglist); } diff --git a/python/src/pynumbuf/adapters/python.h b/python/src/pynumbuf/adapters/python.h index 13ee629d3..1f05e923f 100644 --- a/python/src/pynumbuf/adapters/python.h +++ b/python/src/pynumbuf/adapters/python.h @@ -17,8 +17,6 @@ arrow::Status DeserializeList(std::shared_ptr array, int32_t start arrow::Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); arrow::Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); -arrow::Status python_error_to_status(); - } #endif diff --git a/python/src/pynumbuf/numbuf.cc b/python/src/pynumbuf/numbuf.cc index d89913d6e..de8722a48 100644 --- a/python/src/pynumbuf/numbuf.cc +++ b/python/src/pynumbuf/numbuf.cc @@ -53,7 +53,11 @@ static PyObject* serialize_list(PyObject* self, PyObject* args) { if (PyList_Check(value)) { Status s = SerializeSequences(std::vector({value}), &array); if (!s.ok()) { - PyErr_SetString(NumbufError, s.ToString().c_str()); + // If this condition is true, there was an error in the callback that + // needs to be passed through + if (!PyErr_Occurred()) { + PyErr_SetString(NumbufError, s.ToString().c_str()); + } return NULL; } @@ -131,7 +135,15 @@ static PyObject* deserialize_list(PyObject* self, PyObject* args) { return NULL; } PyObject* result; - ARROW_CHECK_OK(DeserializeList((*data)->column(0), 0, (*data)->num_rows(), base, &result)); + Status s = DeserializeList((*data)->column(0), 0, (*data)->num_rows(), base, &result); + if (!s.ok()) { + // If this condition is true, there was an error in the callback that + // needs to be passed through + if (!PyErr_Occurred()) { + PyErr_SetString(NumbufError, s.ToString().c_str()); + } + return NULL; + } return result; } From 5ac2df4329d2dc3039a503151ab985067e28c733 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Tue, 6 Sep 2016 10:02:30 -0700 Subject: [PATCH 20/32] Don't increment reference count in Py_BuildValue. (#12) * Decrement reference count for result of serialization callback and also for an object created in DeserializeDict. * Check tuples exactly so named tuples go to callback. * Assert that callback is set if _pytype_ key present. --- python/src/pynumbuf/adapters/numpy.cc | 5 ++-- python/src/pynumbuf/adapters/python.cc | 32 ++++++++++++++++++++------ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc index e3e1fa366..3ae7170a3 100644 --- a/python/src/pynumbuf/adapters/numpy.cc +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -114,14 +114,15 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, return Status::NotImplemented(stream.str()); } else { PyObject* arglist = Py_BuildValue("(O)", array); + // The reference count of the result of the call to PyObject_CallObject + // must be decremented. This is done in SerializeDict in python.cc. PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); + Py_XDECREF(arglist); if (!result) { - Py_XDECREF(arglist); return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/pcmoritz/numbuf/issues/10 } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); - Py_XDECREF(arglist); } } Py_XDECREF(contiguous); diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 5b71865ff..7044b5c93 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -96,7 +96,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, } else if (PyDict_Check(elem)) { builder.AppendDict(PyDict_Size(elem)); subdicts.push_back(elem); - } else if (PyTuple_Check(elem)) { + } else if (PyTuple_CheckExact(elem)) { builder.AppendTuple(PyTuple_Size(elem)); subtuples.push_back(elem); } else if (PyArray_IsScalar(elem, Generic)) { @@ -113,14 +113,15 @@ Status append(PyObject* elem, SequenceBuilder& builder, return Status::NotImplemented(ss.str()); } else { PyObject* arglist = Py_BuildValue("(O)", elem); + // The reference count of the result of the call to PyObject_CallObject + // must be decremented. This is done in SerializeDict in this file. PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); + Py_XDECREF(arglist); if (!result) { - Py_XDECREF(arglist); return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/pcmoritz/numbuf/issues/10 } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); - Py_XDECREF(arglist); } } return Status::OK(); @@ -219,6 +220,20 @@ Status SerializeDict(std::vector dicts, std::shared_ptr* out) RETURN_NOT_OK(SerializeDict(val_dicts, &val_dict_arr)); } *out = result.Finish(key_tuples_arr, val_list_arr, val_tuples_arr, val_dict_arr); + + // This block is used to decrement the reference counts of the results + // returned by the serialization callback, which is called in SerializeArray + // in numpy.cc as well as in DeserializeDict and in append in this file. + static PyObject* py_type = PyString_FromString("_pytype_"); + for (const auto& dict : dicts) { + if (PyDict_Contains(dict, py_type)) { + // If the dictionary contains the key "_pytype_", then the user has to + // have registered a callback. + ARROW_CHECK(numbuf_serialize_callback); + Py_XDECREF(dict); + } + } + return Status::OK(); } @@ -237,12 +252,15 @@ Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t static PyObject* py_type = PyString_FromString("_pytype_"); if (PyDict_Contains(result, py_type) && numbuf_deserialize_callback) { PyObject* arglist = Py_BuildValue("(O)", result); - result = PyObject_CallObject(numbuf_deserialize_callback, arglist); - if (!result) { - Py_XDECREF(arglist); + // The result of the call to PyObject_CallObject will be passed to Python + // and its reference count will be decremented by the interpreter. + PyObject* callback_result = PyObject_CallObject(numbuf_deserialize_callback, arglist); + Py_XDECREF(arglist); + Py_XDECREF(result); + result = callback_result; + if (!callback_result) { return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/pcmoritz/numbuf/issues/10 } - Py_XDECREF(arglist); } *out = result; return Status::OK(); From d469366c75bd7364205c0eda205b7a32a1f24bf7 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Thu, 15 Sep 2016 15:11:51 -0700 Subject: [PATCH 21/32] Serialize Python floats properly as doubles. (#14) --- python/src/pynumbuf/adapters/python.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 7044b5c93..23ac894ac 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -64,7 +64,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, if (PyBool_Check(elem)) { RETURN_NOT_OK(builder.AppendBool(elem == Py_True)); } else if (PyFloat_Check(elem)) { - RETURN_NOT_OK(builder.AppendFloat(PyFloat_AS_DOUBLE(elem))); + RETURN_NOT_OK(builder.AppendDouble(PyFloat_AS_DOUBLE(elem))); } else if (PyLong_Check(elem)) { int overflow = 0; int64_t data = PyLong_AsLongLongAndOverflow(elem, &overflow); From 0527736490b2bd1d44ebfbbd17e298f5863d7dfc Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Sat, 17 Sep 2016 00:14:35 -0700 Subject: [PATCH 22/32] Migrate repository to ray-project. (#15) --- python/src/pynumbuf/adapters/numpy.cc | 2 +- python/src/pynumbuf/adapters/python.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/src/pynumbuf/adapters/numpy.cc b/python/src/pynumbuf/adapters/numpy.cc index 3ae7170a3..32901ffad 100644 --- a/python/src/pynumbuf/adapters/numpy.cc +++ b/python/src/pynumbuf/adapters/numpy.cc @@ -119,7 +119,7 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); Py_XDECREF(arglist); if (!result) { - return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/pcmoritz/numbuf/issues/10 + return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/ray-project/numbuf/issues/10 } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 23ac894ac..deafd9d44 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -118,7 +118,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); Py_XDECREF(arglist); if (!result) { - return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/pcmoritz/numbuf/issues/10 + return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/ray-project/numbuf/issues/10 } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); @@ -259,7 +259,7 @@ Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t Py_XDECREF(result); result = callback_result; if (!callback_result) { - return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/pcmoritz/numbuf/issues/10 + return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/ray-project/numbuf/issues/10 } } *out = result; From 7055c6f793f8b0aadb71cef9c81dce615e0cc77f Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Mon, 19 Sep 2016 14:54:56 -0700 Subject: [PATCH 23/32] Limit recursion depth for serializing objects to avoid infinite loops. (#17) --- python/src/pynumbuf/adapters/python.cc | 26 +++++++++++++++++--------- python/src/pynumbuf/adapters/python.h | 4 ++-- python/src/pynumbuf/numbuf.cc | 3 ++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index deafd9d44..87cfef8d7 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -6,6 +6,8 @@ using namespace arrow; +int32_t MAX_RECURSION_DEPTH = 100; + extern PyObject* numbuf_serialize_callback; extern PyObject* numbuf_deserialize_callback; @@ -127,8 +129,11 @@ Status append(PyObject* elem, SequenceBuilder& builder, return Status::OK(); } -Status SerializeSequences(std::vector sequences, std::shared_ptr* out) { +Status SerializeSequences(std::vector sequences, int32_t recursion_depth, std::shared_ptr* out) { DCHECK(out); + if (recursion_depth >= MAX_RECURSION_DEPTH) { + return Status::NotImplemented("This object exceeds the maximum recursion depth. It may contain itself recursively."); + } SequenceBuilder builder(nullptr); std::vector sublists, subtuples, subdicts; for (const auto& sequence : sequences) { @@ -147,15 +152,15 @@ Status SerializeSequences(std::vector sequences, std::shared_ptr list; if (sublists.size() > 0) { - RETURN_NOT_OK(SerializeSequences(sublists, &list)); + RETURN_NOT_OK(SerializeSequences(sublists, recursion_depth + 1, &list)); } std::shared_ptr tuple; if (subtuples.size() > 0) { - RETURN_NOT_OK(SerializeSequences(subtuples, &tuple)); + RETURN_NOT_OK(SerializeSequences(subtuples, recursion_depth + 1, &tuple)); } std::shared_ptr dict; if (subdicts.size() > 0) { - RETURN_NOT_OK(SerializeDict(subdicts, &dict)); + RETURN_NOT_OK(SerializeDict(subdicts, recursion_depth + 1, &dict)); } *out = builder.Finish(list, tuple, dict); return Status::OK(); @@ -191,8 +196,11 @@ Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SetItem) } -Status SerializeDict(std::vector dicts, std::shared_ptr* out) { +Status SerializeDict(std::vector dicts, int32_t recursion_depth, std::shared_ptr* out) { DictBuilder result; + if (recursion_depth >= MAX_RECURSION_DEPTH) { + return Status::NotImplemented("This object exceeds the maximum recursion depth. It may contain itself recursively."); + } std::vector key_tuples, val_lists, val_tuples, val_dicts, dummy; for (const auto& dict : dicts) { PyObject *key, *value; @@ -205,19 +213,19 @@ Status SerializeDict(std::vector dicts, std::shared_ptr* out) } std::shared_ptr key_tuples_arr; if (key_tuples.size() > 0) { - RETURN_NOT_OK(SerializeSequences(key_tuples, &key_tuples_arr)); + RETURN_NOT_OK(SerializeSequences(key_tuples, recursion_depth + 1, &key_tuples_arr)); } std::shared_ptr val_list_arr; if (val_lists.size() > 0) { - RETURN_NOT_OK(SerializeSequences(val_lists, &val_list_arr)); + RETURN_NOT_OK(SerializeSequences(val_lists, recursion_depth + 1, &val_list_arr)); } std::shared_ptr val_tuples_arr; if (val_tuples.size() > 0) { - RETURN_NOT_OK(SerializeSequences(val_tuples, &val_tuples_arr)); + RETURN_NOT_OK(SerializeSequences(val_tuples, recursion_depth + 1, &val_tuples_arr)); } std::shared_ptr val_dict_arr; if (val_dicts.size() > 0) { - RETURN_NOT_OK(SerializeDict(val_dicts, &val_dict_arr)); + RETURN_NOT_OK(SerializeDict(val_dicts, recursion_depth + 1, &val_dict_arr)); } *out = result.Finish(key_tuples_arr, val_list_arr, val_tuples_arr, val_dict_arr); diff --git a/python/src/pynumbuf/adapters/python.h b/python/src/pynumbuf/adapters/python.h index 1f05e923f..024ebebbf 100644 --- a/python/src/pynumbuf/adapters/python.h +++ b/python/src/pynumbuf/adapters/python.h @@ -11,8 +11,8 @@ namespace numbuf { -arrow::Status SerializeSequences(std::vector sequences, std::shared_ptr* out); -arrow::Status SerializeDict(std::vector dicts, std::shared_ptr* out); +arrow::Status SerializeSequences(std::vector sequences, int32_t recursion_depth, std::shared_ptr* out); +arrow::Status SerializeDict(std::vector dicts, int32_t recursion_depth, std::shared_ptr* out); arrow::Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); arrow::Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); arrow::Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); diff --git a/python/src/pynumbuf/numbuf.cc b/python/src/pynumbuf/numbuf.cc index de8722a48..8951b07be 100644 --- a/python/src/pynumbuf/numbuf.cc +++ b/python/src/pynumbuf/numbuf.cc @@ -51,7 +51,8 @@ static PyObject* serialize_list(PyObject* self, PyObject* args) { } std::shared_ptr array; if (PyList_Check(value)) { - Status s = SerializeSequences(std::vector({value}), &array); + int32_t recursion_depth = 0; + Status s = SerializeSequences(std::vector({value}), recursion_depth, &array); if (!s.ok()) { // If this condition is true, there was an error in the callback that // needs to be passed through From 4d2a77901a57177720bb1480f0114a268083f8ed Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 6 Oct 2016 00:58:27 -0700 Subject: [PATCH 24/32] Fixed extern C declaration (found by Mehrdad) --- python/src/pynumbuf/adapters/python.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 87cfef8d7..fd425a861 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -8,9 +8,13 @@ using namespace arrow; int32_t MAX_RECURSION_DEPTH = 100; +extern "C" { + extern PyObject* numbuf_serialize_callback; extern PyObject* numbuf_deserialize_callback; +} + namespace numbuf { Status get_value(ArrayPtr arr, int32_t index, int32_t type, PyObject* base, PyObject** result) { From d1974afbab9f0f1bcf8af15a8c476d868ad31aff Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Thu, 27 Oct 2016 23:40:10 -0700 Subject: [PATCH 25/32] Add setup.py and travis. (#8) * Add setup.py and travis. * Add script to install dependencies for travis. * Put Arrow into thirdparty. * Add .gitmodules file. * Install boost dependencies. * Make tests verbose. * Build arrow and numbuf in setup.py. * Change build_ext to install. * Switch develop to install in pip. * Test * fix numbuf build and installation * make import numbuf work * fix documentation * fix tests * quotes --- .gitmodules | 3 +++ .travis.yml | 25 ++++++++++++++++++++ CMakeLists.txt | 6 +++-- build.sh | 20 ++++++++++++++++ install-dependencies.sh | 39 +++++++++++++++++++++++++++++++ numbuf/__init__.py | 1 + python/test/runtest.py | 32 ++++++++++++------------- setup.py | 23 ++++++++++++++++++ setup.sh | 21 +++++++++++++++++ thirdparty/arrow | 1 + thirdparty/build_thirdparty.sh | 27 +++++++++++++++++++++ thirdparty/download_thirdparty.sh | 8 +++++++ 12 files changed, 188 insertions(+), 18 deletions(-) create mode 100644 .gitmodules create mode 100644 .travis.yml create mode 100755 build.sh create mode 100755 install-dependencies.sh create mode 100644 numbuf/__init__.py create mode 100644 setup.py create mode 100755 setup.sh create mode 160000 thirdparty/arrow create mode 100755 thirdparty/build_thirdparty.sh create mode 100755 thirdparty/download_thirdparty.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..89d94b72e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "thirdparty/arrow"] + path = thirdparty/arrow + url = https://github.com/ray-project/arrow.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..fe784d053 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,25 @@ +sudo: required + +language: generic + +matrix: + include: + - os: linux + dist: trusty + python: "2.7" + - os: linux + dist: trusty + python: "3.5" + - os: osx + osx_image: xcode7 + python: "2.7" + - os: osx + osx_image: xcode7 + python: "3.5" + +install: + - ./install-dependencies.sh + - python setup.py install --user + +script: + - python python/test/runtest.py diff --git a/CMakeLists.txt b/CMakeLists.txt index a46830891..760bdde11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,10 +50,10 @@ if (UNIX AND NOT APPLE) link_libraries(rt) endif() -set(ARROW_DIR "${CMAKE_SOURCE_DIR}/../arrow/" CACHE STRING +set(ARROW_DIR "${CMAKE_SOURCE_DIR}/thirdparty/arrow/" CACHE STRING "Path of the arrow source directory") -set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/../arrow/cpp/build/release/libarrow.a" CACHE STRING +set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a" CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)") include_directories("${ARROW_DIR}/cpp/src/") @@ -78,3 +78,5 @@ if(APPLE) endif(APPLE) target_link_libraries(numbuf ${ARROW_STATIC_LIB} ${PYTHON_LIBRARIES}) + +install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}) diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..33a1355c8 --- /dev/null +++ b/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +# Determine how many parallel jobs to use for make based on the number of cores +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + PARALLEL=$(nproc) +elif [[ "$unamestr" == "Darwin" ]]; then + PARALLEL=$(sysctl -n hw.ncpu) +else + echo "Unrecognized platform." + exit 1 +fi + +mkdir -p "$ROOT_DIR/build" +pushd "$ROOT_DIR/build" + cmake .. + make install -j$PARALLEL +popd diff --git a/install-dependencies.sh b/install-dependencies.sh new file mode 100755 index 000000000..8d7de5288 --- /dev/null +++ b/install-dependencies.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +platform="unknown" +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + echo "Platform is linux." + platform="linux" +elif [[ "$unamestr" == "Darwin" ]]; then + echo "Platform is macosx." + platform="macosx" +else + echo "Unrecognized platform." + exit 1 +fi + +if [[ $platform == "macosx" ]]; then + # check that brew is installed + which -s brew + if [[ $? != 0 ]]; then + echo "Could not find brew, please install brew (see http://brew.sh/)." + exit 1 + else + echo "Updating brew." + brew update + fi +fi + +if [[ $platform == "linux" ]]; then + # These commands must be kept in sync with the installation instructions. + sudo apt-get update + sudo apt-get install -y cmake build-essential autoconf libtool python-dev python-numpy python-pip libboost-all-dev +elif [[ $platform == "macosx" ]]; then + # These commands must be kept in sync with the installation instructions. + brew install cmake automake autoconf libtool boost + sudo easy_install pip + sudo pip install numpy --ignore-installed six +fi diff --git a/numbuf/__init__.py b/numbuf/__init__.py new file mode 100644 index 000000000..21cb259ba --- /dev/null +++ b/numbuf/__init__.py @@ -0,0 +1 @@ +from libnumbuf import * diff --git a/python/test/runtest.py b/python/test/runtest.py index c1b7d21e8..4c9b409f6 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -1,10 +1,10 @@ import unittest -import libnumbuf +import numbuf import numpy as np from numpy.testing import assert_equal -TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world", - u"x", u"\u262F", 42.0, +TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world", + u"x", u"\u262F", 42.0, 1L << 62, (1.0, "hi"), None, (None, None), ("hello", None), True, False, (True, False), "hello", @@ -17,8 +17,8 @@ TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world" class SerializationTests(unittest.TestCase): def roundTripTest(self, data): - schema, size, serialized = libnumbuf.serialize_list(data) - result = libnumbuf.deserialize_list(serialized) + schema, size, serialized = numbuf.serialize_list(data) + result = numbuf.deserialize_list(serialized) assert_equal(data, result) def testSimple(self): @@ -78,10 +78,10 @@ class SerializationTests(unittest.TestCase): bar = Bar() bar.foo.x = 42 - libnumbuf.register_callbacks(serialize, deserialize) + numbuf.register_callbacks(serialize, deserialize) - metadata, size, serialized = libnumbuf.serialize_list([bar]) - self.assertEqual(libnumbuf.deserialize_list(serialized)[0].foo.x, 42) + metadata, size, serialized = numbuf.serialize_list([bar]) + self.assertEqual(numbuf.deserialize_list(serialized)[0].foo.x, 42) def testObjectArray(self): x = np.array([1, 2, "hello"], dtype=object) @@ -94,21 +94,21 @@ class SerializationTests(unittest.TestCase): if obj["_pytype_"] == "numpy.array": return np.array(obj["data"], dtype=object) - libnumbuf.register_callbacks(myserialize, mydeserialize) + numbuf.register_callbacks(myserialize, mydeserialize) - metadata, size, serialized = libnumbuf.serialize_list([x, y]) + metadata, size, serialized = numbuf.serialize_list([x, y]) - assert_equal(libnumbuf.deserialize_list(serialized), [x, y]) + assert_equal(numbuf.deserialize_list(serialized), [x, y]) def testBuffer(self): for (i, obj) in enumerate(TEST_OBJECTS): - schema, size, batch = libnumbuf.serialize_list([obj]) + schema, size, batch = numbuf.serialize_list([obj]) size = size + 4096 # INITIAL_METADATA_SIZE in arrow buff = np.zeros(size, dtype="uint8") - metadata_offset = libnumbuf.write_to_buffer(batch, memoryview(buff)) - array = libnumbuf.read_from_buffer(memoryview(buff), schema, metadata_offset) - result = libnumbuf.deserialize_list(array) + metadata_offset = numbuf.write_to_buffer(batch, memoryview(buff)) + array = numbuf.read_from_buffer(memoryview(buff), schema, metadata_offset) + result = numbuf.deserialize_list(array) assert_equal(result[0], obj) if __name__ == "__main__": - unittest.main() + unittest.main(verbosity=2) diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..27b82a9de --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +import subprocess +from setuptools import setup, find_packages, Extension +import setuptools.command.install as _install + +# Because of relative paths, this must be run from inside numbuf/. + +class install(_install.install): + def run(self): + subprocess.check_call(["./setup.sh"]) + subprocess.check_call(["./build.sh"]) + subprocess.check_call(["cp", "libnumbuf.so", "numbuf/"]) + # Calling _install.install.run(self) does not fetch required packages and + # instead performs an old-style install. See command/install.py in + # setuptools. So, calling do_egg_install() manually here. + self.do_egg_install() + +setup(name="numbuf", + version="0.0.1", + packages=find_packages(), + package_data={"numbuf": ["libnumbuf.so"]}, + cmdclass={"install": install}, + include_package_data=True, + zip_safe=False) diff --git a/setup.sh b/setup.sh new file mode 100755 index 000000000..71482975e --- /dev/null +++ b/setup.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +platform="unknown" +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + echo "Platform is linux." + platform="linux" +elif [[ "$unamestr" == "Darwin" ]]; then + echo "Platform is macosx." + platform="macosx" +else + echo "Unrecognized platform." + exit 1 +fi + +pushd "$ROOT_DIR/thirdparty" + ./download_thirdparty.sh + ./build_thirdparty.sh +popd diff --git a/thirdparty/arrow b/thirdparty/arrow new file mode 160000 index 000000000..c781ef856 --- /dev/null +++ b/thirdparty/arrow @@ -0,0 +1 @@ +Subproject commit c781ef8564714bb5149e8b5a311673f9162232ac diff --git a/thirdparty/build_thirdparty.sh b/thirdparty/build_thirdparty.sh new file mode 100755 index 000000000..7c307f816 --- /dev/null +++ b/thirdparty/build_thirdparty.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -x +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +PREFIX=$TP_DIR/installed + +# Determine how many parallel jobs to use for make based on the number of cores +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + PARALLEL=$(nproc) +elif [[ "$unamestr" == "Darwin" ]]; then + PARALLEL=$(sysctl -n hw.ncpu) + echo "Platform is macosx." +else + echo "Unrecognized platform." + exit 1 +fi + +echo "building arrow" +cd $TP_DIR/arrow/cpp +source setup_build_env.sh +mkdir -p $TP_DIR/arrow/cpp/build +cd $TP_DIR/arrow/cpp/build +cmake -DLIBARROW_LINKAGE=STATIC -DCMAKE_BUILD_TYPE=Release .. +make VERBOSE=1 -j$PARALLEL diff --git a/thirdparty/download_thirdparty.sh b/thirdparty/download_thirdparty.sh new file mode 100755 index 000000000..044565252 --- /dev/null +++ b/thirdparty/download_thirdparty.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -x +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +git submodule update --init --recursive -- "$TP_DIR/arrow" From d26fbcebbe30c75e2a0db34a418e2c6f0e64ef1e Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sat, 12 Nov 2016 19:35:46 -0800 Subject: [PATCH 26/32] switch to release mode (#19) --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 33a1355c8..d82905fb5 100755 --- a/build.sh +++ b/build.sh @@ -15,6 +15,6 @@ fi mkdir -p "$ROOT_DIR/build" pushd "$ROOT_DIR/build" - cmake .. + cmake -DCMAKE_BUILD_TYPE=Release .. make install -j$PARALLEL popd From 488f881d708bc54e86ed375ee97aa94540808fa1 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Sat, 12 Nov 2016 19:36:47 -0800 Subject: [PATCH 27/32] Install numpy before building numbuf in setup.py. (#20) --- install-dependencies.sh | 13 +++++-------- setup.py | 1 + 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/install-dependencies.sh b/install-dependencies.sh index 8d7de5288..88973f6de 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -15,7 +15,11 @@ else exit 1 fi -if [[ $platform == "macosx" ]]; then +if [[ $platform == "linux" ]]; then + # These commands must be kept in sync with the installation instructions. + sudo apt-get update + sudo apt-get install -y cmake build-essential autoconf libtool python-dev python-numpy libboost-all-dev +elif [[ $platform == "macosx" ]]; then # check that brew is installed which -s brew if [[ $? != 0 ]]; then @@ -25,13 +29,6 @@ if [[ $platform == "macosx" ]]; then echo "Updating brew." brew update fi -fi - -if [[ $platform == "linux" ]]; then - # These commands must be kept in sync with the installation instructions. - sudo apt-get update - sudo apt-get install -y cmake build-essential autoconf libtool python-dev python-numpy python-pip libboost-all-dev -elif [[ $platform == "macosx" ]]; then # These commands must be kept in sync with the installation instructions. brew install cmake automake autoconf libtool boost sudo easy_install pip diff --git a/setup.py b/setup.py index 27b82a9de..ce9a51bc7 100644 --- a/setup.py +++ b/setup.py @@ -19,5 +19,6 @@ setup(name="numbuf", packages=find_packages(), package_data={"numbuf": ["libnumbuf.so"]}, cmdclass={"install": install}, + setup_requires=["numpy"], include_package_data=True, zip_safe=False) From c3db225cbe2a792c2dbfbe707107f71eec73d211 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Wed, 16 Nov 2016 19:43:26 -0800 Subject: [PATCH 28/32] Patch arrow to work with xcode 8. This will be unecessary when we rebase on an updated arrow. (#22) --- thirdparty/arrow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/arrow b/thirdparty/arrow index c781ef856..1407126ec 160000 --- a/thirdparty/arrow +++ b/thirdparty/arrow @@ -1 +1 @@ -Subproject commit c781ef8564714bb5149e8b5a311673f9162232ac +Subproject commit 1407126ec0956b6dc6f9873bdeeef4d042ca032f From c3ab68e88ccbd028ac40b3624f143448cd79b399 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sat, 19 Nov 2016 02:27:21 -0800 Subject: [PATCH 29/32] Rebase numbuf to latest arrow (#23) * rebase to latest arrow * fix arrow linking * finish rebase * point arrow to pcmoritz's arrow * fix * fix macOS * fix --- .gitmodules | 2 +- CMakeLists.txt | 39 ++++++++++++++++-- build.sh | 2 +- cpp/src/numbuf/dict.cc | 13 +++--- cpp/src/numbuf/dict.h | 5 ++- cpp/src/numbuf/sequence.cc | 38 ++++++++++------- cpp/src/numbuf/sequence.h | 5 ++- cpp/src/numbuf/tensor.cc | 23 +++++++---- cpp/src/numbuf/tensor.h | 5 ++- python/src/pynumbuf/adapters/python.cc | 7 ++-- python/src/pynumbuf/memory.h | 57 ++++++++++++++++++-------- python/src/pynumbuf/numbuf.cc | 38 ++++++++--------- setup.py | 10 ++++- thirdparty/arrow | 2 +- thirdparty/build_thirdparty.sh | 3 +- 15 files changed, 163 insertions(+), 86 deletions(-) diff --git a/.gitmodules b/.gitmodules index 89d94b72e..29d35a973 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "thirdparty/arrow"] path = thirdparty/arrow - url = https://github.com/ray-project/arrow.git + url = https://github.com/pcmoritz/arrow.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 760bdde11..a8d400ee6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,10 @@ project(numbuf) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) +# Make libnumbuf.so look for shared libraries in the folder libnumbuf.so is in +set(CMAKE_INSTALL_RPATH "$ORIGIN/") +set(CMAKE_MACOSX_RPATH 1) + if(NOT APPLE) find_package(PythonInterp REQUIRED) find_package(PythonLibs REQUIRED) @@ -53,8 +57,25 @@ endif() set(ARROW_DIR "${CMAKE_SOURCE_DIR}/thirdparty/arrow/" CACHE STRING "Path of the arrow source directory") -set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a" CACHE STRING - "Path to libarrow.a (needs to be changed if arrow is build in debug mode)") +if (APPLE) + set(ARROW_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.dylib" CACHE STRING + "Path to libarrow.dylib (needs to be changed if arrow is build in debug mode)") + + set(ARROW_IO_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_io.dylib" CACHE STRING + "Path to libarrow_io.dylib (needs to be changed if arrow is build in debug mode)") + + set(ARROW_IPC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_ipc.dylib" CACHE STRING + "Path to libarrow_ipc.dylib (needs to be changed if arrow is build in debug mode)") +else() + set(ARROW_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.so" CACHE STRING + "Path to libarrow.so (needs to be changed if arrow is build in debug mode)") + + set(ARROW_IO_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_io.so" CACHE STRING + "Path to libarrow_io.so (needs to be changed if arrow is build in debug mode)") + + set(ARROW_IPC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_ipc.so" CACHE STRING + "Path to libarrow_ipc.so (needs to be changed if arrow is build in debug mode)") +endif() include_directories("${ARROW_DIR}/cpp/src/") include_directories("cpp/src/") @@ -75,8 +96,18 @@ if(APPLE) add_custom_command(TARGET numbuf POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libnumbuf.so) + add_custom_command(TARGET numbuf + POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -change "@rpath/libarrow.dylib" "@loader_path/libarrow.dylib" libnumbuf.so) + add_custom_command(TARGET numbuf + POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -change "@rpath/libarrow_io.dylib" "@loader_path/libarrow_io.dylib" libnumbuf.so) + add_custom_command(TARGET numbuf + POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -change "@rpath/libarrow_ipc.dylib" "@loader_path/libarrow_ipc.dylib" libnumbuf.so) endif(APPLE) -target_link_libraries(numbuf ${ARROW_STATIC_LIB} ${PYTHON_LIBRARIES}) +target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES}) -install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}) +install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/) +install(FILES ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/) diff --git a/build.sh b/build.sh index d82905fb5..087c1471f 100755 --- a/build.sh +++ b/build.sh @@ -15,6 +15,6 @@ fi mkdir -p "$ROOT_DIR/build" pushd "$ROOT_DIR/build" - cmake -DCMAKE_BUILD_TYPE=Release .. + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g" -DCMAKE_CXX_FLAGS="-g" .. make install -j$PARALLEL popd diff --git a/cpp/src/numbuf/dict.cc b/cpp/src/numbuf/dict.cc index 98b03894b..74a9cfadf 100644 --- a/cpp/src/numbuf/dict.cc +++ b/cpp/src/numbuf/dict.cc @@ -4,21 +4,24 @@ using namespace arrow; namespace numbuf { -std::shared_ptr DictBuilder::Finish( +Status DictBuilder::Finish( std::shared_ptr key_tuple_data, std::shared_ptr val_list_data, std::shared_ptr val_tuple_data, - std::shared_ptr val_dict_data) { + std::shared_ptr val_dict_data, + std::shared_ptr* out) { // lists and dicts can't be keys of dicts in Python, that is why for // the keys we do not need to collect sublists - auto keys = keys_.Finish(nullptr, key_tuple_data, nullptr); - auto vals = vals_.Finish(val_list_data, val_tuple_data, val_dict_data); + std::shared_ptr keys, vals; + RETURN_NOT_OK(keys_.Finish(nullptr, key_tuple_data, nullptr, &keys)); + RETURN_NOT_OK(vals_.Finish(val_list_data, val_tuple_data, val_dict_data, &vals)); auto keys_field = std::make_shared("keys", keys->type()); auto vals_field = std::make_shared("vals", vals->type()); auto type = std::make_shared(std::vector({keys_field, vals_field})); std::vector field_arrays({keys, vals}); DCHECK(keys->length() == vals->length()); - return std::make_shared(type, keys->length(), field_arrays); + out->reset(new StructArray(type, keys->length(), field_arrays)); + return Status::OK(); } } diff --git a/cpp/src/numbuf/dict.h b/cpp/src/numbuf/dict.h index dcb38fb6b..6497b3eb7 100644 --- a/cpp/src/numbuf/dict.h +++ b/cpp/src/numbuf/dict.h @@ -33,11 +33,12 @@ public: List containing the data from nested dictionaries in the value list of the dictionary */ - std::shared_ptr Finish( + arrow::Status Finish( std::shared_ptr key_tuple_data, std::shared_ptr val_list_data, std::shared_ptr val_tuple_data, - std::shared_ptr val_dict_data); + std::shared_ptr val_dict_data, + std::shared_ptr* out); private: SequenceBuilder keys_; diff --git a/cpp/src/numbuf/sequence.cc b/cpp/src/numbuf/sequence.cc index a5e1cf8ca..9672f5e5d 100644 --- a/cpp/src/numbuf/sequence.cc +++ b/cpp/src/numbuf/sequence.cc @@ -5,13 +5,16 @@ using namespace arrow; namespace numbuf { SequenceBuilder::SequenceBuilder(MemoryPool* pool) - : pool_(pool), types_(pool), offsets_(pool), + : pool_(pool), + types_(pool, std::make_shared()), + offsets_(pool, std::make_shared()), nones_(pool, std::make_shared()), bools_(pool, std::make_shared()), - ints_(pool), + ints_(pool, std::make_shared()), bytes_(pool, std::make_shared()), strings_(pool, std::make_shared()), - floats_(pool), doubles_(pool), + floats_(pool, std::make_shared()), + doubles_(pool, std::make_shared()), uint8_tensors_(std::make_shared(), pool), int8_tensors_(std::make_shared(), pool), uint16_tensors_(std::make_shared(), pool), @@ -76,6 +79,9 @@ Status SequenceBuilder::AppendDouble(double data) { #define DEF_TENSOR_APPEND(NAME, TYPE, TAG) \ Status SequenceBuilder::AppendTensor(const std::vector& dims, TYPE* data) { \ + if (TAG == -1) { \ + NAME.Start(); \ + } \ UPDATE(NAME.length(), TAG); \ return NAME.Append(dims, data); \ } @@ -109,11 +115,11 @@ Status SequenceBuilder::AppendDict(int32_t size) { return Status::OK(); } -#define ADD_ELEMENT(VARNAME, TAG) \ - if (TAG != -1) { \ - types[TAG] = VARNAME.type(); \ - children[TAG] = VARNAME.Finish(); \ - ARROW_CHECK_OK(nones_.AppendToBitmap(true)); \ +#define ADD_ELEMENT(VARNAME, TAG) \ + if (TAG != -1) { \ + types[TAG] = std::make_shared("", VARNAME.type()); \ + RETURN_NOT_OK(VARNAME.Finish(&children[TAG])); \ + RETURN_NOT_OK(nones_.AppendToBitmap(true)); \ } #define ADD_SUBSEQUENCE(DATA, OFFSETS, BUILDER, TAG, NAME) \ @@ -132,12 +138,13 @@ Status SequenceBuilder::AppendDict(int32_t size) { DCHECK(OFFSETS.size() == 1); \ } -std::shared_ptr SequenceBuilder::Finish( +Status SequenceBuilder::Finish( std::shared_ptr list_data, std::shared_ptr tuple_data, - std::shared_ptr dict_data) { + std::shared_ptr dict_data, + std::shared_ptr* out) { - std::vector types(num_tags); + std::vector> types(num_tags); std::vector children(num_tags); ADD_ELEMENT(bools_, bool_tag); @@ -165,11 +172,12 @@ std::shared_ptr SequenceBuilder::Finish( ADD_SUBSEQUENCE(tuple_data, tuple_offsets_, tuple_builder, tuple_tag, "tuple"); ADD_SUBSEQUENCE(dict_data, dict_offsets_, dict_builder, dict_tag, "dict"); - TypePtr type = TypePtr(new DenseUnionType(types)); - - return std::make_shared(type, types_.length(), + std::vector type_ids = {}; + TypePtr type = TypePtr(new UnionType(types, type_ids, UnionMode::DENSE)); + out->reset(new UnionArray(type, types_.length(), children, types_.data(), offsets_.data(), - nones_.null_count(), nones_.null_bitmap()); + nones_.null_count(), nones_.null_bitmap())); + return Status::OK(); } } diff --git a/cpp/src/numbuf/sequence.h b/cpp/src/numbuf/sequence.h index 7237dbd83..bcb1c1afd 100644 --- a/cpp/src/numbuf/sequence.h +++ b/cpp/src/numbuf/sequence.h @@ -79,10 +79,11 @@ class SequenceBuilder { arrow::Status AppendDict(int32_t size); //! Finish building the sequence and return the result - std::shared_ptr Finish( + arrow::Status Finish( std::shared_ptr list_data, std::shared_ptr tuple_data, - std::shared_ptr dict_data); + std::shared_ptr dict_data, + std::shared_ptr* out); private: arrow::MemoryPool* pool_; diff --git a/cpp/src/numbuf/tensor.cc b/cpp/src/numbuf/tensor.cc index f0f7647de..5d95a94a7 100644 --- a/cpp/src/numbuf/tensor.cc +++ b/cpp/src/numbuf/tensor.cc @@ -6,19 +6,24 @@ namespace numbuf { template TensorBuilder::TensorBuilder(const TypePtr& dtype, MemoryPool* pool) - : dtype_(dtype) { - dim_data_ = std::make_shared(pool); - dims_ = std::make_shared(pool, dim_data_); - value_data_ = std::make_shared>(pool, dtype); - values_ = std::make_shared(pool, value_data_); + : dtype_(dtype), pool_(pool) {} + +template +Status TensorBuilder::Start() { + dim_data_ = std::make_shared(pool_, std::make_shared()); + dims_ = std::make_shared(pool_, dim_data_); + value_data_ = std::make_shared>(pool_, dtype_); + values_ = std::make_shared(pool_, value_data_); auto dims_field = std::make_shared("dims", dims_->type()); auto values_field = std::make_shared("data", values_->type()); auto type = std::make_shared(std::vector({dims_field, values_field})); - tensors_ = std::make_shared(pool, type, std::vector>({dims_, values_})); -}; + tensors_ = std::make_shared(pool_, type, std::vector>({dims_, values_})); + return Status::OK(); +} template Status TensorBuilder::Append(const std::vector& dims, const elem_type* data) { + DCHECK(tensors_); RETURN_NOT_OK(tensors_->Append()); RETURN_NOT_OK(dims_->Append()); RETURN_NOT_OK(values_->Append()); @@ -32,8 +37,8 @@ Status TensorBuilder::Append(const std::vector& dims, const elem_typ } template -std::shared_ptr TensorBuilder::Finish() { - return tensors_->Finish(); +Status TensorBuilder::Finish(std::shared_ptr* out) { + return tensors_->Finish(out); } template class TensorBuilder; diff --git a/cpp/src/numbuf/tensor.h b/cpp/src/numbuf/tensor.h index 351892758..e3725fcc9 100644 --- a/cpp/src/numbuf/tensor.h +++ b/cpp/src/numbuf/tensor.h @@ -18,6 +18,8 @@ public: typedef typename T::c_type elem_type; TensorBuilder(const arrow::TypePtr& dtype, arrow::MemoryPool* pool = nullptr); + + arrow::Status Start(); /*! Append a new tensor. @@ -31,7 +33,7 @@ public: arrow::Status Append(const std::vector& dims, const elem_type* data); //! Convert the tensors to an Arrow StructArray - std::shared_ptr Finish(); + arrow::Status Finish(std::shared_ptr* out); //! Number of tensors in the column int32_t length() { @@ -44,6 +46,7 @@ public: private: arrow::TypePtr dtype_; + arrow::MemoryPool* pool_; std::shared_ptr dim_data_; std::shared_ptr dims_; std::shared_ptr> value_data_; diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index fd425a861..2d41cbaa3 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -166,12 +166,11 @@ Status SerializeSequences(std::vector sequences, int32_t recursion_de if (subdicts.size() > 0) { RETURN_NOT_OK(SerializeDict(subdicts, recursion_depth + 1, &dict)); } - *out = builder.Finish(list, tuple, dict); - return Status::OK(); + return builder.Finish(list, tuple, dict, out); } #define DESERIALIZE_SEQUENCE(CREATE, SET_ITEM) \ - auto data = std::dynamic_pointer_cast(array); \ + auto data = std::dynamic_pointer_cast(array); \ int32_t size = array->length(); \ PyObject* result = CREATE(stop_idx - start_idx); \ auto types = std::make_shared(size, data->types()); \ @@ -231,7 +230,7 @@ Status SerializeDict(std::vector dicts, int32_t recursion_depth, std: if (val_dicts.size() > 0) { RETURN_NOT_OK(SerializeDict(val_dicts, recursion_depth + 1, &val_dict_arr)); } - *out = result.Finish(key_tuples_arr, val_list_arr, val_tuples_arr, val_dict_arr); + result.Finish(key_tuples_arr, val_list_arr, val_tuples_arr, val_dict_arr, out); // This block is used to decrement the reference counts of the results // returned by the serialization callback, which is called in SerializeArray diff --git a/python/src/pynumbuf/memory.h b/python/src/pynumbuf/memory.h index ebaa22c51..eb926dbcb 100644 --- a/python/src/pynumbuf/memory.h +++ b/python/src/pynumbuf/memory.h @@ -1,22 +1,32 @@ #ifndef PYNUMBUF_MEMORY_H #define PYNUMBUF_MEMORY_H -#include +#include namespace numbuf { -class BufferSource : public arrow::ipc::MemorySource { +class FixedBufferStream : public arrow::io::OutputStream, public arrow::io::ReadableFileInterface { public: - virtual ~BufferSource() {} + virtual ~FixedBufferStream() {} - explicit BufferSource(uint8_t* data, int64_t nbytes) - : data_(data), size_(nbytes) {} + explicit FixedBufferStream(uint8_t* data, int64_t nbytes) + : data_(data), position_(0), size_(nbytes) {} - arrow::Status ReadAt(int64_t position, int64_t nbytes, - std::shared_ptr* out) override { + arrow::Status Read(int64_t nbytes, std::shared_ptr* out) override { DCHECK(out); - DCHECK(position + nbytes <= size_) << "position: " << position << " nbytes: " << nbytes << "size: " << size_; - *out = std::make_shared(data_ + position, nbytes); + DCHECK(position_ + nbytes <= size_) << "position: " << position_ << " nbytes: " << nbytes << "size: " << size_; + *out = std::make_shared(data_ + position_, nbytes); + position_ += nbytes; + return arrow::Status::OK(); + } + + arrow::Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) { + assert(0); + return arrow::Status::OK(); + } + + arrow::Status Seek(int64_t position) override { + position_ = position; return arrow::Status::OK(); } @@ -24,24 +34,35 @@ class BufferSource : public arrow::ipc::MemorySource { return arrow::Status::OK(); } - arrow::Status Write(int64_t position, const uint8_t* data, - int64_t nbytes) override { - DCHECK(position >= 0 && position < size_); - DCHECK(position + nbytes <= size_) << "position: " << position << " nbytes: " << nbytes << "size: " << size_; - uint8_t* dst = data_ + position; - memcpy(dst, data, nbytes); + arrow::Status Tell(int64_t* position) override { + *position = position_; return arrow::Status::OK(); } - int64_t Size() const override { - return size_; + arrow::Status Write(const uint8_t* data, int64_t nbytes) override { + DCHECK(position_ >= 0 && position_ < size_); + DCHECK(position_ + nbytes <= size_) << "position: " << position_ << " nbytes: " << nbytes << "size: " << size_; + uint8_t* dst = data_ + position_; + memcpy(dst, data, nbytes); + position_ += nbytes; + return arrow::Status::OK(); + } + + arrow::Status GetSize(int64_t *size) override { + *size = size_; + return arrow::Status::OK(); + } + + bool supports_zero_copy() const override { + return true; } private: uint8_t* data_; + int64_t position_; int64_t size_; }; -} // namespace +} // namespace numbuf #endif // PYNUMBUF_MEMORY_H diff --git a/python/src/pynumbuf/numbuf.cc b/python/src/pynumbuf/numbuf.cc index 8951b07be..ba2483810 100644 --- a/python/src/pynumbuf/numbuf.cc +++ b/python/src/pynumbuf/numbuf.cc @@ -1,6 +1,5 @@ #include #include -#include #include #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API @@ -16,10 +15,10 @@ using namespace arrow; using namespace numbuf; -std::shared_ptr make_row_batch(std::shared_ptr data) { +std::shared_ptr make_row_batch(std::shared_ptr data) { auto field = std::make_shared("list", data->type()); std::shared_ptr schema(new Schema({field})); - return std::shared_ptr(new RowBatch(schema, data->length(), {data})); + return std::shared_ptr(new RecordBatch(schema, data->length(), {data})); } extern "C" { @@ -29,9 +28,9 @@ static PyObject *NumbufError; PyObject *numbuf_serialize_callback = NULL; PyObject *numbuf_deserialize_callback = NULL; -int PyObjectToArrow(PyObject* object, std::shared_ptr **result) { +int PyObjectToArrow(PyObject* object, std::shared_ptr **result) { if (PyCapsule_IsValid(object, "arrow")) { - *result = reinterpret_cast*>(PyCapsule_GetPointer(object, "arrow")); + *result = reinterpret_cast*>(PyCapsule_GetPointer(object, "arrow")); return 1; } else { PyErr_SetString(PyExc_TypeError, "must be an 'arrow' capsule"); @@ -40,7 +39,7 @@ int PyObjectToArrow(PyObject* object, std::shared_ptr **result) { } static void ArrowCapsule_Destructor(PyObject* capsule) { - delete reinterpret_cast*>(PyCapsule_GetPointer(capsule, "arrow")); + delete reinterpret_cast*>(PyCapsule_GetPointer(capsule, "arrow")); } /* Documented in doc/numbuf.rst in ray-core */ @@ -62,11 +61,11 @@ static PyObject* serialize_list(PyObject* self, PyObject* args) { return NULL; } - auto batch = new std::shared_ptr(); + auto batch = new std::shared_ptr(); *batch = make_row_batch(array); int64_t size = 0; - ARROW_CHECK_OK(arrow::ipc::GetRowBatchSize(batch->get(), &size)); + ARROW_CHECK_OK(arrow::ipc::GetRecordBatchSize(batch->get(), &size)); std::shared_ptr buffer; ARROW_CHECK_OK(ipc::WriteSchema((*batch)->schema().get(), &buffer)); @@ -84,7 +83,7 @@ static PyObject* serialize_list(PyObject* self, PyObject* args) { /* Documented in doc/numbuf.rst in ray-core */ static PyObject* write_to_buffer(PyObject* self, PyObject* args) { - std::shared_ptr* batch; + std::shared_ptr* batch; PyObject* memoryview; if (!PyArg_ParseTuple(args, "O&O", &PyObjectToArrow, &batch, &memoryview)) { return NULL; @@ -93,10 +92,11 @@ static PyObject* write_to_buffer(PyObject* self, PyObject* args) { return NULL; } Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview); - auto target = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); - int64_t metadata_offset; - ARROW_CHECK_OK(ipc::WriteRowBatch(target.get(), batch->get(), 0, &metadata_offset)); - return PyInt_FromLong(metadata_offset); + auto target = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); + int64_t body_end_offset; + int64_t header_end_offset; + ARROW_CHECK_OK(ipc::WriteRecordBatch((*batch)->columns(), (*batch)->num_rows(), target.get(), &body_end_offset, &header_end_offset)); + return PyInt_FromLong(header_end_offset); } /* Documented in doc/numbuf.rst in ray-core */ @@ -118,11 +118,11 @@ static PyObject* read_from_buffer(PyObject* self, PyObject* args) { ARROW_CHECK_OK(schema_msg->GetSchema(&schema)); Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview); - auto source = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); - std::shared_ptr reader; - ARROW_CHECK_OK(arrow::ipc::RowBatchReader::Open(source.get(), metadata_offset, &reader)); - auto batch = new std::shared_ptr(); - ARROW_CHECK_OK(reader->GetRowBatch(schema, batch)); + auto source = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); + std::shared_ptr reader; + ARROW_CHECK_OK(arrow::ipc::RecordBatchReader::Open(source.get(), metadata_offset, &reader)); + auto batch = new std::shared_ptr(); + ARROW_CHECK_OK(reader->GetRecordBatch(schema, batch)); return PyCapsule_New(reinterpret_cast(batch), "arrow", &ArrowCapsule_Destructor); @@ -130,7 +130,7 @@ static PyObject* read_from_buffer(PyObject* self, PyObject* args) { /* Documented in doc/numbuf.rst in ray-core */ static PyObject* deserialize_list(PyObject* self, PyObject* args) { - std::shared_ptr* data; + std::shared_ptr* data; PyObject* base = Py_None; if (!PyArg_ParseTuple(args, "O&|O", &PyObjectToArrow, &data, &base)) { return NULL; diff --git a/setup.py b/setup.py index ce9a51bc7..066ca4d0c 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,13 @@ import subprocess from setuptools import setup, find_packages, Extension import setuptools.command.install as _install +from sys import platform + +extension = "" +if platform == "linux" or platform == "linux2": + extension = ".so" +elif platform == "darwin": + extension = ".dylib" # Because of relative paths, this must be run from inside numbuf/. @@ -8,7 +15,6 @@ class install(_install.install): def run(self): subprocess.check_call(["./setup.sh"]) subprocess.check_call(["./build.sh"]) - subprocess.check_call(["cp", "libnumbuf.so", "numbuf/"]) # Calling _install.install.run(self) does not fetch required packages and # instead performs an old-style install. See command/install.py in # setuptools. So, calling do_egg_install() manually here. @@ -17,7 +23,7 @@ class install(_install.install): setup(name="numbuf", version="0.0.1", packages=find_packages(), - package_data={"numbuf": ["libnumbuf.so"]}, + package_data={"numbuf": ["libnumbuf.so", "libarrow" + extension, "libarrow_io" + extension, "libarrow_ipc" + extension]}, cmdclass={"install": install}, setup_requires=["numpy"], include_package_data=True, diff --git a/thirdparty/arrow b/thirdparty/arrow index 1407126ec..58bd7bedc 160000 --- a/thirdparty/arrow +++ b/thirdparty/arrow @@ -1 +1 @@ -Subproject commit 1407126ec0956b6dc6f9873bdeeef4d042ca032f +Subproject commit 58bd7bedc63d66d5898297bab25b54dfb67665db diff --git a/thirdparty/build_thirdparty.sh b/thirdparty/build_thirdparty.sh index 7c307f816..e5c4b5b45 100755 --- a/thirdparty/build_thirdparty.sh +++ b/thirdparty/build_thirdparty.sh @@ -20,8 +20,7 @@ fi echo "building arrow" cd $TP_DIR/arrow/cpp -source setup_build_env.sh mkdir -p $TP_DIR/arrow/cpp/build cd $TP_DIR/arrow/cpp/build -cmake -DLIBARROW_LINKAGE=STATIC -DCMAKE_BUILD_TYPE=Release .. +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g" -DCMAKE_CXX_FLAGS="-g" .. make VERBOSE=1 -j$PARALLEL From 9d6da1281ebfed8d4ab01b5441aa0c75b7e98837 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sat, 19 Nov 2016 12:56:11 -0800 Subject: [PATCH 30/32] add clang-format for numbuf (arrow style) --- numbuf/cpp/.clang-format | 65 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 numbuf/cpp/.clang-format diff --git a/numbuf/cpp/.clang-format b/numbuf/cpp/.clang-format new file mode 100644 index 000000000..7d5b3cf30 --- /dev/null +++ b/numbuf/cpp/.clang-format @@ -0,0 +1,65 @@ +--- +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: false +AlignConsecutiveAssignments: false +AlignEscapedNewlinesLeft: true +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: true +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: true +BinPackParameters: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +ColumnLimit: 90 +CommentPragmas: '^ IWYU pragma:' +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +IndentCaseLabels: true +IndentWidth: 2 +IndentWrappedFunctionNames: false +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 1000 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 8 +UseTab: Never From 05e0226047c4fbbaad2197ebdf3f60feaaec0d46 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sat, 19 Nov 2016 13:24:51 -0800 Subject: [PATCH 31/32] clang-fixes --- numbuf/{cpp => }/.clang-format | 0 numbuf/cpp/src/numbuf/dict.cc | 13 +- numbuf/cpp/src/numbuf/dict.h | 18 +-- numbuf/cpp/src/numbuf/sequence.cc | 37 ++--- numbuf/cpp/src/numbuf/sequence.h | 16 +- numbuf/cpp/src/numbuf/tensor.cc | 17 ++- numbuf/cpp/src/numbuf/tensor.h | 25 ++-- numbuf/python/src/pynumbuf/adapters/numpy.cc | 44 +++--- numbuf/python/src/pynumbuf/adapters/numpy.h | 11 +- numbuf/python/src/pynumbuf/adapters/python.cc | 139 ++++++++++-------- numbuf/python/src/pynumbuf/adapters/python.h | 16 +- numbuf/python/src/pynumbuf/adapters/scalars.h | 30 ++-- numbuf/python/src/pynumbuf/memory.h | 33 ++--- numbuf/python/src/pynumbuf/numbuf.cc | 83 +++++------ 14 files changed, 239 insertions(+), 243 deletions(-) rename numbuf/{cpp => }/.clang-format (100%) diff --git a/numbuf/cpp/.clang-format b/numbuf/.clang-format similarity index 100% rename from numbuf/cpp/.clang-format rename to numbuf/.clang-format diff --git a/numbuf/cpp/src/numbuf/dict.cc b/numbuf/cpp/src/numbuf/dict.cc index 74a9cfadf..047191d8c 100644 --- a/numbuf/cpp/src/numbuf/dict.cc +++ b/numbuf/cpp/src/numbuf/dict.cc @@ -4,12 +4,9 @@ using namespace arrow; namespace numbuf { -Status DictBuilder::Finish( - std::shared_ptr key_tuple_data, - std::shared_ptr val_list_data, - std::shared_ptr val_tuple_data, - std::shared_ptr val_dict_data, - std::shared_ptr* out) { +Status DictBuilder::Finish(std::shared_ptr key_tuple_data, + std::shared_ptr val_list_data, std::shared_ptr val_tuple_data, + std::shared_ptr val_dict_data, std::shared_ptr* out) { // lists and dicts can't be keys of dicts in Python, that is why for // the keys we do not need to collect sublists std::shared_ptr keys, vals; @@ -17,11 +14,11 @@ Status DictBuilder::Finish( RETURN_NOT_OK(vals_.Finish(val_list_data, val_tuple_data, val_dict_data, &vals)); auto keys_field = std::make_shared("keys", keys->type()); auto vals_field = std::make_shared("vals", vals->type()); - auto type = std::make_shared(std::vector({keys_field, vals_field})); + auto type = + std::make_shared(std::vector({keys_field, vals_field})); std::vector field_arrays({keys, vals}); DCHECK(keys->length() == vals->length()); out->reset(new StructArray(type, keys->length(), field_arrays)); return Status::OK(); } - } diff --git a/numbuf/cpp/src/numbuf/dict.h b/numbuf/cpp/src/numbuf/dict.h index 6497b3eb7..c8f5925a7 100644 --- a/numbuf/cpp/src/numbuf/dict.h +++ b/numbuf/cpp/src/numbuf/dict.h @@ -13,9 +13,8 @@ namespace numbuf { can be obtained via the Finish method. */ class DictBuilder { -public: - DictBuilder(arrow::MemoryPool* pool = nullptr) - : keys_(pool), vals_(pool) {} + public: + DictBuilder(arrow::MemoryPool* pool = nullptr) : keys_(pool), vals_(pool) {} //! Builder for the keys of the dictionary SequenceBuilder& keys() { return keys_; } @@ -33,18 +32,15 @@ public: List containing the data from nested dictionaries in the value list of the dictionary */ - arrow::Status Finish( - std::shared_ptr key_tuple_data, - std::shared_ptr val_list_data, - std::shared_ptr val_tuple_data, - std::shared_ptr val_dict_data, - std::shared_ptr* out); + arrow::Status Finish(std::shared_ptr key_tuple_data, + std::shared_ptr val_list_data, + std::shared_ptr val_tuple_data, + std::shared_ptr val_dict_data, std::shared_ptr* out); -private: + private: SequenceBuilder keys_; SequenceBuilder vals_; }; - } #endif diff --git a/numbuf/cpp/src/numbuf/sequence.cc b/numbuf/cpp/src/numbuf/sequence.cc index 9672f5e5d..b1b5fe2e6 100644 --- a/numbuf/cpp/src/numbuf/sequence.cc +++ b/numbuf/cpp/src/numbuf/sequence.cc @@ -25,15 +25,17 @@ SequenceBuilder::SequenceBuilder(MemoryPool* pool) int64_tensors_(std::make_shared(), pool), float_tensors_(std::make_shared(), pool), double_tensors_(std::make_shared(), pool), - list_offsets_({0}), tuple_offsets_({0}), dict_offsets_({0}) {} + list_offsets_({0}), + tuple_offsets_({0}), + dict_offsets_({0}) {} -#define UPDATE(OFFSET, TAG) \ - if (TAG == -1) { \ - TAG = num_tags; \ - num_tags += 1; \ - } \ - RETURN_NOT_OK(offsets_.Append(OFFSET)); \ - RETURN_NOT_OK(types_.Append(TAG)); \ +#define UPDATE(OFFSET, TAG) \ + if (TAG == -1) { \ + TAG = num_tags; \ + num_tags += 1; \ + } \ + RETURN_NOT_OK(offsets_.Append(OFFSET)); \ + RETURN_NOT_OK(types_.Append(TAG)); \ RETURN_NOT_OK(nones_.AppendToBitmap(true)); Status SequenceBuilder::AppendNone() { @@ -79,9 +81,7 @@ Status SequenceBuilder::AppendDouble(double data) { #define DEF_TENSOR_APPEND(NAME, TYPE, TAG) \ Status SequenceBuilder::AppendTensor(const std::vector& dims, TYPE* data) { \ - if (TAG == -1) { \ - NAME.Start(); \ - } \ + if (TAG == -1) { NAME.Start(); } \ UPDATE(NAME.length(), TAG); \ return NAME.Append(dims, data); \ } @@ -138,12 +138,9 @@ Status SequenceBuilder::AppendDict(int32_t size) { DCHECK(OFFSETS.size() == 1); \ } -Status SequenceBuilder::Finish( - std::shared_ptr list_data, - std::shared_ptr tuple_data, - std::shared_ptr dict_data, - std::shared_ptr* out) { - +Status SequenceBuilder::Finish(std::shared_ptr list_data, + std::shared_ptr tuple_data, std::shared_ptr dict_data, + std::shared_ptr* out) { std::vector> types(num_tags); std::vector children(num_tags); @@ -174,10 +171,8 @@ Status SequenceBuilder::Finish( std::vector type_ids = {}; TypePtr type = TypePtr(new UnionType(types, type_ids, UnionMode::DENSE)); - out->reset(new UnionArray(type, types_.length(), - children, types_.data(), offsets_.data(), - nones_.null_count(), nones_.null_bitmap())); + out->reset(new UnionArray(type, types_.length(), children, types_.data(), + offsets_.data(), nones_.null_count(), nones_.null_bitmap())); return Status::OK(); } - } diff --git a/numbuf/cpp/src/numbuf/sequence.h b/numbuf/cpp/src/numbuf/sequence.h index bcb1c1afd..4c87d81a6 100644 --- a/numbuf/cpp/src/numbuf/sequence.h +++ b/numbuf/cpp/src/numbuf/sequence.h @@ -1,9 +1,9 @@ #ifndef NUMBUF_LIST_H #define NUMBUF_LIST_H +#include "tensor.h" #include #include -#include "tensor.h" namespace numbuf { @@ -25,7 +25,7 @@ class SequenceBuilder { //! Appending an uint64_t to the sequence arrow::Status AppendUInt64(uint64_t data); - + //! Append a list of bytes to the sequence arrow::Status AppendBytes(const uint8_t* data, int32_t length); @@ -79,11 +79,9 @@ class SequenceBuilder { arrow::Status AppendDict(int32_t size); //! Finish building the sequence and return the result - arrow::Status Finish( - std::shared_ptr list_data, - std::shared_ptr tuple_data, - std::shared_ptr dict_data, - std::shared_ptr* out); + arrow::Status Finish(std::shared_ptr list_data, + std::shared_ptr tuple_data, std::shared_ptr dict_data, + std::shared_ptr* out); private: arrow::MemoryPool* pool_; @@ -139,6 +137,6 @@ class SequenceBuilder { int8_t num_tags = 0; }; -} // namespace numbuf +} // namespace numbuf -#endif // NUMBUF_LIST_H +#endif // NUMBUF_LIST_H diff --git a/numbuf/cpp/src/numbuf/tensor.cc b/numbuf/cpp/src/numbuf/tensor.cc index 5d95a94a7..db9325563 100644 --- a/numbuf/cpp/src/numbuf/tensor.cc +++ b/numbuf/cpp/src/numbuf/tensor.cc @@ -4,11 +4,11 @@ using namespace arrow; namespace numbuf { -template +template TensorBuilder::TensorBuilder(const TypePtr& dtype, MemoryPool* pool) : dtype_(dtype), pool_(pool) {} -template +template Status TensorBuilder::Start() { dim_data_ = std::make_shared(pool_, std::make_shared()); dims_ = std::make_shared(pool_, dim_data_); @@ -16,12 +16,14 @@ Status TensorBuilder::Start() { values_ = std::make_shared(pool_, value_data_); auto dims_field = std::make_shared("dims", dims_->type()); auto values_field = std::make_shared("data", values_->type()); - auto type = std::make_shared(std::vector({dims_field, values_field})); - tensors_ = std::make_shared(pool_, type, std::vector>({dims_, values_})); + auto type = + std::make_shared(std::vector({dims_field, values_field})); + tensors_ = std::make_shared( + pool_, type, std::vector>({dims_, values_})); return Status::OK(); } -template +template Status TensorBuilder::Append(const std::vector& dims, const elem_type* data) { DCHECK(tensors_); RETURN_NOT_OK(tensors_->Append()); @@ -33,10 +35,10 @@ Status TensorBuilder::Append(const std::vector& dims, const elem_typ RETURN_NOT_OK(dim_data_->Append(dim)); } RETURN_NOT_OK(value_data_->Append(data, size)); - return Status::OK(); // tensors_->Append(); + return Status::OK(); // tensors_->Append(); } -template +template Status TensorBuilder::Finish(std::shared_ptr* out) { return tensors_->Finish(out); } @@ -51,5 +53,4 @@ template class TensorBuilder; template class TensorBuilder; template class TensorBuilder; template class TensorBuilder; - } diff --git a/numbuf/cpp/src/numbuf/tensor.h b/numbuf/cpp/src/numbuf/tensor.h index e3725fcc9..2725f4615 100644 --- a/numbuf/cpp/src/numbuf/tensor.h +++ b/numbuf/cpp/src/numbuf/tensor.h @@ -1,9 +1,9 @@ #ifndef NUMBUF_TENSOR_H #define NUMBUF_TENSOR_H -#include -#include #include +#include +#include namespace numbuf { @@ -12,13 +12,13 @@ namespace numbuf { columns, "dims" which contains an array of dimensions for each Tensor and "data" which contains data buffer of the Tensor as a flattened array. */ -template +template class TensorBuilder { -public: + public: typedef typename T::c_type elem_type; TensorBuilder(const arrow::TypePtr& dtype, arrow::MemoryPool* pool = nullptr); - + arrow::Status Start(); /*! Append a new tensor. @@ -36,16 +36,12 @@ public: arrow::Status Finish(std::shared_ptr* out); //! Number of tensors in the column - int32_t length() { - return tensors_->length(); - } + int32_t length() { return tensors_->length(); } - const arrow::TypePtr& type() { - return tensors_->type(); - } + const arrow::TypePtr& type() { return tensors_->type(); } -private: - arrow::TypePtr dtype_; + private: + arrow::TypePtr dtype_; arrow::MemoryPool* pool_; std::shared_ptr dim_data_; std::shared_ptr dims_; @@ -64,7 +60,6 @@ typedef TensorBuilder UInt64TensorBuilder; typedef TensorBuilder Int64TensorBuilder; typedef TensorBuilder FloatTensorBuilder; typedef TensorBuilder DoubleTensorBuilder; - } -#endif // NUMBUF_TENSOR_H +#endif // NUMBUF_TENSOR_H diff --git a/numbuf/python/src/pynumbuf/adapters/numpy.cc b/numbuf/python/src/pynumbuf/adapters/numpy.cc index 32901ffad..4964f61f2 100644 --- a/numbuf/python/src/pynumbuf/adapters/numpy.cc +++ b/numbuf/python/src/pynumbuf/adapters/numpy.cc @@ -8,8 +8,8 @@ using namespace arrow; extern "C" { - extern PyObject *numbuf_serialize_callback; - extern PyObject *numbuf_deserialize_callback; +extern PyObject* numbuf_serialize_callback; +extern PyObject* numbuf_deserialize_callback; } namespace numbuf { @@ -18,22 +18,20 @@ namespace numbuf { case Type::TYPE: \ return NPY_##TYPE; -#define DESERIALIZE_ARRAY_CASE(TYPE, ArrayType, type) \ - case Type::TYPE: { \ - auto values = std::dynamic_pointer_cast(content->values()); \ - DCHECK(values); \ - type* data = const_cast(values->raw_data()) \ - + content->offset(offset); \ - *out = PyArray_SimpleNewFromData(num_dims, dim.data(), NPY_##TYPE, \ - reinterpret_cast(data)); \ - if (base != Py_None) { \ - PyArray_SetBaseObject((PyArrayObject*) *out, base); \ - } \ - Py_XINCREF(base); \ - } \ - return Status::OK(); +#define DESERIALIZE_ARRAY_CASE(TYPE, ArrayType, type) \ + case Type::TYPE: { \ + auto values = std::dynamic_pointer_cast(content->values()); \ + DCHECK(values); \ + type* data = const_cast(values->raw_data()) + content->offset(offset); \ + *out = PyArray_SimpleNewFromData( \ + num_dims, dim.data(), NPY_##TYPE, reinterpret_cast(data)); \ + if (base != Py_None) { PyArray_SetBaseObject((PyArrayObject*)*out, base); } \ + Py_XINCREF(base); \ + } \ + return Status::OK(); -Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject* base, PyObject** out) { +Status DeserializeArray( + std::shared_ptr array, int32_t offset, PyObject* base, PyObject** out) { DCHECK(array); auto tensor = std::dynamic_pointer_cast(array); DCHECK(tensor); @@ -41,9 +39,9 @@ Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject* auto content = std::dynamic_pointer_cast(tensor->field(1)); npy_intp num_dims = dims->value_length(offset); std::vector dim(num_dims); - for (int i = dims->offset(offset); i < dims->offset(offset+1); ++i) { + for (int i = dims->offset(offset); i < dims->offset(offset + 1); ++i) { dim[i - dims->offset(offset)] = - std::dynamic_pointer_cast(dims->values())->Value(i); + std::dynamic_pointer_cast(dims->values())->Value(i); } switch (content->value_type()->type) { DESERIALIZE_ARRAY_CASE(INT8, Int8Array, int8_t) @@ -62,8 +60,8 @@ Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject* return Status::OK(); } -Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, - std::vector& subdicts) { +Status SerializeArray( + PyArrayObject* array, SequenceBuilder& builder, std::vector& subdicts) { size_t ndim = PyArray_NDIM(array); int dtype = PyArray_TYPE(array); std::vector dims(ndim); @@ -119,7 +117,8 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); Py_XDECREF(arglist); if (!result) { - return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/ray-project/numbuf/issues/10 + return Status::NotImplemented("python error"); // TODO(pcm): + // https://github.com/ray-project/numbuf/issues/10 } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); @@ -128,5 +127,4 @@ Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, Py_XDECREF(contiguous); return Status::OK(); } - } diff --git a/numbuf/python/src/pynumbuf/adapters/numpy.h b/numbuf/python/src/pynumbuf/adapters/numpy.h index c0fcbb7f6..91f0473da 100644 --- a/numbuf/python/src/pynumbuf/adapters/numpy.h +++ b/numbuf/python/src/pynumbuf/adapters/numpy.h @@ -1,22 +1,23 @@ #ifndef PYNUMBUF_NUMPY_H #define PYNUMBUF_NUMPY_H -#include #include +#include #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #define NO_IMPORT_ARRAY #define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API #include -#include #include +#include namespace numbuf { -arrow::Status SerializeArray(PyArrayObject* array, SequenceBuilder& builder, std::vector& subdicts); -arrow::Status DeserializeArray(std::shared_ptr array, int32_t offset, PyObject* base, PyObject** out); - +arrow::Status SerializeArray( + PyArrayObject* array, SequenceBuilder& builder, std::vector& subdicts); +arrow::Status DeserializeArray( + std::shared_ptr array, int32_t offset, PyObject* base, PyObject** out); } #endif diff --git a/numbuf/python/src/pynumbuf/adapters/python.cc b/numbuf/python/src/pynumbuf/adapters/python.cc index 2d41cbaa3..eada6e958 100644 --- a/numbuf/python/src/pynumbuf/adapters/python.cc +++ b/numbuf/python/src/pynumbuf/adapters/python.cc @@ -12,46 +12,54 @@ extern "C" { extern PyObject* numbuf_serialize_callback; extern PyObject* numbuf_deserialize_callback; - } namespace numbuf { -Status get_value(ArrayPtr arr, int32_t index, int32_t type, PyObject* base, PyObject** result) { +Status get_value( + ArrayPtr arr, int32_t index, int32_t type, PyObject* base, PyObject** result) { switch (arr->type()->type) { case Type::BOOL: - *result = PyBool_FromLong(std::static_pointer_cast(arr)->Value(index)); + *result = + PyBool_FromLong(std::static_pointer_cast(arr)->Value(index)); return Status::OK(); case Type::INT64: *result = PyInt_FromLong(std::static_pointer_cast(arr)->Value(index)); return Status::OK(); case Type::BINARY: { int32_t nchars; - const uint8_t* str = std::static_pointer_cast(arr)->GetValue(index, &nchars); + const uint8_t* str = + std::static_pointer_cast(arr)->GetValue(index, &nchars); *result = PyString_FromStringAndSize(reinterpret_cast(str), nchars); return Status::OK(); } case Type::STRING: { int32_t nchars; - const uint8_t* str = std::static_pointer_cast(arr)->GetValue(index, &nchars); + const uint8_t* str = + std::static_pointer_cast(arr)->GetValue(index, &nchars); *result = PyUnicode_FromStringAndSize(reinterpret_cast(str), nchars); return Status::OK(); } case Type::FLOAT: - *result = PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + *result = + PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); return Status::OK(); case Type::DOUBLE: - *result = PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); + *result = + PyFloat_FromDouble(std::static_pointer_cast(arr)->Value(index)); return Status::OK(); case Type::STRUCT: { auto s = std::static_pointer_cast(arr); auto l = std::static_pointer_cast(s->field(0)); if (s->type()->child(0)->name == "list") { - return DeserializeList(l->values(), l->value_offset(index), l->value_offset(index+1), base, result); + return DeserializeList(l->values(), l->value_offset(index), + l->value_offset(index + 1), base, result); } else if (s->type()->child(0)->name == "tuple") { - return DeserializeTuple(l->values(), l->value_offset(index), l->value_offset(index+1), base, result); + return DeserializeTuple(l->values(), l->value_offset(index), + l->value_offset(index + 1), base, result); } else if (s->type()->child(0)->name == "dict") { - return DeserializeDict(l->values(), l->value_offset(index), l->value_offset(index+1), base, result); + return DeserializeDict(l->values(), l->value_offset(index), + l->value_offset(index + 1), base, result); } else { return DeserializeArray(arr, index, base, result); } @@ -62,10 +70,8 @@ Status get_value(ArrayPtr arr, int32_t index, int32_t type, PyObject* base, PyOb return Status::OK(); } -Status append(PyObject* elem, SequenceBuilder& builder, - std::vector& sublists, - std::vector& subtuples, - std::vector& subdicts) { +Status append(PyObject* elem, SequenceBuilder& builder, std::vector& sublists, + std::vector& subtuples, std::vector& subdicts) { // The bool case must precede the int case (PyInt_Check passes for bools) if (PyBool_Check(elem)) { RETURN_NOT_OK(builder.AppendBool(elem == Py_True)); @@ -75,9 +81,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, int overflow = 0; int64_t data = PyLong_AsLongLongAndOverflow(elem, &overflow); RETURN_NOT_OK(builder.AppendInt64(data)); - if(overflow) { - return Status::NotImplemented("long overflow"); - } + if (overflow) { return Status::NotImplemented("long overflow"); } } else if (PyInt_Check(elem)) { RETURN_NOT_OK(builder.AppendInt64(static_cast(PyInt_AS_LONG(elem)))); } else if (PyString_Check(elem)) { @@ -86,13 +90,14 @@ Status append(PyObject* elem, SequenceBuilder& builder, RETURN_NOT_OK(builder.AppendBytes(data, size)); } else if (PyUnicode_Check(elem)) { Py_ssize_t size; - #if PY_MAJOR_VERSION >= 3 - char* data = PyUnicode_AsUTF8AndSize(elem, &size); // TODO(pcm): Check if this is correct - #else - PyObject* str = PyUnicode_AsUTF8String(elem); - char* data = PyString_AS_STRING(str); - size = PyString_GET_SIZE(str); - #endif +#if PY_MAJOR_VERSION >= 3 + char* data = + PyUnicode_AsUTF8AndSize(elem, &size); // TODO(pcm): Check if this is correct +#else + PyObject* str = PyUnicode_AsUTF8String(elem); + char* data = PyString_AS_STRING(str); + size = PyString_GET_SIZE(str); +#endif Status s = builder.AppendString(data, size); Py_XDECREF(str); RETURN_NOT_OK(s); @@ -108,14 +113,14 @@ Status append(PyObject* elem, SequenceBuilder& builder, } else if (PyArray_IsScalar(elem, Generic)) { RETURN_NOT_OK(AppendScalar(elem, builder)); } else if (PyArray_Check(elem)) { - RETURN_NOT_OK(SerializeArray((PyArrayObject*) elem, builder, subdicts)); + RETURN_NOT_OK(SerializeArray((PyArrayObject*)elem, builder, subdicts)); } else if (elem == Py_None) { RETURN_NOT_OK(builder.AppendNone()); } else { if (!numbuf_serialize_callback) { std::stringstream ss; ss << "data type of " << PyString_AS_STRING(PyObject_Repr(elem)) - << " not recognized and custom serialization handler not registered"; + << " not recognized and custom serialization handler not registered"; return Status::NotImplemented(ss.str()); } else { PyObject* arglist = Py_BuildValue("(O)", elem); @@ -124,7 +129,8 @@ Status append(PyObject* elem, SequenceBuilder& builder, PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); Py_XDECREF(arglist); if (!result) { - return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/ray-project/numbuf/issues/10 + return Status::NotImplemented("python error"); // TODO(pcm): + // https://github.com/ray-project/numbuf/issues/10 } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); @@ -133,13 +139,16 @@ Status append(PyObject* elem, SequenceBuilder& builder, return Status::OK(); } -Status SerializeSequences(std::vector sequences, int32_t recursion_depth, std::shared_ptr* out) { +Status SerializeSequences(std::vector sequences, int32_t recursion_depth, + std::shared_ptr* out) { DCHECK(out); if (recursion_depth >= MAX_RECURSION_DEPTH) { - return Status::NotImplemented("This object exceeds the maximum recursion depth. It may contain itself recursively."); + return Status::NotImplemented( + "This object exceeds the maximum recursion depth. It may contain itself " + "recursively."); } SequenceBuilder builder(nullptr); - std::vector sublists, subtuples, subdicts; + std::vector sublists, subtuples, subdicts; for (const auto& sequence : sequences) { PyObject* item; PyObject* iterator = PyObject_GetIter(sequence); @@ -169,42 +178,47 @@ Status SerializeSequences(std::vector sequences, int32_t recursion_de return builder.Finish(list, tuple, dict, out); } -#define DESERIALIZE_SEQUENCE(CREATE, SET_ITEM) \ - auto data = std::dynamic_pointer_cast(array); \ - int32_t size = array->length(); \ - PyObject* result = CREATE(stop_idx - start_idx); \ - auto types = std::make_shared(size, data->types()); \ - auto offsets = std::make_shared(size, data->offset_buf()); \ - for (size_t i = start_idx; i < stop_idx; ++i) { \ - if (data->IsNull(i)) { \ - Py_INCREF(Py_None); \ - SET_ITEM(result, i-start_idx, Py_None); \ - } else { \ - int32_t offset = offsets->Value(i); \ - int8_t type = types->Value(i); \ - ArrayPtr arr = data->child(type); \ - PyObject* value; \ - RETURN_NOT_OK(get_value(arr, offset, type, base, &value)); \ - SET_ITEM(result, i-start_idx, value); \ - } \ - } \ - *out = result; \ +#define DESERIALIZE_SEQUENCE(CREATE, SET_ITEM) \ + auto data = std::dynamic_pointer_cast(array); \ + int32_t size = array->length(); \ + PyObject* result = CREATE(stop_idx - start_idx); \ + auto types = std::make_shared(size, data->types()); \ + auto offsets = std::make_shared(size, data->offset_buf()); \ + for (size_t i = start_idx; i < stop_idx; ++i) { \ + if (data->IsNull(i)) { \ + Py_INCREF(Py_None); \ + SET_ITEM(result, i - start_idx, Py_None); \ + } else { \ + int32_t offset = offsets->Value(i); \ + int8_t type = types->Value(i); \ + ArrayPtr arr = data->child(type); \ + PyObject* value; \ + RETURN_NOT_OK(get_value(arr, offset, type, base, &value)); \ + SET_ITEM(result, i - start_idx, value); \ + } \ + } \ + *out = result; \ return Status::OK(); -Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out) { +Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, + PyObject* base, PyObject** out) { DESERIALIZE_SEQUENCE(PyList_New, PyList_SetItem) } -Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out) { +Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, + PyObject* base, PyObject** out) { DESERIALIZE_SEQUENCE(PyTuple_New, PyTuple_SetItem) } -Status SerializeDict(std::vector dicts, int32_t recursion_depth, std::shared_ptr* out) { +Status SerializeDict( + std::vector dicts, int32_t recursion_depth, std::shared_ptr* out) { DictBuilder result; if (recursion_depth >= MAX_RECURSION_DEPTH) { - return Status::NotImplemented("This object exceeds the maximum recursion depth. It may contain itself recursively."); + return Status::NotImplemented( + "This object exceeds the maximum recursion depth. It may contain itself " + "recursively."); } - std::vector key_tuples, val_lists, val_tuples, val_dicts, dummy; + std::vector key_tuples, val_lists, val_tuples, val_dicts, dummy; for (const auto& dict : dicts) { PyObject *key, *value; Py_ssize_t pos = 0; @@ -248,7 +262,8 @@ Status SerializeDict(std::vector dicts, int32_t recursion_depth, std: return Status::OK(); } -Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out) { +Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, + PyObject* base, PyObject** out) { auto data = std::dynamic_pointer_cast(array); // TODO(pcm): error handling, get rid of the temporary copy of the list PyObject *keys, *vals; @@ -256,10 +271,11 @@ Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t ARROW_RETURN_NOT_OK(DeserializeList(data->field(0), start_idx, stop_idx, base, &keys)); ARROW_RETURN_NOT_OK(DeserializeList(data->field(1), start_idx, stop_idx, base, &vals)); for (size_t i = start_idx; i < stop_idx; ++i) { - PyDict_SetItem(result, PyList_GetItem(keys, i - start_idx), PyList_GetItem(vals, i - start_idx)); + PyDict_SetItem( + result, PyList_GetItem(keys, i - start_idx), PyList_GetItem(vals, i - start_idx)); } - Py_XDECREF(keys); // PyList_GetItem(keys, ...) incremented the reference count - Py_XDECREF(vals); // PyList_GetItem(vals, ...) incremented the reference count + Py_XDECREF(keys); // PyList_GetItem(keys, ...) incremented the reference count + Py_XDECREF(vals); // PyList_GetItem(vals, ...) incremented the reference count static PyObject* py_type = PyString_FromString("_pytype_"); if (PyDict_Contains(result, py_type) && numbuf_deserialize_callback) { PyObject* arglist = Py_BuildValue("(O)", result); @@ -270,12 +286,11 @@ Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t Py_XDECREF(result); result = callback_result; if (!callback_result) { - return Status::NotImplemented("python error"); // TODO(pcm): https://github.com/ray-project/numbuf/issues/10 + return Status::NotImplemented("python error"); // TODO(pcm): + // https://github.com/ray-project/numbuf/issues/10 } } *out = result; return Status::OK(); } - - } diff --git a/numbuf/python/src/pynumbuf/adapters/python.h b/numbuf/python/src/pynumbuf/adapters/python.h index 024ebebbf..dc25ad018 100644 --- a/numbuf/python/src/pynumbuf/adapters/python.h +++ b/numbuf/python/src/pynumbuf/adapters/python.h @@ -11,12 +11,16 @@ namespace numbuf { -arrow::Status SerializeSequences(std::vector sequences, int32_t recursion_depth, std::shared_ptr* out); -arrow::Status SerializeDict(std::vector dicts, int32_t recursion_depth, std::shared_ptr* out); -arrow::Status DeserializeList(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); -arrow::Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); -arrow::Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t stop_idx, PyObject* base, PyObject** out); - +arrow::Status SerializeSequences(std::vector sequences, + int32_t recursion_depth, std::shared_ptr* out); +arrow::Status SerializeDict(std::vector dicts, int32_t recursion_depth, + std::shared_ptr* out); +arrow::Status DeserializeList(std::shared_ptr array, int32_t start_idx, + int32_t stop_idx, PyObject* base, PyObject** out); +arrow::Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, + int32_t stop_idx, PyObject* base, PyObject** out); +arrow::Status DeserializeDict(std::shared_ptr array, int32_t start_idx, + int32_t stop_idx, PyObject* base, PyObject** out); } #endif diff --git a/numbuf/python/src/pynumbuf/adapters/scalars.h b/numbuf/python/src/pynumbuf/adapters/scalars.h index e680b446f..e64027c9e 100644 --- a/numbuf/python/src/pynumbuf/adapters/scalars.h +++ b/numbuf/python/src/pynumbuf/adapters/scalars.h @@ -16,39 +16,39 @@ namespace numbuf { arrow::Status AppendScalar(PyObject* obj, SequenceBuilder& builder) { if (PyArray_IsScalar(obj, Bool)) { - return builder.AppendBool(((PyBoolScalarObject *)obj)->obval != 0); + return builder.AppendBool(((PyBoolScalarObject*)obj)->obval != 0); } else if (PyArray_IsScalar(obj, Float)) { - return builder.AppendFloat(((PyFloatScalarObject *)obj)->obval); + return builder.AppendFloat(((PyFloatScalarObject*)obj)->obval); } else if (PyArray_IsScalar(obj, Double)) { - return builder.AppendDouble(((PyDoubleScalarObject *)obj)->obval); + return builder.AppendDouble(((PyDoubleScalarObject*)obj)->obval); } int64_t value = 0; if (PyArray_IsScalar(obj, Byte)) { - value = ((PyByteScalarObject *)obj)->obval; + value = ((PyByteScalarObject*)obj)->obval; } else if (PyArray_IsScalar(obj, UByte)) { - value = ((PyUByteScalarObject *)obj)->obval; + value = ((PyUByteScalarObject*)obj)->obval; } else if (PyArray_IsScalar(obj, Short)) { - value = ((PyShortScalarObject *)obj)->obval; + value = ((PyShortScalarObject*)obj)->obval; } else if (PyArray_IsScalar(obj, UShort)) { - value = ((PyUShortScalarObject *)obj)->obval; + value = ((PyUShortScalarObject*)obj)->obval; } else if (PyArray_IsScalar(obj, Int)) { - value = ((PyIntScalarObject *)obj)->obval; + value = ((PyIntScalarObject*)obj)->obval; } else if (PyArray_IsScalar(obj, UInt)) { - value = ((PyUIntScalarObject *)obj)->obval; + value = ((PyUIntScalarObject*)obj)->obval; } else if (PyArray_IsScalar(obj, Long)) { - value = ((PyLongScalarObject *)obj)->obval; + value = ((PyLongScalarObject*)obj)->obval; } else if (PyArray_IsScalar(obj, ULong)) { - value = ((PyULongScalarObject *)obj)->obval; + value = ((PyULongScalarObject*)obj)->obval; } else if (PyArray_IsScalar(obj, LongLong)) { - value = ((PyLongLongScalarObject *)obj)->obval; + value = ((PyLongLongScalarObject*)obj)->obval; } else if (PyArray_IsScalar(obj, ULongLong)) { - value = ((PyULongLongScalarObject *)obj)->obval; + value = ((PyULongLongScalarObject*)obj)->obval; } else { DCHECK(false) << "scalar type not recognized"; } return builder.AppendInt64(value); } -} // namespace +} // namespace -#endif // PYNUMBUF_SCALARS_H +#endif // PYNUMBUF_SCALARS_H diff --git a/numbuf/python/src/pynumbuf/memory.h b/numbuf/python/src/pynumbuf/memory.h index eb926dbcb..acd62f523 100644 --- a/numbuf/python/src/pynumbuf/memory.h +++ b/numbuf/python/src/pynumbuf/memory.h @@ -5,16 +5,18 @@ namespace numbuf { -class FixedBufferStream : public arrow::io::OutputStream, public arrow::io::ReadableFileInterface { +class FixedBufferStream : public arrow::io::OutputStream, + public arrow::io::ReadableFileInterface { public: virtual ~FixedBufferStream() {} explicit FixedBufferStream(uint8_t* data, int64_t nbytes) - : data_(data), position_(0), size_(nbytes) {} + : data_(data), position_(0), size_(nbytes) {} arrow::Status Read(int64_t nbytes, std::shared_ptr* out) override { DCHECK(out); - DCHECK(position_ + nbytes <= size_) << "position: " << position_ << " nbytes: " << nbytes << "size: " << size_; + DCHECK(position_ + nbytes <= size_) << "position: " << position_ + << " nbytes: " << nbytes << "size: " << size_; *out = std::make_shared(data_ + position_, nbytes); position_ += nbytes; return arrow::Status::OK(); @@ -30,9 +32,7 @@ class FixedBufferStream : public arrow::io::OutputStream, public arrow::io::Read return arrow::Status::OK(); } - arrow::Status Close() override { - return arrow::Status::OK(); - } + arrow::Status Close() override { return arrow::Status::OK(); } arrow::Status Tell(int64_t* position) override { *position = position_; @@ -41,28 +41,27 @@ class FixedBufferStream : public arrow::io::OutputStream, public arrow::io::Read arrow::Status Write(const uint8_t* data, int64_t nbytes) override { DCHECK(position_ >= 0 && position_ < size_); - DCHECK(position_ + nbytes <= size_) << "position: " << position_ << " nbytes: " << nbytes << "size: " << size_; + DCHECK(position_ + nbytes <= size_) << "position: " << position_ + << " nbytes: " << nbytes << "size: " << size_; uint8_t* dst = data_ + position_; memcpy(dst, data, nbytes); position_ += nbytes; return arrow::Status::OK(); } - arrow::Status GetSize(int64_t *size) override { + arrow::Status GetSize(int64_t* size) override { *size = size_; return arrow::Status::OK(); } - bool supports_zero_copy() const override { - return true; - } + bool supports_zero_copy() const override { return true; } -private: - uint8_t* data_; - int64_t position_; - int64_t size_; + private: + uint8_t* data_; + int64_t position_; + int64_t size_; }; -} // namespace numbuf +} // namespace numbuf -#endif // PYNUMBUF_MEMORY_H +#endif // PYNUMBUF_MEMORY_H diff --git a/numbuf/python/src/pynumbuf/numbuf.cc b/numbuf/python/src/pynumbuf/numbuf.cc index ba2483810..70be81b26 100644 --- a/numbuf/python/src/pynumbuf/numbuf.cc +++ b/numbuf/python/src/pynumbuf/numbuf.cc @@ -23,14 +23,15 @@ std::shared_ptr make_row_batch(std::shared_ptr data) { extern "C" { -static PyObject *NumbufError; +static PyObject* NumbufError; -PyObject *numbuf_serialize_callback = NULL; -PyObject *numbuf_deserialize_callback = NULL; +PyObject* numbuf_serialize_callback = NULL; +PyObject* numbuf_deserialize_callback = NULL; -int PyObjectToArrow(PyObject* object, std::shared_ptr **result) { +int PyObjectToArrow(PyObject* object, std::shared_ptr** result) { if (PyCapsule_IsValid(object, "arrow")) { - *result = reinterpret_cast*>(PyCapsule_GetPointer(object, "arrow")); + *result = reinterpret_cast*>( + PyCapsule_GetPointer(object, "arrow")); return 1; } else { PyErr_SetString(PyExc_TypeError, "must be an 'arrow' capsule"); @@ -39,25 +40,23 @@ int PyObjectToArrow(PyObject* object, std::shared_ptr **result) { } static void ArrowCapsule_Destructor(PyObject* capsule) { - delete reinterpret_cast*>(PyCapsule_GetPointer(capsule, "arrow")); + delete reinterpret_cast*>( + PyCapsule_GetPointer(capsule, "arrow")); } /* Documented in doc/numbuf.rst in ray-core */ static PyObject* serialize_list(PyObject* self, PyObject* args) { PyObject* value; - if (!PyArg_ParseTuple(args, "O", &value)) { - return NULL; - } + if (!PyArg_ParseTuple(args, "O", &value)) { return NULL; } std::shared_ptr array; if (PyList_Check(value)) { int32_t recursion_depth = 0; - Status s = SerializeSequences(std::vector({value}), recursion_depth, &array); + Status s = + SerializeSequences(std::vector({value}), recursion_depth, &array); if (!s.ok()) { // If this condition is true, there was an error in the callback that // needs to be passed through - if (!PyErr_Occurred()) { - PyErr_SetString(NumbufError, s.ToString().c_str()); - } + if (!PyErr_Occurred()) { PyErr_SetString(NumbufError, s.ToString().c_str()); } return NULL; } @@ -74,8 +73,8 @@ static PyObject* serialize_list(PyObject* self, PyObject* args) { PyObject* r = PyTuple_New(3); PyTuple_SetItem(r, 0, PyByteArray_FromStringAndSize(ptr, buffer->size())); PyTuple_SetItem(r, 1, PyInt_FromLong(size)); - PyTuple_SetItem(r, 2, PyCapsule_New(reinterpret_cast(batch), - "arrow", &ArrowCapsule_Destructor)); + PyTuple_SetItem(r, 2, + PyCapsule_New(reinterpret_cast(batch), "arrow", &ArrowCapsule_Destructor)); return r; } return NULL; @@ -88,14 +87,14 @@ static PyObject* write_to_buffer(PyObject* self, PyObject* args) { if (!PyArg_ParseTuple(args, "O&O", &PyObjectToArrow, &batch, &memoryview)) { return NULL; } - if (!PyMemoryView_Check(memoryview)) { - return NULL; - } + if (!PyMemoryView_Check(memoryview)) { return NULL; } Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview); - auto target = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); + auto target = std::make_shared( + reinterpret_cast(buffer->buf), buffer->len); int64_t body_end_offset; int64_t header_end_offset; - ARROW_CHECK_OK(ipc::WriteRecordBatch((*batch)->columns(), (*batch)->num_rows(), target.get(), &body_end_offset, &header_end_offset)); + ARROW_CHECK_OK(ipc::WriteRecordBatch((*batch)->columns(), (*batch)->num_rows(), + target.get(), &body_end_offset, &header_end_offset)); return PyInt_FromLong(header_end_offset); } @@ -118,31 +117,28 @@ static PyObject* read_from_buffer(PyObject* self, PyObject* args) { ARROW_CHECK_OK(schema_msg->GetSchema(&schema)); Py_buffer* buffer = PyMemoryView_GET_BUFFER(memoryview); - auto source = std::make_shared(reinterpret_cast(buffer->buf), buffer->len); + auto source = std::make_shared( + reinterpret_cast(buffer->buf), buffer->len); std::shared_ptr reader; - ARROW_CHECK_OK(arrow::ipc::RecordBatchReader::Open(source.get(), metadata_offset, &reader)); + ARROW_CHECK_OK( + arrow::ipc::RecordBatchReader::Open(source.get(), metadata_offset, &reader)); auto batch = new std::shared_ptr(); ARROW_CHECK_OK(reader->GetRecordBatch(schema, batch)); - return PyCapsule_New(reinterpret_cast(batch), - "arrow", &ArrowCapsule_Destructor); + return PyCapsule_New(reinterpret_cast(batch), "arrow", &ArrowCapsule_Destructor); } /* Documented in doc/numbuf.rst in ray-core */ static PyObject* deserialize_list(PyObject* self, PyObject* args) { std::shared_ptr* data; PyObject* base = Py_None; - if (!PyArg_ParseTuple(args, "O&|O", &PyObjectToArrow, &data, &base)) { - return NULL; - } + if (!PyArg_ParseTuple(args, "O&|O", &PyObjectToArrow, &data, &base)) { return NULL; } PyObject* result; Status s = DeserializeList((*data)->column(0), 0, (*data)->num_rows(), base, &result); if (!s.ok()) { // If this condition is true, there was an error in the callback that // needs to be passed through - if (!PyErr_Occurred()) { - PyErr_SetString(NumbufError, s.ToString().c_str()); - } + if (!PyErr_Occurred()) { PyErr_SetString(NumbufError, s.ToString().c_str()); } return NULL; } return result; @@ -152,7 +148,8 @@ static PyObject* register_callbacks(PyObject* self, PyObject* args) { PyObject* result = NULL; PyObject* serialize_callback; PyObject* deserialize_callback; - if (PyArg_ParseTuple(args, "OO:register_callbacks", &serialize_callback, &deserialize_callback)) { + if (PyArg_ParseTuple( + args, "OO:register_callbacks", &serialize_callback, &deserialize_callback)) { if (!PyCallable_Check(serialize_callback)) { PyErr_SetString(PyExc_TypeError, "serialize_callback must be callable"); return NULL; @@ -161,10 +158,10 @@ static PyObject* register_callbacks(PyObject* self, PyObject* args) { PyErr_SetString(PyExc_TypeError, "deserialize_callback must be callable"); return NULL; } - Py_XINCREF(serialize_callback); // Add a reference to new serialization callback - Py_XINCREF(deserialize_callback); // Add a reference to new deserialization callback - Py_XDECREF(numbuf_serialize_callback); // Dispose of old serialization callback - Py_XDECREF(numbuf_deserialize_callback); // Dispose of old deserialization callback + Py_XINCREF(serialize_callback); // Add a reference to new serialization callback + Py_XINCREF(deserialize_callback); // Add a reference to new deserialization callback + Py_XDECREF(numbuf_serialize_callback); // Dispose of old serialization callback + Py_XDECREF(numbuf_deserialize_callback); // Dispose of old deserialization callback numbuf_serialize_callback = serialize_callback; numbuf_deserialize_callback = deserialize_callback; Py_INCREF(Py_None); @@ -174,13 +171,14 @@ static PyObject* register_callbacks(PyObject* self, PyObject* args) { } static PyMethodDef NumbufMethods[] = { - { "serialize_list", serialize_list, METH_VARARGS, "serialize a Python list" }, - { "deserialize_list", deserialize_list, METH_VARARGS, "deserialize a Python list" }, - { "write_to_buffer", write_to_buffer, METH_VARARGS, "write serialized data to buffer"}, - { "read_from_buffer", read_from_buffer, METH_VARARGS, "read serialized data from buffer"}, - { "register_callbacks", register_callbacks, METH_VARARGS, "set serialization and deserialization callbacks"}, - { NULL, NULL, 0, NULL } -}; + {"serialize_list", serialize_list, METH_VARARGS, "serialize a Python list"}, + {"deserialize_list", deserialize_list, METH_VARARGS, "deserialize a Python list"}, + {"write_to_buffer", write_to_buffer, METH_VARARGS, "write serialized data to buffer"}, + {"read_from_buffer", read_from_buffer, METH_VARARGS, + "read serialized data from buffer"}, + {"register_callbacks", register_callbacks, METH_VARARGS, + "set serialization and deserialization callbacks"}, + {NULL, NULL, 0, NULL}}; PyMODINIT_FUNC initlibnumbuf(void) { PyObject* m; @@ -191,5 +189,4 @@ PyMODINIT_FUNC initlibnumbuf(void) { PyModule_AddObject(m, "numbuf_error", NumbufError); import_array(); } - } From d64f69f215916b010bc910a2dfa2ccd34b150860 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Sat, 19 Nov 2016 17:38:36 -0800 Subject: [PATCH 32/32] switch from submodule to cloning arrow, travis fixes & Robert's comments --- .travis.yml | 6 + doc/install-on-macosx.md | 4 +- doc/install-on-ubuntu.md | 4 +- install-dependencies.sh | 1 - numbuf/.gitignore | 32 --- numbuf/.gitmodules | 3 - numbuf/.travis.yml | 25 --- numbuf/LICENSE | 201 ------------------ numbuf/README.md | 2 - numbuf/install-dependencies.sh | 36 ---- numbuf/python/src/pynumbuf/adapters/numpy.cc | 5 +- numbuf/python/src/pynumbuf/adapters/python.cc | 10 +- numbuf/python/test/runtest.py | 2 +- numbuf/setup.py | 7 +- numbuf/setup.sh | 6 +- numbuf/thirdparty/arrow | 1 - numbuf/thirdparty/download_thirdparty.sh | 4 +- 17 files changed, 25 insertions(+), 324 deletions(-) delete mode 100644 numbuf/.gitignore delete mode 100644 numbuf/.gitmodules delete mode 100644 numbuf/.travis.yml delete mode 100644 numbuf/LICENSE delete mode 100644 numbuf/README.md delete mode 100755 numbuf/install-dependencies.sh delete mode 160000 numbuf/thirdparty/arrow diff --git a/.travis.yml b/.travis.yml index d44a93d84..7e8293e8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,10 @@ matrix: install: - ./install-dependencies.sh - ./build.sh + + - cd numbuf + - sudo python setup.py install + - cd .. - cd src/common/lib/python - sudo python setup.py install @@ -62,6 +66,8 @@ install: - cd ../.. script: + - python numbuf/python/test/runtest.py + - python src/common/test/test.py - python src/plasma/test/test.py - python src/photon/test/test.py diff --git a/doc/install-on-macosx.md b/doc/install-on-macosx.md index bc208c9f0..02b3c9b74 100644 --- a/doc/install-on-macosx.md +++ b/doc/install-on-macosx.md @@ -1,6 +1,6 @@ # Installation on Mac OS X -Ray should work with Python 2 and Python 3. We have tested Ray on OS X 10.11. +Ray should work with Python 2. We have tested Ray on OS X 10.11. ## Dependencies @@ -14,7 +14,7 @@ sudo easy_install pip # If you're using Anaconda, then this is unnecessary. pip install numpy funcsigs colorama psutil redis --ignore-installed six pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4 # We use the latest version of cloudpickle because it can serialize named tuples. -pip install --upgrade --verbose git+git://github.com/ray-project/numbuf.git@488f881d708bc54e86ed375ee97aa94540808fa1 +pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=ray&subdirectory=numbuf" ``` # Install Ray diff --git a/doc/install-on-ubuntu.md b/doc/install-on-ubuntu.md index fc4638dcc..9d8c50a77 100644 --- a/doc/install-on-ubuntu.md +++ b/doc/install-on-ubuntu.md @@ -1,6 +1,6 @@ # Installation on Ubuntu -Ray should work with Python 2 and Python 3. We have tested Ray on Ubuntu 14.04 +Ray should work with Python 2. We have tested Ray on Ubuntu 14.04 and Ubuntu 16.04 ## Dependencies @@ -14,7 +14,7 @@ sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev p pip install numpy funcsigs colorama psutil redis pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4 # We use the latest version of cloudpickle because it can serialize named tuples. -pip install --upgrade --verbose git+git://github.com/ray-project/numbuf.git@488f881d708bc54e86ed375ee97aa94540808fa1 +pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=ray&subdirectory=numbuf" ``` # Install Ray diff --git a/install-dependencies.sh b/install-dependencies.sh index d6104f50b..546989d15 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -37,4 +37,3 @@ elif [[ $platform == "macosx" ]]; then fi sudo pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4 # We use the latest version of cloudpickle because it can serialize named tuples. -sudo pip install --upgrade --verbose git+git://github.com/ray-project/numbuf.git@488f881d708bc54e86ed375ee97aa94540808fa1 diff --git a/numbuf/.gitignore b/numbuf/.gitignore deleted file mode 100644 index 2a83b5467..000000000 --- a/numbuf/.gitignore +++ /dev/null @@ -1,32 +0,0 @@ -# Compiled Object files -*.slo -*.lo -*.o -*.obj - -# Precompiled Headers -*.gch -*.pch - -# Compiled Dynamic libraries -*.so -*.dylib -*.dll - -# Fortran module files -*.mod -*.smod - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - -# Build directory -build diff --git a/numbuf/.gitmodules b/numbuf/.gitmodules deleted file mode 100644 index 29d35a973..000000000 --- a/numbuf/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "thirdparty/arrow"] - path = thirdparty/arrow - url = https://github.com/pcmoritz/arrow.git diff --git a/numbuf/.travis.yml b/numbuf/.travis.yml deleted file mode 100644 index fe784d053..000000000 --- a/numbuf/.travis.yml +++ /dev/null @@ -1,25 +0,0 @@ -sudo: required - -language: generic - -matrix: - include: - - os: linux - dist: trusty - python: "2.7" - - os: linux - dist: trusty - python: "3.5" - - os: osx - osx_image: xcode7 - python: "2.7" - - os: osx - osx_image: xcode7 - python: "3.5" - -install: - - ./install-dependencies.sh - - python setup.py install --user - -script: - - python python/test/runtest.py diff --git a/numbuf/LICENSE b/numbuf/LICENSE deleted file mode 100644 index 8dada3eda..000000000 --- a/numbuf/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/numbuf/README.md b/numbuf/README.md deleted file mode 100644 index b94616b43..000000000 --- a/numbuf/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# numbuf -Serializing primitive Python types in Arrow diff --git a/numbuf/install-dependencies.sh b/numbuf/install-dependencies.sh deleted file mode 100755 index 88973f6de..000000000 --- a/numbuf/install-dependencies.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -platform="unknown" -unamestr="$(uname)" -if [[ "$unamestr" == "Linux" ]]; then - echo "Platform is linux." - platform="linux" -elif [[ "$unamestr" == "Darwin" ]]; then - echo "Platform is macosx." - platform="macosx" -else - echo "Unrecognized platform." - exit 1 -fi - -if [[ $platform == "linux" ]]; then - # These commands must be kept in sync with the installation instructions. - sudo apt-get update - sudo apt-get install -y cmake build-essential autoconf libtool python-dev python-numpy libboost-all-dev -elif [[ $platform == "macosx" ]]; then - # check that brew is installed - which -s brew - if [[ $? != 0 ]]; then - echo "Could not find brew, please install brew (see http://brew.sh/)." - exit 1 - else - echo "Updating brew." - brew update - fi - # These commands must be kept in sync with the installation instructions. - brew install cmake automake autoconf libtool boost - sudo easy_install pip - sudo pip install numpy --ignore-installed six -fi diff --git a/numbuf/python/src/pynumbuf/adapters/numpy.cc b/numbuf/python/src/pynumbuf/adapters/numpy.cc index 4964f61f2..238eb0092 100644 --- a/numbuf/python/src/pynumbuf/adapters/numpy.cc +++ b/numbuf/python/src/pynumbuf/adapters/numpy.cc @@ -116,10 +116,7 @@ Status SerializeArray( // must be decremented. This is done in SerializeDict in python.cc. PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); Py_XDECREF(arglist); - if (!result) { - return Status::NotImplemented("python error"); // TODO(pcm): - // https://github.com/ray-project/numbuf/issues/10 - } + if (!result) { return Status::NotImplemented("python error"); } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); } diff --git a/numbuf/python/src/pynumbuf/adapters/python.cc b/numbuf/python/src/pynumbuf/adapters/python.cc index eada6e958..8bebe2810 100644 --- a/numbuf/python/src/pynumbuf/adapters/python.cc +++ b/numbuf/python/src/pynumbuf/adapters/python.cc @@ -128,10 +128,7 @@ Status append(PyObject* elem, SequenceBuilder& builder, std::vector& // must be decremented. This is done in SerializeDict in this file. PyObject* result = PyObject_CallObject(numbuf_serialize_callback, arglist); Py_XDECREF(arglist); - if (!result) { - return Status::NotImplemented("python error"); // TODO(pcm): - // https://github.com/ray-project/numbuf/issues/10 - } + if (!result) { return Status::NotImplemented("python error"); } builder.AppendDict(PyDict_Size(result)); subdicts.push_back(result); } @@ -285,10 +282,7 @@ Status DeserializeDict(std::shared_ptr array, int32_t start_idx, int32_t Py_XDECREF(arglist); Py_XDECREF(result); result = callback_result; - if (!callback_result) { - return Status::NotImplemented("python error"); // TODO(pcm): - // https://github.com/ray-project/numbuf/issues/10 - } + if (!callback_result) { return Status::NotImplemented("python error"); } } *out = result; return Status::OK(); diff --git a/numbuf/python/test/runtest.py b/numbuf/python/test/runtest.py index 4c9b409f6..56a1f0996 100644 --- a/numbuf/python/test/runtest.py +++ b/numbuf/python/test/runtest.py @@ -38,7 +38,7 @@ class SerializationTests(unittest.TestCase): self.roundTripTest([{"hello": [1, 2, 3]}]) self.roundTripTest([{"hello": [1, [2, 3]]}]) self.roundTripTest([{"hello": (None, 2, [3, 4])}]) - self.roundTripTest([{"hello": (None, 2, [3, 4], np.ndarray([1.0, 2.0, 3.0]))}]) + self.roundTripTest([{"hello": (None, 2, [3, 4], np.array([1.0, 2.0, 3.0]))}]) def numpyTest(self, t): a = np.random.randint(0, 10, size=(100, 100)).astype(t) diff --git a/numbuf/setup.py b/numbuf/setup.py index 066ca4d0c..26b4f208a 100644 --- a/numbuf/setup.py +++ b/numbuf/setup.py @@ -1,5 +1,5 @@ import subprocess -from setuptools import setup, find_packages, Extension +from setuptools import setup, find_packages import setuptools.command.install as _install from sys import platform @@ -23,7 +23,10 @@ class install(_install.install): setup(name="numbuf", version="0.0.1", packages=find_packages(), - package_data={"numbuf": ["libnumbuf.so", "libarrow" + extension, "libarrow_io" + extension, "libarrow_ipc" + extension]}, + package_data={"numbuf": ["libnumbuf.so", + "libarrow" + extension, + "libarrow_io" + extension, + "libarrow_ipc" + extension]}, cmdclass={"install": install}, setup_requires=["numpy"], include_package_data=True, diff --git a/numbuf/setup.sh b/numbuf/setup.sh index 71482975e..02a5acd73 100755 --- a/numbuf/setup.sh +++ b/numbuf/setup.sh @@ -15,7 +15,7 @@ else exit 1 fi -pushd "$ROOT_DIR/thirdparty" - ./download_thirdparty.sh - ./build_thirdparty.sh +pushd "$ROOT_DIR" + ./thirdparty/download_thirdparty.sh + ./thirdparty/build_thirdparty.sh popd diff --git a/numbuf/thirdparty/arrow b/numbuf/thirdparty/arrow deleted file mode 160000 index 58bd7bedc..000000000 --- a/numbuf/thirdparty/arrow +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 58bd7bedc63d66d5898297bab25b54dfb67665db diff --git a/numbuf/thirdparty/download_thirdparty.sh b/numbuf/thirdparty/download_thirdparty.sh index 044565252..c6a9ea19b 100755 --- a/numbuf/thirdparty/download_thirdparty.sh +++ b/numbuf/thirdparty/download_thirdparty.sh @@ -5,4 +5,6 @@ set -e TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -git submodule update --init --recursive -- "$TP_DIR/arrow" +git clone https://github.com/pcmoritz/arrow.git "$TP_DIR/arrow" +cd "$TP_DIR/arrow" +git checkout 58bd7bedc63d66d5898297bab25b54dfb67665db