From f468871b5d3f35de633a01e5aad1a61748e5ef9d Mon Sep 17 00:00:00 2001 From: William Paul Date: Wed, 24 Aug 2016 23:31:09 +0000 Subject: [PATCH 1/2] Tuples as keys in dicts can be serialized --- cpp/src/numbuf/dict.cc | 3 ++- cpp/src/numbuf/dict.h | 1 + python/src/pynumbuf/adapters/python.cc | 10 +++++++--- python/test/runtest.py | 5 +++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cpp/src/numbuf/dict.cc b/cpp/src/numbuf/dict.cc index ff5f7239f..14bf7f2f3 100644 --- a/cpp/src/numbuf/dict.cc +++ b/cpp/src/numbuf/dict.cc @@ -5,12 +5,13 @@ using namespace arrow; namespace numbuf { std::shared_ptr DictBuilder::Finish( + std::shared_ptr key_tuple_data, std::shared_ptr list_data, std::shared_ptr tuple_data, std::shared_ptr dict_data) { // lists and dicts can't be keys of dicts in Python, that is why for // the keys we do not need to collect sublists - auto keys = keys_.Finish(nullptr, nullptr, nullptr); + auto keys = keys_.Finish(nullptr, key_tuple_data, nullptr); auto vals = vals_.Finish(list_data, tuple_data, dict_data); auto keys_field = std::make_shared("keys", keys->type()); auto vals_field = std::make_shared("vals", vals->type()); diff --git a/cpp/src/numbuf/dict.h b/cpp/src/numbuf/dict.h index 5c2abeacd..02eb6035d 100644 --- a/cpp/src/numbuf/dict.h +++ b/cpp/src/numbuf/dict.h @@ -34,6 +34,7 @@ public: value list of the dictionary */ std::shared_ptr Finish( + std::shared_ptr key_tuple_data, std::shared_ptr list_data, std::shared_ptr tuple_data, std::shared_ptr dict_data); diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 790fd5497..871e0e097 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -171,16 +171,20 @@ Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t Status SerializeDict(std::vector dicts, std::shared_ptr* out) { DictBuilder result; - std::vector sublists, subtuples, subdicts, dummy; + std::vector sublists, subtuples, subdicts, keytuples, dummy; for (const auto& dict : dicts) { PyObject *key, *value; Py_ssize_t pos = 0; while (PyDict_Next(dict, &pos, &key, &value)) { - RETURN_NOT_OK(append(key, result.keys(), dummy, dummy, dummy)); + RETURN_NOT_OK(append(key, result.keys(), dummy, keytuples, dummy)); DCHECK(dummy.size() == 0); RETURN_NOT_OK(append(value, result.vals(), sublists, subtuples, subdicts)); } } + std::shared_ptr key_val_tuples; + if (keytuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(keytuples, &key_val_tuples)); + } std::shared_ptr val_list; if (sublists.size() > 0) { RETURN_NOT_OK(SerializeSequences(sublists, &val_list)); @@ -193,7 +197,7 @@ Status SerializeDict(std::vector dicts, std::shared_ptr* out) if (subdicts.size() > 0) { RETURN_NOT_OK(SerializeDict(subdicts, &val_dict)); } - *out = result.Finish(val_list, val_tuples, val_dict); + *out = result.Finish(key_val_tuples, val_list, val_tuples, val_dict); return Status::OK(); } diff --git a/python/test/runtest.py b/python/test/runtest.py index 1bb4addba..f8966b5ef 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -3,8 +3,9 @@ import libnumbuf import numpy as np from numpy.testing import assert_equal -TEST_OBJECTS = [[1, "hello", 3.0], 42, 43L, "hello world", u"x", u"\u262F", - 42.0, 1L << 62, (1.0, "hi"), +TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world", + u"x", u"\u262F", 42.0, + 1L << 62, (1.0, "hi"), None, (None, None), ("hello", None), True, False, (True, False), "hello", {True: "hello", False: "world"}, From 5c409c391667cedf872a549dc5ba7a429e26ed17 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 25 Aug 2016 16:04:31 -0700 Subject: [PATCH 2/2] make variable names more uniform --- cpp/src/numbuf/dict.cc | 8 +++---- cpp/src/numbuf/dict.h | 6 ++--- python/src/pynumbuf/adapters/python.cc | 32 +++++++++++++------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cpp/src/numbuf/dict.cc b/cpp/src/numbuf/dict.cc index 14bf7f2f3..98b03894b 100644 --- a/cpp/src/numbuf/dict.cc +++ b/cpp/src/numbuf/dict.cc @@ -6,13 +6,13 @@ namespace numbuf { std::shared_ptr DictBuilder::Finish( std::shared_ptr key_tuple_data, - std::shared_ptr list_data, - std::shared_ptr tuple_data, - std::shared_ptr dict_data) { + std::shared_ptr val_list_data, + std::shared_ptr val_tuple_data, + std::shared_ptr val_dict_data) { // lists and dicts can't be keys of dicts in Python, that is why for // the keys we do not need to collect sublists auto keys = keys_.Finish(nullptr, key_tuple_data, nullptr); - auto vals = vals_.Finish(list_data, tuple_data, dict_data); + auto vals = vals_.Finish(val_list_data, val_tuple_data, val_dict_data); auto keys_field = std::make_shared("keys", keys->type()); auto vals_field = std::make_shared("vals", vals->type()); auto type = std::make_shared(std::vector({keys_field, vals_field})); diff --git a/cpp/src/numbuf/dict.h b/cpp/src/numbuf/dict.h index 02eb6035d..dcb38fb6b 100644 --- a/cpp/src/numbuf/dict.h +++ b/cpp/src/numbuf/dict.h @@ -35,9 +35,9 @@ public: */ std::shared_ptr Finish( std::shared_ptr key_tuple_data, - std::shared_ptr list_data, - std::shared_ptr tuple_data, - std::shared_ptr dict_data); + std::shared_ptr val_list_data, + std::shared_ptr val_tuple_data, + std::shared_ptr val_dict_data); private: SequenceBuilder keys_; diff --git a/python/src/pynumbuf/adapters/python.cc b/python/src/pynumbuf/adapters/python.cc index 871e0e097..ba2c8b0e3 100644 --- a/python/src/pynumbuf/adapters/python.cc +++ b/python/src/pynumbuf/adapters/python.cc @@ -171,33 +171,33 @@ Status DeserializeTuple(std::shared_ptr array, int32_t start_idx, int32_t Status SerializeDict(std::vector dicts, std::shared_ptr* out) { DictBuilder result; - std::vector sublists, subtuples, subdicts, keytuples, dummy; + std::vector key_tuples, val_lists, val_tuples, val_dicts, dummy; for (const auto& dict : dicts) { PyObject *key, *value; Py_ssize_t pos = 0; while (PyDict_Next(dict, &pos, &key, &value)) { - RETURN_NOT_OK(append(key, result.keys(), dummy, keytuples, dummy)); + RETURN_NOT_OK(append(key, result.keys(), dummy, key_tuples, dummy)); DCHECK(dummy.size() == 0); - RETURN_NOT_OK(append(value, result.vals(), sublists, subtuples, subdicts)); + RETURN_NOT_OK(append(value, result.vals(), val_lists, val_tuples, val_dicts)); } } - std::shared_ptr key_val_tuples; - if (keytuples.size() > 0) { - RETURN_NOT_OK(SerializeSequences(keytuples, &key_val_tuples)); + std::shared_ptr key_tuples_arr; + if (key_tuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(key_tuples, &key_tuples_arr)); } - std::shared_ptr val_list; - if (sublists.size() > 0) { - RETURN_NOT_OK(SerializeSequences(sublists, &val_list)); + std::shared_ptr val_list_arr; + if (val_lists.size() > 0) { + RETURN_NOT_OK(SerializeSequences(val_lists, &val_list_arr)); } - std::shared_ptr val_tuples; - if (subtuples.size() > 0) { - RETURN_NOT_OK(SerializeSequences(subtuples, &val_tuples)); + std::shared_ptr val_tuples_arr; + if (val_tuples.size() > 0) { + RETURN_NOT_OK(SerializeSequences(val_tuples, &val_tuples_arr)); } - std::shared_ptr val_dict; - if (subdicts.size() > 0) { - RETURN_NOT_OK(SerializeDict(subdicts, &val_dict)); + std::shared_ptr val_dict_arr; + if (val_dicts.size() > 0) { + RETURN_NOT_OK(SerializeDict(val_dicts, &val_dict_arr)); } - *out = result.Finish(key_val_tuples, val_list, val_tuples, val_dict); + *out = result.Finish(key_tuples_arr, val_list_arr, val_tuples_arr, val_dict_arr); return Status::OK(); }