wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-30 12:30:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Capability to serialize most primitive Python types

Browse Source
This commit is contained in:
Philipp Moritz
2016-07-20 21:47:37 -07:00
parent b143b36792
commit 9bc7e7843f
16 changed files with 1281 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cpp/src/numbuf/sequence.cc
+167
View File
@@ -0,0 +1,167 @@
#include "sequence.h"
using namespace arrow;
namespace numbuf {
SequenceBuilder::SequenceBuilder(MemoryPool* pool)
: pool_(pool), types_(pool), offsets_(pool),
nones_(pool, std::make_shared<NullType>()),
bools_(pool, std::make_shared<BooleanType>()),
ints_(pool), strings_(pool, std::make_shared<StringType>()),
floats_(pool), doubles_(pool),
uint8_tensors_(std::make_shared<UInt8Type>(), pool),
int8_tensors_(std::make_shared<Int8Type>(), pool),
uint16_tensors_(std::make_shared<UInt16Type>(), pool),
int16_tensors_(std::make_shared<Int16Type>(), pool),
uint32_tensors_(std::make_shared<UInt32Type>(), pool),
int32_tensors_(std::make_shared<Int32Type>(), pool),
uint64_tensors_(std::make_shared<UInt64Type>(), pool),
int64_tensors_(std::make_shared<Int64Type>(), pool),
float_tensors_(std::make_shared<FloatType>(), pool),
double_tensors_(std::make_shared<DoubleType>(), pool),
list_offsets_({0}), tuple_offsets_({0}), dict_offsets_({0}) {}
#define UPDATE(OFFSET, TAG) \
if (TAG == -1) { \
TAG = num_tags; \
num_tags += 1; \
} \
RETURN_NOT_OK(offsets_.Append(OFFSET)); \
RETURN_NOT_OK(types_.Append(TAG)); \
RETURN_NOT_OK(nones_.AppendToBitmap(true));
Status SequenceBuilder::Append() {
RETURN_NOT_OK(offsets_.Append(0));
RETURN_NOT_OK(types_.Append(0));
return nones_.AppendToBitmap(false);
}
Status SequenceBuilder::Append(bool data) {
UPDATE(bools_.length(), bool_tag);
return bools_.Append(data);
}
Status SequenceBuilder::Append(int64_t data) {
UPDATE(ints_.length(), int_tag);
return ints_.Append(data);
}
Status SequenceBuilder::Append(uint64_t data) {
UPDATE(ints_.length(), int_tag);
return ints_.Append(data);
}
Status SequenceBuilder::Append(const char* data, int32_t length) {
UPDATE(strings_.length(), string_tag);
return strings_.Append(data, length);
}
Status SequenceBuilder::Append(float data) {
UPDATE(floats_.length(), float_tag);
return floats_.Append(data);
}
Status SequenceBuilder::Append(double data) {
UPDATE(doubles_.length(), double_tag);
return doubles_.Append(data);
}
#define DEF_TENSOR_APPEND(NAME, TYPE, TAG) \
Status SequenceBuilder::Append(const std::vector<int64_t>& dims, TYPE* data) { \
UPDATE(NAME.length(), TAG); \
return NAME.Append(dims, data); \
}
DEF_TENSOR_APPEND(uint8_tensors_, uint8_t, uint8_tensor_tag);
DEF_TENSOR_APPEND(int8_tensors_, int8_t, int8_tensor_tag);
DEF_TENSOR_APPEND(uint16_tensors_, uint16_t, uint16_tensor_tag);
DEF_TENSOR_APPEND(int16_tensors_, int16_t, int16_tensor_tag);
DEF_TENSOR_APPEND(uint32_tensors_, uint32_t, uint32_tensor_tag);
DEF_TENSOR_APPEND(int32_tensors_, int32_t, int32_tensor_tag);
DEF_TENSOR_APPEND(uint64_tensors_, uint64_t, uint64_tensor_tag);
DEF_TENSOR_APPEND(int64_tensors_, int64_t, int64_tensor_tag);
DEF_TENSOR_APPEND(float_tensors_, float, float_tensor_tag);
DEF_TENSOR_APPEND(double_tensors_, double, double_tensor_tag);
Status SequenceBuilder::AppendList(int32_t size) {
UPDATE(list_offsets_.size() - 1, list_tag);
list_offsets_.push_back(list_offsets_.back() + size);
return Status::OK();
}
Status SequenceBuilder::AppendTuple(int32_t size) {
UPDATE(tuple_offsets_.size() - 1, tuple_tag);
tuple_offsets_.push_back(tuple_offsets_.back() + size);
return Status::OK();
}
Status SequenceBuilder::AppendDict(int32_t size) {
UPDATE(dict_offsets_.size() - 1, dict_tag);
dict_offsets_.push_back(dict_offsets_.back() + size);
return Status::OK();
}
#define ADD_ELEMENT(VARNAME, TAG) \
if (TAG != -1) { \
types[TAG] = VARNAME.type(); \
children[TAG] = VARNAME.Finish(); \
ARROW_CHECK_OK(nones_.AppendToBitmap(true)); \
}
#define ADD_SUBSEQUENCE(DATA, OFFSETS, BUILDER, TAG, NAME) \
if (DATA) { \
DCHECK(DATA->length() == OFFSETS.back()); \
auto list_builder = std::make_shared<ListBuilder>(pool_, DATA); \
auto field = std::make_shared<Field>(NAME, list_builder->type()); \
auto type = std::make_shared<StructType>(std::vector<FieldPtr>({field})); \
auto lists = std::vector<std::shared_ptr<ArrayBuilder>>({list_builder}); \
StructBuilder builder(pool_, type, lists); \
OFFSETS.pop_back(); \
ARROW_CHECK_OK(list_builder->Append(OFFSETS.data(), OFFSETS.size())); \
builder.Append(); \
ADD_ELEMENT(builder, TAG); \
} else { \
DCHECK(OFFSETS.size() == 1); \
}
std::shared_ptr<DenseUnionArray> SequenceBuilder::Finish(
std::shared_ptr<Array> list_data,
std::shared_ptr<Array> tuple_data,
std::shared_ptr<Array> dict_data) {
std::vector<TypePtr> types(num_tags);
std::vector<ArrayPtr> children(num_tags);
ADD_ELEMENT(bools_, bool_tag);
ADD_ELEMENT(ints_, int_tag);
ADD_ELEMENT(strings_, string_tag);
ADD_ELEMENT(floats_, float_tag);
ADD_ELEMENT(doubles_, double_tag);
ADD_ELEMENT(uint8_tensors_, uint8_tensor_tag);
ADD_ELEMENT(int8_tensors_, int8_tensor_tag);
ADD_ELEMENT(uint16_tensors_, uint16_tensor_tag);
ADD_ELEMENT(int16_tensors_, int16_tensor_tag);
ADD_ELEMENT(uint32_tensors_, uint32_tensor_tag);
ADD_ELEMENT(int32_tensors_, int32_tensor_tag);
ADD_ELEMENT(uint64_tensors_, uint64_tensor_tag);
ADD_ELEMENT(int64_tensors_, int64_tensor_tag);
ADD_ELEMENT(float_tensors_, float_tensor_tag);
ADD_ELEMENT(double_tensors_, double_tensor_tag);
ADD_SUBSEQUENCE(list_data, list_offsets_, list_builder, list_tag, "list");
ADD_SUBSEQUENCE(tuple_data, tuple_offsets_, tuple_builder, tuple_tag, "tuple");
ADD_SUBSEQUENCE(dict_data, dict_offsets_, dict_builder, dict_tag, "dict");
TypePtr type = TypePtr(new DenseUnionType(types));
return std::make_shared<DenseUnionArray>(type, types_.length(),
children, types_.data(), offsets_.data(),
nones_.null_count(), nones_.null_bitmap());
}
}