wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-11 11:26:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement unicode serialization

Browse Source
This commit is contained in:
Philipp Moritz
2016-08-04 15:34:47 -07:00
parent a81dc0c541
commit 8e165d43d4
6 changed files with 85 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cpp/src/numbuf/sequence.cc
+20 -12
View File
@@ -8,7 +8,9 @@ SequenceBuilder::SequenceBuilder(MemoryPool* pool)
: pool_(pool), types_(pool), offsets_(pool),
nones_(pool, std::make_shared<NullType>()),
bools_(pool, std::make_shared<BooleanType>()),
ints_(pool), strings_(pool, std::make_shared<StringType>()),
ints_(pool),
bytes_(pool, std::make_shared<BinaryType>()),
strings_(pool, std::make_shared<StringType>()),
floats_(pool), doubles_(pool),
uint8_tensors_(std::make_shared<UInt8Type>(), pool),
int8_tensors_(std::make_shared<Int8Type>(), pool),
@@ -31,46 +33,51 @@ SequenceBuilder::SequenceBuilder(MemoryPool* pool)
RETURN_NOT_OK(types_.Append(TAG)); \
RETURN_NOT_OK(nones_.AppendToBitmap(true));
Status SequenceBuilder::Append() {
Status SequenceBuilder::AppendNone() {
RETURN_NOT_OK(offsets_.Append(0));
RETURN_NOT_OK(types_.Append(0));
return nones_.AppendToBitmap(false);
}
Status SequenceBuilder::Append(bool data) {
Status SequenceBuilder::AppendBool(bool data) {
UPDATE(bools_.length(), bool_tag);
return bools_.Append(data);
}
Status SequenceBuilder::Append(int64_t data) {
Status SequenceBuilder::AppendInt64(int64_t data) {
UPDATE(ints_.length(), int_tag);
return ints_.Append(data);
}
Status SequenceBuilder::Append(uint64_t data) {
Status SequenceBuilder::AppendUInt64(uint64_t data) {
UPDATE(ints_.length(), int_tag);
return ints_.Append(data);
}
Status SequenceBuilder::Append(const char* data, int32_t length) {
Status SequenceBuilder::AppendBytes(const uint8_t* data, int32_t length) {
UPDATE(bytes_.length(), bytes_tag);
return bytes_.Append(data, length);
}
Status SequenceBuilder::AppendString(const char* data, int32_t length) {
UPDATE(strings_.length(), string_tag);
return strings_.Append(data, length);
}
Status SequenceBuilder::Append(float data) {
Status SequenceBuilder::AppendFloat(float data) {
UPDATE(floats_.length(), float_tag);
return floats_.Append(data);
}
Status SequenceBuilder::Append(double data) {
Status SequenceBuilder::AppendDouble(double data) {
UPDATE(doubles_.length(), double_tag);
return doubles_.Append(data);
}
#define DEF_TENSOR_APPEND(NAME, TYPE, TAG) \
Status SequenceBuilder::Append(const std::vector<int64_t>& dims, TYPE* data) { \
UPDATE(NAME.length(), TAG); \
return NAME.Append(dims, data); \
#define DEF_TENSOR_APPEND(NAME, TYPE, TAG) \
Status SequenceBuilder::AppendTensor(const std::vector<int64_t>& dims, TYPE* data) { \
UPDATE(NAME.length(), TAG); \
return NAME.Append(dims, data); \
}
DEF_TENSOR_APPEND(uint8_tensors_, uint8_t, uint8_tensor_tag);
@@ -136,6 +143,7 @@ std::shared_ptr<DenseUnionArray> SequenceBuilder::Finish(
ADD_ELEMENT(bools_, bool_tag);
ADD_ELEMENT(ints_, int_tag);
ADD_ELEMENT(strings_, string_tag);
ADD_ELEMENT(bytes_, bytes_tag);
ADD_ELEMENT(floats_, float_tag);
ADD_ELEMENT(doubles_, double_tag);
cpp/src/numbuf/sequence.h
+22 -17
View File
@@ -15,25 +15,28 @@ class SequenceBuilder {
SequenceBuilder(arrow::MemoryPool* pool = nullptr);
//! Appending a none to the sequence
arrow::Status Append();
arrow::Status AppendNone();
//! Appending a boolean to the sequence
arrow::Status Append(bool data);
arrow::Status AppendBool(bool data);
//! Appending an int64_t to the sequence
arrow::Status Append(int64_t data);
arrow::Status AppendInt64(int64_t data);
//! Appending an uint64_t to the sequence
arrow::Status Append(uint64_t data);
arrow::Status AppendUInt64(uint64_t data);
//! Append a list of bytes to the sequence
arrow::Status AppendBytes(const uint8_t* data, int32_t length);
//! Appending a string to the sequence
arrow::Status Append(const char* data, int32_t length);
arrow::Status AppendString(const char* data, int32_t length);
//! Appending a float to the sequence
arrow::Status Append(float data);
arrow::Status AppendFloat(float data);
//! Appending a double to the sequence
arrow::Status Append(double data);
arrow::Status AppendDouble(double data);
/*! Appending a tensor to the sequence
@@ -44,16 +47,16 @@ class SequenceBuilder {
A pointer to the start of the data block. The length of the data block
will be the product of the dimensions
*/
arrow::Status Append(const std::vector<int64_t>& dims, uint8_t* data);
arrow::Status Append(const std::vector<int64_t>& dims, int8_t* data);
arrow::Status Append(const std::vector<int64_t>& dims, uint16_t* data);
arrow::Status Append(const std::vector<int64_t>& dims, int16_t* data);
arrow::Status Append(const std::vector<int64_t>& dims, uint32_t* data);
arrow::Status Append(const std::vector<int64_t>& dims, int32_t* data);
arrow::Status Append(const std::vector<int64_t>& dims, uint64_t* data);
arrow::Status Append(const std::vector<int64_t>& dims, int64_t* data);
arrow::Status Append(const std::vector<int64_t>& dims, float* data);
arrow::Status Append(const std::vector<int64_t>& dims, double* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, uint8_t* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, int8_t* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, uint16_t* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, int16_t* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, uint32_t* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, int32_t* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, uint64_t* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, int64_t* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, float* data);
arrow::Status AppendTensor(const std::vector<int64_t>& dims, double* data);
/*! Add a sublist to the sequenc. The data contained in the sublist will be
specified in the "Finish" method.
@@ -90,6 +93,7 @@ class SequenceBuilder {
arrow::NullArrayBuilder nones_;
arrow::BooleanBuilder bools_;
arrow::Int64Builder ints_;
arrow::BinaryBuilder bytes_;
arrow::StringBuilder strings_;
arrow::FloatBuilder floats_;
arrow::DoubleBuilder doubles_;
@@ -112,6 +116,7 @@ class SequenceBuilder {
int8_t bool_tag = -1;
int8_t int_tag = -1;
int8_t string_tag = -1;
int8_t bytes_tag = -1;
int8_t float_tag = -1;
int8_t double_tag = -1;