wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-27 11:26:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[Streaming] Fault Tolerance Implementation (#10008)" (#10582)

Browse Source 
This reverts commit 1b1466748f.
This commit is contained in:
SangBin Cho
2020-09-04 13:21:18 -07:00
committed by GitHub
parent da83bbd764
commit cb919c5e5c
158 changed files with 1226 additions and 7039 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/python/includes/libstreaming.pxd
+4 -50
View File
@@ -11,9 +11,6 @@ from libcpp.vector cimport vector as c_vector
from libcpp.list cimport list as c_list
from cpython cimport PyObject
cimport cpython
from libcpp.unordered_map cimport unordered_map as c_unordered_map
from cython.operator cimport dereference, postincrement
cdef inline object PyObject_to_object(PyObject* o):
# Cast to "object" increments reference count
@@ -35,7 +32,7 @@ from ray.includes.unique_ids cimport (
CObjectID,
)
cdef extern from "common/status.h" namespace "ray::streaming" nogil:
cdef extern from "status.h" namespace "ray::streaming" nogil:
cdef cppclass CStreamingStatus "ray::streaming::StreamingStatus":
pass
cdef CStreamingStatus StatusOK "ray::streaming::StreamingStatus::OK"
@@ -73,21 +70,9 @@ cdef extern from "message/message.h" namespace "ray::streaming" nogil:
cdef CStreamingMessageType MessageTypeMessage "ray::streaming::StreamingMessageType::Message"
cdef cppclass CStreamingMessage "ray::streaming::StreamingMessage":
inline uint8_t *RawData() const
inline uint8_t *Payload() const
inline uint32_t PayloadSize() const
inline uint32_t GetDataSize() const
inline CStreamingMessageType GetMessageType() const
inline uint64_t GetMessageId() const
@staticmethod
inline void GetBarrierIdFromRawData(const uint8_t *data,
CStreamingBarrierHeader *barrier_header)
cdef struct CStreamingBarrierHeader "ray::streaming::StreamingBarrierHeader":
CStreamingBarrierType barrier_type;
uint64_t barrier_id;
cdef cppclass CStreamingBarrierType "ray::streaming::StreamingBarrierType":
pass
cdef uint32_t kMessageHeaderSize;
cdef uint32_t kBarrierHeaderSize;
inline uint64_t GetMessageSeqId() const
cdef extern from "message/message_bundle.h" namespace "ray::streaming" nogil:
cdef cppclass CStreamingMessageBundleType "ray::streaming::StreamingMessageBundleType":
@@ -112,40 +97,13 @@ cdef extern from "message/message_bundle.h" namespace "ray::streaming" nogil:
void GetMessageListFromRawData(const uint8_t *data, uint32_t size, uint32_t msg_nums,
c_list[shared_ptr[CStreamingMessage]] &msg_list);
cdef extern from "channel/channel.h" namespace "ray::streaming" nogil:
cdef extern from "channel.h" namespace "ray::streaming" nogil:
cdef struct CChannelCreationParameter "ray::streaming::ChannelCreationParameter":
CChannelCreationParameter()
CActorID actor_id;
shared_ptr[CRayFunction] async_function;
shared_ptr[CRayFunction] sync_function;
cdef struct CStreamingQueueInfo "ray::streaming::StreamingQueueInfo":
uint64_t first_seq_id;
uint64_t last_message_id;
uint64_t target_message_id;
uint64_t consumed_message_id;
cdef struct CConsumerChannelInfo "ray::streaming::ConsumerChannelInfo":
CObjectID channel_id;
uint64_t current_message_id;
uint64_t barrier_id;
uint64_t partial_barrier_id;
CStreamingQueueInfo queue_info;
uint64_t last_queue_item_delay;
uint64_t last_queue_item_latency;
uint64_t last_queue_target_diff;
uint64_t get_queue_item_times;
uint64_t notify_cnt;
CChannelCreationParameter parameter;
cdef enum CTransferCreationStatus "ray::streaming::TransferCreationStatus":
FreshStarted = 0
PullOk = 1
Timeout = 2
DataLost = 3
Invalid = 999
cdef extern from "queue/queue_client.h" namespace "ray::streaming" nogil:
cdef cppclass CReaderClient "ray::streaming::ReaderClient":
CReaderClient()
@@ -170,12 +128,11 @@ cdef extern from "data_reader.h" namespace "ray::streaming" nogil:
CDataReader(shared_ptr[CRuntimeContext] &runtime_context)
void Init(const c_vector[CObjectID] &input_ids,
const c_vector[CChannelCreationParameter] &params,
const c_vector[uint64_t] &seq_ids,
const c_vector[uint64_t] &msg_ids,
c_vector[CTransferCreationStatus] &creation_status,
int64_t timer_interval);
CStreamingStatus GetBundle(const uint32_t timeout_ms,
shared_ptr[CDataBundle] &message)
void GetOffsetInfo(c_unordered_map[CObjectID, CConsumerChannelInfo] *&offset_map);
void Stop()
@@ -188,9 +145,6 @@ cdef extern from "data_writer.h" namespace "ray::streaming" nogil:
const c_vector[uint64_t] &queue_size_vec);
long WriteMessageToBufferRing(
const CObjectID &q_id, uint8_t *data, uint32_t data_size)
void BroadcastBarrier(uint64_t checkpoint_id, const uint8_t *data, uint32_t data_size)
void GetChannelOffset(c_vector[uint64_t] &result)
void ClearCheckpoint(uint64_t checkpoint_id)
void Run()
void Stop()