wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-12 18:32:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Support streaming flow control (#7152)

Browse Source 
* streaming writer use event driven model.

* add RefreshChannelInfo

* fix name

* minor changes according reviewer comments

* Fix according to reviewer's comments

* fix bazel lint

* code polished

* Add more comments

* rename Stop & Start of EventQueue to Freeze and Unfreeze.

* add override

* fix

* fix return value

* support flow control

* add flow control ut in mock transfer

* minor changes according to comments

* add java and python worker adaption

Co-authored-by: wanxing <wanxing.wwx@alibaba-inc.com>
This commit is contained in:
Lingxuan Zuo
2020-02-24 23:48:04 +08:00
committed by GitHub
parent 7e115490d9
commit f995099e00
16 changed files with 410 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/test/mock_transfer_tests.cc
+54 -1
View File
@@ -37,7 +37,7 @@ TEST(StreamingMockTransfer, mock_produce_consume) {
class StreamingTransferTest : public ::testing::Test {
public:
StreamingTransferTest() {
std::shared_ptr<RuntimeContext> runtime_context(new RuntimeContext());
runtime_context = std::make_shared<RuntimeContext>();
runtime_context->MarkMockTest();
writer = std::make_shared<DataWriter>(runtime_context);
reader = std::make_shared<DataReader>(runtime_context);
@@ -64,6 +64,7 @@ class StreamingTransferTest : public ::testing::Test {
std::shared_ptr<DataWriter> writer;
std::shared_ptr<DataReader> reader;
std::vector<ObjectID> queue_vec;
std::shared_ptr<RuntimeContext> runtime_context;
};
TEST_F(StreamingTransferTest, exchange_single_channel_test) {
@@ -130,6 +131,58 @@ TEST_F(StreamingTransferTest, exchange_consumed_test) {
write_thread.join();
}
TEST_F(StreamingTransferTest, flow_control_test) {
InitTransfer();
writer->Run();
uint32_t data_size = 8196;
std::shared_ptr<uint8_t> data(new uint8_t[data_size]);
auto func = [data, data_size](int index) { std::fill_n(data.get(), data_size, index); };
size_t num = 10000;
std::thread write_thread([this, data, data_size, &func, num]() {
for (size_t i = 0; i < num; ++i) {
func(i);
writer->WriteMessageToBufferRing(queue_vec[0], data.get(), data_size);
}
});
std::unordered_map<ObjectID, ProducerChannelInfo> *writer_offset_info = nullptr;
std::unordered_map<ObjectID, ConsumerChannelInfo> *reader_offset_info = nullptr;
writer->GetOffsetInfo(writer_offset_info);
reader->GetOffsetInfo(reader_offset_info);
uint32_t writer_step = runtime_context->GetConfig().GetWriterConsumedStep();
uint32_t reader_step = runtime_context->GetConfig().GetReaderConsumedStep();
uint64_t &writer_current_seq_id = (*writer_offset_info)[queue_vec[0]].current_seq_id;
uint64_t &writer_current_message_id =
(*writer_offset_info)[queue_vec[0]].current_message_id;
uint64_t &reader_target_seq_id =
(*reader_offset_info)[queue_vec[0]].queue_info.target_seq_id;
while (writer_current_seq_id < writer_step) {
STREAMING_LOG(INFO) << "Writer currrent seq id " << writer_current_seq_id
<< " message " << writer_current_message_id << " consumer step "
<< writer_step;
std::this_thread::sleep_for(
std::chrono::milliseconds(StreamingConfig::TIME_WAIT_UINT));
}
std::list<StreamingMessagePtr> read_message_list;
while (read_message_list.size() < num) {
std::shared_ptr<DataBundle> msg;
reader->GetBundle(5000, msg);
StreamingMessageBundlePtr bundle_ptr = StreamingMessageBundle::FromBytes(msg->data);
auto &message_list = bundle_ptr->GetMessageList();
std::copy(message_list.begin(), message_list.end(),
std::back_inserter(read_message_list));
ASSERT_GE(writer_step, writer_current_seq_id - msg->seq_id);
ASSERT_GE(msg->seq_id + reader_step, reader_target_seq_id);
}
int index = 0;
for (auto &message : read_message_list) {
func(index++);
EXPECT_EQ(std::memcmp(message->RawData(), data.get(), data_size), 0);
}
write_thread.join();
}
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();