#define BOOST_BIND_NO_PLACEHOLDERS #include "common/status.h" #include "data_reader.h" #include "data_writer.h" #include "gtest/gtest.h" #include "message/message.h" #include "message/message_bundle.h" #include "queue/queue_client.h" #include "ray/common/test_util.h" #include "ray/core_worker/context.h" #include "ray/core_worker/core_worker.h" #include "ring_buffer/ring_buffer.h" using namespace std::placeholders; const uint32_t MESSAGE_BOUND_SIZE = 10000; const uint32_t DEFAULT_STREAMING_MESSAGE_BUFFER_SIZE = 1000; namespace ray { namespace streaming { class StreamingQueueTestSuite { public: StreamingQueueTestSuite(ActorID &peer_actor_id, std::vector queue_ids, std::vector rescale_queue_ids) : peer_actor_id_(peer_actor_id), queue_ids_(queue_ids), rescale_queue_ids_(rescale_queue_ids) {} virtual void ExecuteTest(std::string test_name) { auto it = test_func_map_.find(test_name); STREAMING_CHECK(it != test_func_map_.end()); current_test_ = test_name; status_ = false; auto func = it->second; executor_thread_ = std::make_shared(func); executor_thread_->detach(); } virtual std::shared_ptr CheckCurTestStatus() { TestCheckStatusRspMsg msg(current_test_, status_); return msg.ToBytes(); } virtual bool TestDone() { return status_; } virtual ~StreamingQueueTestSuite() {} protected: std::unordered_map> test_func_map_; std::string current_test_; bool status_; std::shared_ptr executor_thread_; ActorID peer_actor_id_; std::vector queue_ids_; std::vector rescale_queue_ids_; }; class StreamingQueueWriterTestSuite : public StreamingQueueTestSuite { public: StreamingQueueWriterTestSuite(ActorID &peer_actor_id, std::vector queue_ids, std::vector rescale_queue_ids) : StreamingQueueTestSuite(peer_actor_id, queue_ids, rescale_queue_ids) { test_func_map_ = { {"streaming_writer_exactly_once_test", std::bind(&StreamingQueueWriterTestSuite::StreamingWriterExactlyOnceTest, this)}}; } private: void StreamingWriterExactlyOnceTest() { StreamingConfig config; StreamingWriterStrategyTest(config); STREAMING_LOG(INFO) << "StreamingQueueWriterTestSuite::StreamingWriterExactlyOnceTest"; status_ = true; } void StreamingWriterStrategyTest(StreamingConfig &config) { for (auto &queue_id : queue_ids_) { STREAMING_LOG(INFO) << "queue_id: " << queue_id; } ChannelCreationParameter param{ peer_actor_id_, std::make_shared( ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "reader_async_call_func", ""})), std::make_shared( ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "reader_sync_call_func", ""}))}; std::vector params(queue_ids_.size(), param); STREAMING_LOG(INFO) << "writer actor_ids size: " << params.size() << " actor_id: " << peer_actor_id_; std::shared_ptr runtime_context(new RuntimeContext()); runtime_context->SetConfig(config); // Create writer. std::shared_ptr streaming_writer_client(new DataWriter(runtime_context)); uint64_t queue_size = 10 * 1000 * 1000; std::vector channel_seq_id_vec(queue_ids_.size(), 0); streaming_writer_client->Init(queue_ids_, params, channel_seq_id_vec, std::vector(queue_ids_.size(), queue_size)); STREAMING_LOG(INFO) << "streaming_writer_client Init done"; streaming_writer_client->Run(); // Write some data. std::thread test_loop_thread( &StreamingQueueWriterTestSuite::TestWriteMessageToBufferRing, this, streaming_writer_client, std::ref(queue_ids_)); if (test_loop_thread.joinable()) { test_loop_thread.join(); } } void TestWriteMessageToBufferRing(std::shared_ptr writer_client, std::vector &q_list) { uint32_t i = 1; while (i <= MESSAGE_BOUND_SIZE) { for (auto &q_id : q_list) { uint64_t buffer_len = (i % DEFAULT_STREAMING_MESSAGE_BUFFER_SIZE); uint8_t *data = new uint8_t[buffer_len]; for (uint32_t j = 0; j < buffer_len; ++j) { data[j] = j % 128; } STREAMING_LOG(DEBUG) << "Write data to queue, count=" << i << ", queue_id=" << q_id; writer_client->WriteMessageToBufferRing(q_id, data, buffer_len, StreamingMessageType::Message); if (i % 10 == 0) { writer_client->BroadcastBarrier(i / 10, nullptr, 0); } } ++i; } STREAMING_LOG(INFO) << "Write data done."; // Wait a while. std::this_thread::sleep_for(std::chrono::milliseconds(5000)); } }; class StreamingQueueReaderTestSuite : public StreamingQueueTestSuite { public: StreamingQueueReaderTestSuite(ActorID peer_actor_id, std::vector queue_ids, std::vector rescale_queue_ids) : StreamingQueueTestSuite(peer_actor_id, queue_ids, rescale_queue_ids) { test_func_map_ = { {"streaming_writer_exactly_once_test", std::bind(&StreamingQueueReaderTestSuite::StreamingWriterExactlyOnceTest, this)}}; } private: void ReaderLoopForward(std::shared_ptr reader_client, std::shared_ptr writer_client, std::vector &queue_id_vec) { uint64_t recevied_message_cnt = 0; std::unordered_map queue_last_cp_id; for (auto &q_id : queue_id_vec) { queue_last_cp_id[q_id] = 0; } STREAMING_LOG(INFO) << "Start read message bundle, queue_id_size=" << queue_id_vec.size(); while (true) { std::shared_ptr msg; StreamingStatus st = reader_client->GetBundle(100, msg); if (st != StreamingStatus::OK || !msg->data) { STREAMING_LOG(DEBUG) << "read bundle timeout, status = " << (int)st; continue; } STREAMING_CHECK(msg.get() && msg->meta.get()) << "read null pointer message, queue id => " << msg->from.Hex(); if (msg->meta->GetBundleType() == StreamingMessageBundleType::Barrier) { StreamingBarrierHeader barrier_header; StreamingMessage::GetBarrierIdFromRawData(msg->data + kMessageHeaderSize, &barrier_header); STREAMING_LOG(DEBUG) << "barrier message recevied, time=" << msg->meta->GetMessageBundleTs() << ", barrier_id=" << barrier_header.barrier_id << ", data=" << Util::Byte2hex(msg->data, msg->data_size); std::unordered_map *offset_map; reader_client->GetOffsetInfo(offset_map); for (auto &q_id : queue_id_vec) { reader_client->NotifyConsumedItem((*offset_map)[q_id], (*offset_map)[q_id].current_message_id); } // writer_client->ClearCheckpoint(msg->last_barrier_id); continue; } else if (msg->meta->GetBundleType() == StreamingMessageBundleType::Empty) { STREAMING_LOG(DEBUG) << "empty message recevied => " << msg->meta->GetMessageBundleTs(); continue; } StreamingMessageBundlePtr bundlePtr; bundlePtr = StreamingMessageBundle::FromBytes(msg->data); std::list message_list; bundlePtr->GetMessageList(message_list); STREAMING_LOG(INFO) << "message size => " << message_list.size() << " from queue id => " << msg->from.Hex() << " last message id => " << msg->meta->GetLastMessageId(); recevied_message_cnt += message_list.size(); for (auto &item : message_list) { uint64_t i = item->GetMessageId(); uint32_t buff_len = i % DEFAULT_STREAMING_MESSAGE_BUFFER_SIZE; if (i > MESSAGE_BOUND_SIZE) break; EXPECT_EQ(buff_len, item->PayloadSize()); uint8_t *compared_data = new uint8_t[buff_len]; for (uint32_t j = 0; j < item->PayloadSize(); ++j) { compared_data[j] = j % 128; } EXPECT_EQ(std::memcmp(compared_data, item->Payload(), item->PayloadSize()), 0); delete[] compared_data; } STREAMING_LOG(DEBUG) << "Received message count => " << recevied_message_cnt; if (recevied_message_cnt == queue_id_vec.size() * MESSAGE_BOUND_SIZE) { STREAMING_LOG(INFO) << "recevied message count => " << recevied_message_cnt << ", break"; break; } } } void StreamingReaderStrategyTest(StreamingConfig &config) { ChannelCreationParameter param{ peer_actor_id_, std::make_shared( ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "writer_async_call_func", ""})), std::make_shared( ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "writer_sync_call_func", ""}))}; std::vector params(queue_ids_.size(), param); STREAMING_LOG(INFO) << "reader actor_ids size: " << params.size() << " actor_id: " << peer_actor_id_; std::shared_ptr runtime_context(new RuntimeContext()); runtime_context->SetConfig(config); std::shared_ptr reader(new DataReader(runtime_context)); reader->Init(queue_ids_, params, -1); ReaderLoopForward(reader, nullptr, queue_ids_); STREAMING_LOG(INFO) << "Reader exit"; } void StreamingWriterExactlyOnceTest() { STREAMING_LOG(INFO) << "StreamingQueueReaderTestSuite::StreamingWriterExactlyOnceTest"; StreamingConfig config; StreamingReaderStrategyTest(config); status_ = true; } }; class StreamingQueueUpStreamTestSuite : public StreamingQueueTestSuite { public: StreamingQueueUpStreamTestSuite(ActorID &peer_actor_id, std::vector queue_ids, std::vector rescale_queue_ids) : StreamingQueueTestSuite(peer_actor_id, queue_ids, rescale_queue_ids) { test_func_map_ = { {"pull_peer_async_test", std::bind(&StreamingQueueUpStreamTestSuite::PullPeerAsyncTest, this)}, {"get_queue_test", std::bind(&StreamingQueueUpStreamTestSuite::GetQueueTest, this)}}; } void GetQueueTest() { // Sleep 2s, queue shoulde not exist when reader pull. std::this_thread::sleep_for(std::chrono::milliseconds(2000)); auto upstream_handler = ray::streaming::UpstreamQueueMessageHandler::GetService(); ObjectID &queue_id = queue_ids_[0]; RayFunction async_call_func{ ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "reader_async_call_func", ""})}; RayFunction sync_call_func{ ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "reader_sync_call_func", ""})}; upstream_handler->SetPeerActorID(queue_id, peer_actor_id_, async_call_func, sync_call_func); upstream_handler->CreateUpstreamQueue(queue_id, peer_actor_id_, 10240); STREAMING_LOG(INFO) << "IsQueueExist: " << upstream_handler->UpstreamQueueExists(queue_id); // Sleep 2s, No valid data when reader pull std::this_thread::sleep_for(std::chrono::milliseconds(2000)); std::this_thread::sleep_for(std::chrono::milliseconds(10 * 1000)); STREAMING_LOG(INFO) << "StreamingQueueUpStreamTestSuite::GetQueueTest done"; status_ = true; } void PullPeerAsyncTest() { // Sleep 2s, queue should not exist when reader pull. std::this_thread::sleep_for(std::chrono::milliseconds(2000)); auto upstream_handler = ray::streaming::UpstreamQueueMessageHandler::GetService(); ObjectID &queue_id = queue_ids_[0]; RayFunction async_call_func{ ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "reader_async_call_func", ""})}; RayFunction sync_call_func{ ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "reader_sync_call_func", ""})}; upstream_handler->SetPeerActorID(queue_id, peer_actor_id_, async_call_func, sync_call_func); std::shared_ptr queue = upstream_handler->CreateUpstreamQueue(queue_id, peer_actor_id_, 10240); STREAMING_LOG(INFO) << "IsQueueExist: " << upstream_handler->UpstreamQueueExists(queue_id); // Sleep 2s, No valid data when reader pull std::this_thread::sleep_for(std::chrono::milliseconds(2000)); // message id starts from 1 for (int msg_id = 1; msg_id <= 80; msg_id++) { uint8_t data[100]; memset(data, msg_id, 100); STREAMING_LOG(INFO) << "Writer User Push item msg_id: " << msg_id; ASSERT_TRUE( queue->Push(data, 100, current_sys_time_ms(), msg_id, msg_id, true).ok()); queue->Send(); } std::this_thread::sleep_for(std::chrono::milliseconds(5000)); STREAMING_LOG(INFO) << "StreamingQueueUpStreamTestSuite::PullPeerAsyncTest done"; status_ = true; } }; class StreamingQueueDownStreamTestSuite : public StreamingQueueTestSuite { public: StreamingQueueDownStreamTestSuite(ActorID peer_actor_id, std::vector queue_ids, std::vector rescale_queue_ids) : StreamingQueueTestSuite(peer_actor_id, queue_ids, rescale_queue_ids) { test_func_map_ = { {"pull_peer_async_test", std::bind(&StreamingQueueDownStreamTestSuite::PullPeerAsyncTest, this)}, {"get_queue_test", std::bind(&StreamingQueueDownStreamTestSuite::GetQueueTest, this)}}; }; void GetQueueTest() { auto downstream_handler = ray::streaming::DownstreamQueueMessageHandler::GetService(); ObjectID &queue_id = queue_ids_[0]; RayFunction async_call_func{ ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "writer_async_call_func", ""})}; RayFunction sync_call_func{ ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "writer_sync_call_func", ""})}; downstream_handler->SetPeerActorID(queue_id, peer_actor_id_, async_call_func, sync_call_func); downstream_handler->CreateDownstreamQueue(queue_id, peer_actor_id_); bool is_upstream_first_pull_ = false; downstream_handler->PullQueue(queue_id, 1, is_upstream_first_pull_, 10 * 1000); ASSERT_TRUE(is_upstream_first_pull_); downstream_handler->PullQueue(queue_id, 1, is_upstream_first_pull_, 10 * 1000); ASSERT_FALSE(is_upstream_first_pull_); STREAMING_LOG(INFO) << "StreamingQueueDownStreamTestSuite::GetQueueTest done"; status_ = true; } void PullPeerAsyncTest() { auto downstream_handler = ray::streaming::DownstreamQueueMessageHandler::GetService(); ObjectID &queue_id = queue_ids_[0]; RayFunction async_call_func{ ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "writer_async_call_func", ""})}; RayFunction sync_call_func{ ray::Language::PYTHON, ray::FunctionDescriptorBuilder::FromVector( ray::Language::PYTHON, {"", "", "writer_sync_call_func", ""})}; downstream_handler->SetPeerActorID(queue_id, peer_actor_id_, async_call_func, sync_call_func); std::shared_ptr queue = downstream_handler->CreateDownstreamQueue(queue_id, peer_actor_id_); bool is_first_pull; downstream_handler->PullQueue(queue_id, 1, is_first_pull, 10 * 1000); uint64_t count = 0; uint8_t msg_id = 1; while (true) { uint8_t *data = nullptr; uint32_t data_size = 0; uint64_t timeout_ms = 1000; QueueItem item = queue->PopPendingBlockTimeout(timeout_ms * 1000); if (item.SeqId() == QUEUE_INVALID_SEQ_ID) { STREAMING_LOG(INFO) << "PopPendingBlockTimeout timeout."; data = nullptr; data_size = 0; } else { data = item.Buffer()->Data(); data_size = item.Buffer()->Size(); } STREAMING_LOG(INFO) << "[Reader] count: " << count; if (data == nullptr) { STREAMING_LOG(INFO) << "[Reader] data null"; continue; } for (uint32_t i = 0; i < data_size; i++) { ASSERT_EQ(data[i], msg_id); } count++; if (count == 80) { bool is_upstream_first_pull; msg_id = 50; downstream_handler->PullPeerAsync(queue_id, 50, is_upstream_first_pull, 1000); continue; } msg_id++; STREAMING_LOG(INFO) << "[Reader] count: " << count; if (count == 110) { break; } } STREAMING_LOG(INFO) << "StreamingQueueDownStreamTestSuite::PullPeerAsyncTest done"; status_ = true; } }; class TestSuiteFactory { public: static std::shared_ptr CreateTestSuite( std::shared_ptr message) { std::shared_ptr test_suite = nullptr; std::string suite_name = message->TestSuiteName(); queue::protobuf::StreamingQueueTestRole role = message->Role(); const std::vector &queue_ids = message->QueueIds(); const std::vector &rescale_queue_ids = message->RescaleQueueIds(); ActorID peer_actor_id = message->PeerActorId(); if (role == queue::protobuf::StreamingQueueTestRole::WRITER) { if (suite_name == "StreamingWriterTest") { test_suite = std::make_shared( peer_actor_id, queue_ids, rescale_queue_ids); } else if (suite_name == "StreamingQueueTest") { test_suite = std::make_shared( peer_actor_id, queue_ids, rescale_queue_ids); } else { STREAMING_CHECK(false) << "unsurported suite_name: " << suite_name; } } else { if (suite_name == "StreamingWriterTest") { test_suite = std::make_shared( peer_actor_id, queue_ids, rescale_queue_ids); } else if (suite_name == "StreamingQueueTest") { test_suite = std::make_shared( peer_actor_id, queue_ids, rescale_queue_ids); } else { STREAMING_CHECK(false) << "unsupported suite_name: " << suite_name; } } return test_suite; } }; class StreamingWorker { public: StreamingWorker(const std::string &store_socket, const std::string &raylet_socket, int node_manager_port, const gcs::GcsClientOptions &gcs_options) : test_suite_(nullptr), peer_actor_handle_(nullptr) { // You must keep it same with `src/ray/core_worker/core_worker.h:CoreWorkerOptions` CoreWorkerOptions options; options.worker_type = WorkerType::WORKER; options.language = Language::PYTHON; options.store_socket = store_socket; options.raylet_socket = raylet_socket; options.gcs_options = gcs_options; options.enable_logging = true; options.install_failure_signal_handler = true; options.node_ip_address = "127.0.0.1"; options.node_manager_port = node_manager_port; options.raylet_ip_address = "127.0.0.1"; options.task_execution_callback = std::bind(&StreamingWorker::ExecuteTask, this, _1, _2, _3, _4, _5, _6, _7, _8); options.ref_counting_enabled = true; options.num_workers = 1; options.metrics_agent_port = -1; CoreWorkerProcess::Initialize(options); STREAMING_LOG(INFO) << "StreamingWorker constructor"; } void RunTaskExecutionLoop() { // Start executing tasks. CoreWorkerProcess::RunTaskExecutionLoop(); } private: Status ExecuteTask(TaskType task_type, const std::string task_name, const RayFunction &ray_function, const std::unordered_map &required_resources, const std::vector> &args, const std::vector &arg_reference_ids, const std::vector &return_ids, std::vector> *results) { // Only one arg param used in streaming. STREAMING_CHECK(args.size() >= 1) << "args.size() = " << args.size(); ray::FunctionDescriptor function_descriptor = ray_function.GetFunctionDescriptor(); RAY_CHECK(function_descriptor->Type() == ray::FunctionDescriptorType::kPythonFunctionDescriptor); auto typed_descriptor = function_descriptor->As(); STREAMING_LOG(DEBUG) << "StreamingWorker::ExecuteTask " << typed_descriptor->ToString(); std::string func_name = typed_descriptor->FunctionName(); if (func_name == "init") { std::shared_ptr local_buffer = std::make_shared(args[0]->GetData()->Data(), args[0]->GetData()->Size(), true); HandleInitTask(local_buffer); } else if (func_name == "execute_test") { STREAMING_LOG(INFO) << "Test name: " << typed_descriptor->ClassName(); test_suite_->ExecuteTest(typed_descriptor->ClassName()); } else if (func_name == "check_current_test_status") { results->push_back(std::make_shared(test_suite_->CheckCurTestStatus(), nullptr, std::vector())); } else if (func_name == "reader_sync_call_func") { if (test_suite_->TestDone()) { STREAMING_LOG(WARNING) << "Test has done!!"; return Status::OK(); } std::shared_ptr local_buffer = std::make_shared(args[1]->GetData()->Data(), args[1]->GetData()->Size(), true); auto result_buffer = reader_client_->OnReaderMessageSync(local_buffer); results->push_back( std::make_shared(result_buffer, nullptr, std::vector())); } else if (func_name == "reader_async_call_func") { if (test_suite_->TestDone()) { STREAMING_LOG(WARNING) << "Test has done!!"; return Status::OK(); } std::shared_ptr local_buffer = std::make_shared(args[1]->GetData()->Data(), args[1]->GetData()->Size(), true); reader_client_->OnReaderMessage(local_buffer); } else if (func_name == "writer_sync_call_func") { if (test_suite_->TestDone()) { STREAMING_LOG(WARNING) << "Test has done!!"; return Status::OK(); } std::shared_ptr local_buffer = std::make_shared(args[1]->GetData()->Data(), args[1]->GetData()->Size(), true); auto result_buffer = writer_client_->OnWriterMessageSync(local_buffer); results->push_back( std::make_shared(result_buffer, nullptr, std::vector())); } else if (func_name == "writer_async_call_func") { if (test_suite_->TestDone()) { STREAMING_LOG(WARNING) << "Test has done!!"; return Status::OK(); } std::shared_ptr local_buffer = std::make_shared(args[1]->GetData()->Data(), args[1]->GetData()->Size(), true); writer_client_->OnWriterMessage(local_buffer); } else { STREAMING_LOG(WARNING) << "Invalid function name " << func_name; } return Status::OK(); } private: void HandleInitTask(std::shared_ptr buffer) { reader_client_ = std::make_shared(); writer_client_ = std::make_shared(); uint8_t *bytes = buffer->Data(); uint8_t *p_cur = bytes; uint32_t *magic_num = (uint32_t *)p_cur; STREAMING_CHECK(*magic_num == Message::MagicNum); p_cur += sizeof(Message::MagicNum); queue::protobuf::StreamingQueueMessageType *type = (queue::protobuf::StreamingQueueMessageType *)p_cur; STREAMING_CHECK( *type == queue::protobuf::StreamingQueueMessageType::StreamingQueueTestInitMsgType); std::shared_ptr message = TestInitMessage::FromBytes(bytes); std::string actor_handle_serialized = message->ActorHandleSerialized(); CoreWorkerProcess::GetCoreWorker().DeserializeAndRegisterActorHandle( actor_handle_serialized, ObjectID::Nil()); std::shared_ptr actor_handle(new ActorHandle(actor_handle_serialized)); STREAMING_CHECK(actor_handle != nullptr); STREAMING_LOG(INFO) << "Actor id from handle: " << actor_handle->GetActorID(); STREAMING_LOG(INFO) << "HandleInitTask queues:"; for (auto qid : message->QueueIds()) { STREAMING_LOG(INFO) << "queue: " << qid; } for (auto qid : message->RescaleQueueIds()) { STREAMING_LOG(INFO) << "rescale queue: " << qid; } test_suite_ = TestSuiteFactory::CreateTestSuite(message); STREAMING_CHECK(test_suite_ != nullptr); } private: std::shared_ptr reader_client_; std::shared_ptr writer_client_; std::shared_ptr test_thread_; std::shared_ptr test_suite_; std::shared_ptr peer_actor_handle_; }; } // namespace streaming } // namespace ray int main(int argc, char **argv) { RAY_CHECK(argc == 4); auto store_socket = std::string(argv[1]); auto raylet_socket = std::string(argv[2]); auto node_manager_port = std::stoi(std::string(argv[3])); ray::gcs::GcsClientOptions gcs_options("127.0.0.1", 6379, ""); ray::streaming::StreamingWorker worker(store_socket, raylet_socket, node_manager_port, gcs_options); worker.RunTaskExecutionLoop(); return 0; }