wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-11 11:24:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Streaming data transfer supports cross language. (#7961)

Browse Source 
* add init parameters for java

* fix bug

* cython

* fix compile

* fix test_direct_tranfer

* comment

* ChannelCreationParameter

* fix comment

* builder

* lint and fix tests

* fix single process test

* fix checkstyle and lint

* checkstyle

* lint python

Co-authored-by: wanxing <wanxing@B-458DMD6M-1753.local>
This commit is contained in:
wanxing
2020-04-16 15:16:48 +08:00
committed by GitHub
co-authored by wanxing
parent 5a7882bb44
commit 9345d03ffb
36 changed files with 618 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/test/mock_actor.cc
+30 -25
View File
@@ -94,8 +94,18 @@ class StreamingQueueWriterTestSuite : public StreamingQueueTestSuite {
for (auto &queue_id : queue_ids_) {
STREAMING_LOG(INFO) << "queue_id: " << queue_id;
}
std::vector<ActorID> actor_ids(queue_ids_.size(), peer_actor_id_);
STREAMING_LOG(INFO) << "writer actor_ids size: " << actor_ids.size()
ChannelCreationParameter param{
peer_actor_id_,
std::make_shared<RayFunction>(
ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::FromVector(
ray::Language::PYTHON, {"", "", "reader_async_call_func", ""})),
std::make_shared<RayFunction>(
ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::FromVector(
ray::Language::PYTHON, {"", "", "reader_sync_call_func", ""}))};
std::vector<ChannelCreationParameter> params(queue_ids_.size(), param);
STREAMING_LOG(INFO) << "writer actor_ids size: " << params.size()
<< " actor_id: " << peer_actor_id_;
std::shared_ptr<RuntimeContext> runtime_context(new RuntimeContext());
@@ -104,7 +114,7 @@ class StreamingQueueWriterTestSuite : public StreamingQueueTestSuite {
std::shared_ptr<DataWriter> streaming_writer_client(new DataWriter(runtime_context));
uint64_t queue_size = 10 * 1000 * 1000;
std::vector<uint64_t> channel_seq_id_vec(queue_ids_.size(), 0);
streaming_writer_client->Init(queue_ids_, actor_ids, channel_seq_id_vec,
streaming_writer_client->Init(queue_ids_, params, channel_seq_id_vec,
std::vector<uint64_t>(queue_ids_.size(), queue_size));
STREAMING_LOG(INFO) << "streaming_writer_client Init done";
@@ -214,14 +224,24 @@ class StreamingQueueReaderTestSuite : public StreamingQueueTestSuite {
}
void StreamingReaderStrategyTest(StreamingConfig &config) {
std::vector<ActorID> actor_ids(queue_ids_.size(), peer_actor_id_);
STREAMING_LOG(INFO) << "reader actor_ids size: " << actor_ids.size()
ChannelCreationParameter param{
peer_actor_id_,
std::make_shared<RayFunction>(
ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::FromVector(
ray::Language::PYTHON, {"", "", "writer_async_call_func", ""})),
std::make_shared<RayFunction>(
ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::FromVector(
ray::Language::PYTHON, {"", "", "writer_sync_call_func", ""}))};
std::vector<ChannelCreationParameter> params(queue_ids_.size(), param);
STREAMING_LOG(INFO) << "reader actor_ids size: " << params.size()
<< " actor_id: " << peer_actor_id_;
std::shared_ptr<RuntimeContext> runtime_context(new RuntimeContext());
runtime_context->SetConfig(config);
std::shared_ptr<DataReader> reader(new DataReader(runtime_context));
reader->Init(queue_ids_, actor_ids, -1);
reader->Init(queue_ids_, params, -1);
ReaderLoopForward(reader, nullptr, queue_ids_);
STREAMING_LOG(INFO) << "Reader exit";
@@ -298,23 +318,8 @@ class StreamingWorker {
};
CoreWorkerProcess::Initialize(options);
RayFunction reader_async_call_func{ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::BuildPython(
"reader_async_call_func", "", "", "")};
RayFunction reader_sync_call_func{
ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::BuildPython("reader_sync_call_func", "", "", "")};
RayFunction writer_async_call_func{ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::BuildPython(
"writer_async_call_func", "", "", "")};
RayFunction writer_sync_call_func{
ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::BuildPython("writer_sync_call_func", "", "", "")};
reader_client_ =
std::make_shared<ReaderClient>(reader_async_call_func, reader_sync_call_func);
writer_client_ =
std::make_shared<WriterClient>(writer_async_call_func, writer_sync_call_func);
reader_client_ = std::make_shared<ReaderClient>();
writer_client_ = std::make_shared<WriterClient>();
STREAMING_LOG(INFO) << "StreamingWorker constructor";
}
@@ -338,9 +343,9 @@ class StreamingWorker {
ray::FunctionDescriptorType::kPythonFunctionDescriptor);
auto typed_descriptor = function_descriptor->As<ray::PythonFunctionDescriptor>();
STREAMING_LOG(INFO) << "StreamingWorker::ExecuteTask "
<< typed_descriptor->ModuleName();
<< typed_descriptor->ToString();
std::string func_name = typed_descriptor->ModuleName();
std::string func_name = typed_descriptor->FunctionName();
if (func_name == "init") {
std::shared_ptr<LocalMemoryBuffer> local_buffer =
std::make_shared<LocalMemoryBuffer>(args[0]->GetData()->Data(),
streaming/src/test/mock_transfer_tests.cc
+3 -5
View File
@@ -51,11 +51,9 @@ class StreamingTransferTest : public ::testing::Test {
}
std::vector<uint64_t> channel_id_vec(queue_vec.size(), 0);
std::vector<uint64_t> queue_size_vec(queue_vec.size(), 10000);
// actor ids are not used in this test, so we can just use Nil.
std::vector<ActorID> actor_id_vec(queue_vec.size(),
ActorID::NilFromJob(JobID::FromInt(0)));
writer->Init(queue_vec, actor_id_vec, channel_id_vec, queue_size_vec);
reader->Init(queue_vec, actor_id_vec, channel_id_vec, queue_size_vec, -1);
std::vector<ChannelCreationParameter> params(queue_vec.size());
writer->Init(queue_vec, params, channel_id_vec, queue_size_vec);
reader->Init(queue_vec, params, channel_id_vec, queue_size_vec, -1);
}
void DestroyTransfer() {
writer.reset();
streaming/src/test/queue_tests_base.h
+4 -4
View File
@@ -175,7 +175,7 @@ class StreamingQueueTestBase : public ::testing::TestWithParam<uint64_t> {
TaskOptions options{0, resources};
std::vector<ObjectID> return_ids;
RayFunction func{ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::BuildPython("init", "", "", "")};
ray::FunctionDescriptorBuilder::BuildPython("", "", "init", "")};
RAY_CHECK_OK(driver.SubmitActorTask(self_actor_id, func, args, options, &return_ids));
}
@@ -191,7 +191,7 @@ class StreamingQueueTestBase : public ::testing::TestWithParam<uint64_t> {
TaskOptions options{0, resources};
std::vector<ObjectID> return_ids;
RayFunction func{ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
"execute_test", test, "", "")};
"", test, "execute_test", "")};
RAY_CHECK_OK(driver.SubmitActorTask(actor_id, func, args, options, &return_ids));
}
@@ -207,7 +207,7 @@ class StreamingQueueTestBase : public ::testing::TestWithParam<uint64_t> {
TaskOptions options{1, resources};
std::vector<ObjectID> return_ids;
RayFunction func{ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
"check_current_test_status", "", "", "")};
"", "", "check_current_test_status", "")};
RAY_CHECK_OK(driver.SubmitActorTask(actor_id, func, args, options, &return_ids));
@@ -267,7 +267,7 @@ class StreamingQueueTestBase : public ::testing::TestWithParam<uint64_t> {
auto buffer = std::make_shared<LocalMemoryBuffer>(array, sizeof(array));
RayFunction func{ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
"actor creation task", "", "", "")};
"", "", "actor creation task", "")};
std::vector<TaskArg> args;
args.emplace_back(TaskArg::PassByValue(
std::make_shared<RayObject>(buffer, nullptr, std::vector<ObjectID>())));