wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-12 06:37:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[Streaming] Fault Tolerance Implementation (#10008)" (#10582)

Browse Source 
This reverts commit 1b1466748f.
This commit is contained in:
SangBin Cho
2020-09-04 13:21:18 -07:00
committed by GitHub
parent da83bbd764
commit cb919c5e5c
158 changed files with 1227 additions and 7040 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/lib/java/io_ray_streaming_runtime_transfer_channel_ChannelId.cc → streaming/src/lib/java/io_ray_streaming_runtime_transfer_ChannelId.cc
+4 -5
View File
@@ -1,18 +1,17 @@
#include "io_ray_streaming_runtime_transfer_channel_ChannelId.h"
#include "io_ray_streaming_runtime_transfer_ChannelId.h"
#include "streaming_jni_common.h"
using namespace ray::streaming;
JNIEXPORT jlong JNICALL
Java_io_ray_streaming_runtime_transfer_channel_ChannelId_createNativeId(
JNIEXPORT jlong JNICALL Java_io_ray_streaming_runtime_transfer_ChannelId_createNativeId(
JNIEnv *env, jclass cls, jlong qid_address) {
auto id = ray::ObjectID::FromBinary(
std::string(reinterpret_cast<const char *>(qid_address), ray::ObjectID::Size()));
return reinterpret_cast<jlong>(new ray::ObjectID(id));
}
JNIEXPORT void JNICALL
Java_io_ray_streaming_runtime_transfer_channel_ChannelId_destroyNativeId(
JNIEXPORT void JNICALL Java_io_ray_streaming_runtime_transfer_ChannelId_destroyNativeId(
JNIEnv *env, jclass cls, jlong native_id_ptr) {
auto id = reinterpret_cast<ray::ObjectID *>(native_id_ptr);
STREAMING_CHECK(id != nullptr);
streaming/src/lib/java/io_ray_streaming_runtime_transfer_ChannelId.h
+31
View File
@@ -0,0 +1,31 @@
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class io_ray_streaming_runtime_transfer_ChannelId */
#ifndef _Included_io_ray_streaming_runtime_transfer_ChannelId
#define _Included_io_ray_streaming_runtime_transfer_ChannelId
#ifdef __cplusplus
extern "C" {
#endif
#undef io_ray_streaming_runtime_transfer_ChannelId_ID_LENGTH
#define io_ray_streaming_runtime_transfer_ChannelId_ID_LENGTH 20L
/*
* Class: io_ray_streaming_runtime_transfer_ChannelId
* Method: createNativeId
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL
Java_io_ray_streaming_runtime_transfer_ChannelId_createNativeId(JNIEnv *, jclass, jlong);
/*
* Class: io_ray_streaming_runtime_transfer_ChannelId
* Method: destroyNativeId
* Signature: (J)V
*/
JNIEXPORT void JNICALL
Java_io_ray_streaming_runtime_transfer_ChannelId_destroyNativeId(JNIEnv *, jclass, jlong);
#ifdef __cplusplus
}
#endif
#endif
streaming/src/lib/java/io_ray_streaming_runtime_transfer_DataReader.cc
+7 -50
View File
@@ -12,13 +12,15 @@ using namespace ray::streaming;
JNIEXPORT jlong JNICALL
Java_io_ray_streaming_runtime_transfer_DataReader_createDataReaderNative(
JNIEnv *env, jclass, jobject streaming_queue_initial_parameters,
jobjectArray input_channels, jlongArray msg_id_array, jlong timer_interval,
jobject creation_status, jbyteArray config_bytes, jboolean is_mock) {
jobjectArray input_channels, jlongArray seq_id_array, jlongArray msg_id_array,
jlong timer_interval, jboolean isRecreate, jbyteArray config_bytes,
jboolean is_mock) {
STREAMING_LOG(INFO) << "[JNI]: create DataReader.";
std::vector<ray::streaming::ChannelCreationParameter> parameter_vec;
ParseChannelInitParameters(env, streaming_queue_initial_parameters, parameter_vec);
std::vector<ray::ObjectID> input_channels_ids =
jarray_to_object_id_vec(env, input_channels);
std::vector<uint64_t> seq_ids = LongVectorFromJLongArray(env, seq_id_array).data;
std::vector<uint64_t> msg_ids = LongVectorFromJLongArray(env, msg_id_array).data;
auto ctx = std::make_shared<RuntimeContext>();
@@ -30,24 +32,8 @@ Java_io_ray_streaming_runtime_transfer_DataReader_createDataReaderNative(
if (is_mock) {
ctx->MarkMockTest();
}
// init reader
auto reader = new DataReader(ctx);
std::vector<TransferCreationStatus> creation_status_vec;
reader->Init(input_channels_ids, parameter_vec, msg_ids, creation_status_vec,
timer_interval);
// add creation status to Java's List
jclass array_list_cls = env->GetObjectClass(creation_status);
jclass integer_cls = env->FindClass("java/lang/Integer");
jmethodID array_list_add =
env->GetMethodID(array_list_cls, "add", "(Ljava/lang/Object;)Z");
for (auto &status : creation_status_vec) {
jmethodID integer_init = env->GetMethodID(integer_cls, "<init>", "(I)V");
jobject integer_obj =
env->NewObject(integer_cls, integer_init, static_cast<int>(status));
env->CallBooleanMethod(creation_status, array_list_add, integer_obj);
}
reader->Init(input_channels_ids, parameter_vec, seq_ids, msg_ids, timer_interval);
STREAMING_LOG(INFO) << "create native DataReader succeed";
return reinterpret_cast<jlong>(reader);
}
@@ -65,6 +51,8 @@ JNIEXPORT void JNICALL Java_io_ray_streaming_runtime_transfer_DataReader_getBund
} else if (StreamingStatus::GetBundleTimeOut == status) {
} else if (StreamingStatus::InitQueueFailed == status) {
throwRuntimeException(env, "init channel failed");
} else if (StreamingStatus::WaitQueueTimeOut == status) {
throwRuntimeException(env, "wait channel object timeout");
}
if (StreamingStatus::OK != status) {
@@ -100,34 +88,3 @@ Java_io_ray_streaming_runtime_transfer_DataReader_closeReaderNative(JNIEnv *env,
jlong ptr) {
delete reinterpret_cast<DataReader *>(ptr);
}
JNIEXPORT jbyteArray JNICALL
Java_io_ray_streaming_runtime_transfer_DataReader_getOffsetsInfoNative(JNIEnv *env,
jobject thisObj,
jlong ptr) {
auto reader = reinterpret_cast<ray::streaming::DataReader *>(ptr);
std::unordered_map<ray::ObjectID, ConsumerChannelInfo> *offset_map = nullptr;
reader->GetOffsetInfo(offset_map);
STREAMING_CHECK(offset_map);
// queue nums + (queue id + seq id + message id) * queue nums
int offset_data_size =
sizeof(uint32_t) + (kUniqueIDSize + sizeof(uint64_t) * 2) * offset_map->size();
jbyteArray offsets_info = env->NewByteArray(offset_data_size);
int offset = 0;
// total queue nums
auto queue_nums = static_cast<uint32_t>(offset_map->size());
env->SetByteArrayRegion(offsets_info, offset, sizeof(uint32_t),
reinterpret_cast<jbyte *>(&queue_nums));
offset += sizeof(uint32_t);
// queue name & offset
for (auto &p : *offset_map) {
env->SetByteArrayRegion(offsets_info, offset, kUniqueIDSize,
reinterpret_cast<const jbyte *>(p.first.Data()));
offset += kUniqueIDSize;
// msg_id
env->SetByteArrayRegion(offsets_info, offset, sizeof(uint64_t),
reinterpret_cast<jbyte *>(&p.second.current_message_id));
offset += sizeof(uint64_t);
}
return offsets_info;
}
streaming/src/lib/java/io_ray_streaming_runtime_transfer_DataReader.h
+3 -26
View File
@@ -1,17 +1,3 @@
// Copyright 2017 The Ray Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class io_ray_streaming_runtime_transfer_DataReader */
@@ -25,12 +11,12 @@ extern "C" {
* Class: io_ray_streaming_runtime_transfer_DataReader
* Method: createDataReaderNative
* Signature:
* (Lio/ray/streaming/runtime/transfer/ChannelCreationParametersBuilder;[[B[JJLjava/util/List;[BZ)J
* (Lio/ray/streaming/runtime/transfer/ChannelCreationParametersBuilder;[[B[J[JJZ[BZ)J
*/
JNIEXPORT jlong JNICALL
Java_io_ray_streaming_runtime_transfer_DataReader_createDataReaderNative(
JNIEnv *, jclass, jobject, jobjectArray, jlongArray, jlong, jobject, jbyteArray,
jboolean);
JNIEnv *, jclass, jobject, jobjectArray, jlongArray, jlongArray, jlong, jboolean,
jbyteArray, jboolean);
/*
* Class: io_ray_streaming_runtime_transfer_DataReader
@@ -40,15 +26,6 @@ Java_io_ray_streaming_runtime_transfer_DataReader_createDataReaderNative(
JNIEXPORT void JNICALL Java_io_ray_streaming_runtime_transfer_DataReader_getBundleNative(
JNIEnv *, jobject, jlong, jlong, jlong, jlong);
/*
* Class: io_ray_streaming_runtime_transfer_DataReader
* Method: getOffsetsInfoNative
* Signature: (J)[B
*/
JNIEXPORT jbyteArray JNICALL
Java_io_ray_streaming_runtime_transfer_DataReader_getOffsetsInfoNative(JNIEnv *, jobject,
jlong);
/*
* Class: io_ray_streaming_runtime_transfer_DataReader
* Method: stopReaderNative
streaming/src/lib/java/io_ray_streaming_runtime_transfer_DataWriter.cc
+1 -37
View File
@@ -79,40 +79,4 @@ Java_io_ray_streaming_runtime_transfer_DataWriter_closeWriterNative(JNIEnv *env,
jlong ptr) {
auto *data_writer = reinterpret_cast<DataWriter *>(ptr);
delete data_writer;
}
JNIEXPORT jlongArray JNICALL
Java_io_ray_streaming_runtime_transfer_DataWriter_getOutputMsgIdNative(JNIEnv *env,
jobject thisObj,
jlong ptr) {
DataWriter *writer_client = reinterpret_cast<DataWriter *>(ptr);
std::vector<uint64_t> result;
writer_client->GetChannelOffset(result);
jlongArray jArray = env->NewLongArray(result.size());
jlong jdata[result.size()];
for (size_t i = 0; i < result.size(); ++i) {
*(jdata + i) = result[i];
}
env->SetLongArrayRegion(jArray, 0, result.size(), jdata);
return jArray;
}
JNIEXPORT void JNICALL
Java_io_ray_streaming_runtime_transfer_DataWriter_broadcastBarrierNative(
JNIEnv *env, jobject thisObj, jlong ptr, jlong checkpointId, jbyteArray data) {
STREAMING_LOG(INFO) << "jni: broadcast barrier, cp_id=" << checkpointId;
RawDataFromJByteArray raw_data(env, data);
DataWriter *writer_client = reinterpret_cast<DataWriter *>(ptr);
writer_client->BroadcastBarrier(checkpointId, raw_data.data, raw_data.data_size);
}
JNIEXPORT void JNICALL
Java_io_ray_streaming_runtime_transfer_DataWriter_clearCheckpointNative(
JNIEnv *env, jobject thisObj, jlong ptr, jlong checkpointId) {
STREAMING_LOG(INFO) << "[Producer] jni: clearCheckpoints.";
auto *writer = reinterpret_cast<DataWriter *>(ptr);
writer->ClearCheckpoint(checkpointId);
STREAMING_LOG(INFO) << "[Producer] clear checkpoint done.";
}
}
streaming/src/lib/java/io_ray_streaming_runtime_transfer_DataWriter.h
-43
View File
@@ -1,17 +1,3 @@
// Copyright 2017 The Ray Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class io_ray_streaming_runtime_transfer_DataWriter */
@@ -58,35 +44,6 @@ JNIEXPORT void JNICALL
Java_io_ray_streaming_runtime_transfer_DataWriter_closeWriterNative(JNIEnv *, jobject,
jlong);
/*
* Class: io_ray_streaming_runtime_transfer_DataWriter
* Method: getOutputMsgIdNative
* Signature: (J)[J
*/
JNIEXPORT jlongArray JNICALL
Java_io_ray_streaming_runtime_transfer_DataWriter_getOutputMsgIdNative(JNIEnv *, jobject,
jlong);
/*
* Class: io_ray_streaming_runtime_transfer_DataWriter
* Method: broadcastBarrierNative
* Signature: (JJJ[B)V
*/
JNIEXPORT void JNICALL
Java_io_ray_streaming_runtime_transfer_DataWriter_broadcastBarrierNative(JNIEnv *,
jobject, jlong,
jlong,
jbyteArray);
/*
* Class: io_ray_streaming_runtime_transfer_DataWriter
* Method: clearCheckpointNative
* Signature: (JJ)V
*/
JNIEXPORT void JNICALL
Java_io_ray_streaming_runtime_transfer_DataWriter_clearCheckpointNative(JNIEnv *, jobject,
jlong, jlong);
#ifdef __cplusplus
}
#endif
streaming/src/lib/java/io_ray_streaming_runtime_transfer_TransferHandler.h
+2 -16
View File
@@ -1,17 +1,3 @@
// Copyright 2017 The Ray Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class io_ray_streaming_runtime_transfer_TransferHandler */
@@ -24,7 +10,7 @@ extern "C" {
/*
* Class: io_ray_streaming_runtime_transfer_TransferHandler
* Method: createWriterClientNative
* Signature: ()J
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL
Java_io_ray_streaming_runtime_transfer_TransferHandler_createWriterClientNative(JNIEnv *,
@@ -33,7 +19,7 @@ Java_io_ray_streaming_runtime_transfer_TransferHandler_createWriterClientNative(
/*
* Class: io_ray_streaming_runtime_transfer_TransferHandler
* Method: createReaderClientNative
* Signature: ()J
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL
Java_io_ray_streaming_runtime_transfer_TransferHandler_createReaderClientNative(JNIEnv *,
streaming/src/lib/java/io_ray_streaming_runtime_transfer_channel_ChannelId.h
-47
View File
@@ -1,47 +0,0 @@
// Copyright 2017 The Ray Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class io_ray_streaming_runtime_transfer_channel_ChannelId */
#ifndef _Included_io_ray_streaming_runtime_transfer_channel_ChannelId
#define _Included_io_ray_streaming_runtime_transfer_channel_ChannelId
#ifdef __cplusplus
extern "C" {
#endif
#undef io_ray_streaming_runtime_transfer_channel_ChannelId_ID_LENGTH
#define io_ray_streaming_runtime_transfer_channel_ChannelId_ID_LENGTH 20L
/*
* Class: io_ray_streaming_runtime_transfer_channel_ChannelId
* Method: createNativeId
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL
Java_io_ray_streaming_runtime_transfer_channel_ChannelId_createNativeId(JNIEnv *, jclass,
jlong);
/*
* Class: io_ray_streaming_runtime_transfer_channel_ChannelId
* Method: destroyNativeId
* Signature: (J)V
*/
JNIEXPORT void JNICALL
Java_io_ray_streaming_runtime_transfer_channel_ChannelId_destroyNativeId(JNIEnv *, jclass,
jlong);
#ifdef __cplusplus
}
#endif
#endif
streaming/src/lib/java/streaming_jni_common.h
+1 -1
View File
@@ -4,7 +4,7 @@
#include <string>
#include "channel/channel.h"
#include "channel.h"
#include "ray/core_worker/common.h"
#include "util/streaming_logging.h"