wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-02 13:01:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update arrow and remove plasma_manager references. (#3545)

Browse Source
This commit is contained in:
Robert Nishihara
2018-12-15 23:36:02 -08:00
committed by Philipp Moritz
parent b3bf608608
commit 417c7f2d6f
15 changed files with 18 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/gcs/redis_module/ray_redis_module.cc
-35
View File
@@ -20,41 +20,6 @@
extern RedisChainModule module;
#endif
// Various tables are maintained in redis:
//
// == OBJECT TABLE ==
//
// This consists of two parts:
// - The object location table, indexed by OL:object_id, which is the set of
// plasma manager indices that have access to the object.
// (In redis this is represented by a zset (sorted set).)
//
// - The object info table, indexed by OI:object_id, which is a hashmap of:
// "hash" -> the hash of the object,
// "data_size" -> the size of the object in bytes,
// "task" -> the task ID that generated this object.
// "is_put" -> 0 or 1.
//
// == TASK TABLE ==
//
// It maps each TT:task_id to a hash:
// "state" -> the state of the task, encoded as a bit mask of scheduling_state
// enum values in task.h,
// "local_scheduler_id" -> the ID of the local scheduler the task is assigned
// to,
// "TaskSpec" -> serialized bytes of a TaskInfo (defined in common.fbs), which
// describes the details this task.
//
// See also the definition of TaskReply in common.fbs.
#define OBJECT_INFO_PREFIX "OI:"
#define OBJECT_LOCATION_PREFIX "OL:"
#define OBJECT_NOTIFICATION_PREFIX "ON:"
#define TASK_PREFIX "TT:"
#define OBJECT_BCAST "BCAST"
#define OBJECT_CHANNEL_PREFIX "OC:"
#define CHECK_ERROR(STATUS, MESSAGE) \
if ((STATUS) == REDISMODULE_ERR) { \
return RedisModule_ReplyWithError(ctx, (MESSAGE)); \
src/ray/object_manager/object_buffer_pool.cc
+1 -1
View File
@@ -6,7 +6,7 @@ ObjectBufferPool::ObjectBufferPool(const std::string &store_socket_name,
uint64_t chunk_size)
: default_chunk_size_(chunk_size) {
store_socket_name_ = store_socket_name;
ARROW_CHECK_OK(store_client_.Connect(store_socket_name_.c_str(), ""));
ARROW_CHECK_OK(store_client_.Connect(store_socket_name_.c_str()));
}
ObjectBufferPool::~ObjectBufferPool() {
src/ray/object_manager/object_store_notification_manager.cc
+1 -1
View File
@@ -15,7 +15,7 @@ namespace ray {
ObjectStoreNotificationManager::ObjectStoreNotificationManager(
boost::asio::io_service &io_service, const std::string &store_socket_name)
: store_client_(), socket_(io_service) {
ARROW_CHECK_OK(store_client_.Connect(store_socket_name.c_str(), ""));
ARROW_CHECK_OK(store_client_.Connect(store_socket_name.c_str()));
ARROW_CHECK_OK(store_client_.Subscribe(&c_socket_));
boost::system::error_code ec;
src/ray/object_manager/test/object_manager_stress_test.cc
+2 -2
View File
@@ -154,8 +154,8 @@ class TestObjectManagerBase : public ::testing::Test {
server2.reset(new MockServer(main_service, om_config_2, gcs_client_2));
// connect to stores.
ARROW_CHECK_OK(client1.Connect(store_id_1, ""));
ARROW_CHECK_OK(client2.Connect(store_id_2, ""));
ARROW_CHECK_OK(client1.Connect(store_id_1));
ARROW_CHECK_OK(client2.Connect(store_id_2));
}
void TearDown() {
src/ray/object_manager/test/object_manager_test.cc
+2 -2
View File
@@ -139,8 +139,8 @@ class TestObjectManagerBase : public ::testing::Test {
server2.reset(new MockServer(main_service, om_config_2, gcs_client_2));
// connect to stores.
ARROW_CHECK_OK(client1.Connect(store_id_1, ""));
ARROW_CHECK_OK(client2.Connect(store_id_2, ""));
ARROW_CHECK_OK(client1.Connect(store_id_1));
ARROW_CHECK_OK(client2.Connect(store_id_2));
}
void TearDown() {
src/ray/ray_config.h
+1 -16
View File
@@ -64,10 +64,6 @@ class RayConfig {
return kill_worker_timeout_milliseconds_;
}
int64_t manager_timeout_milliseconds() const { return manager_timeout_milliseconds_; }
int64_t buf_size() const { return buf_size_; }
int64_t max_time_for_handler_milliseconds() const {
return max_time_for_handler_milliseconds_;
}
@@ -154,10 +150,6 @@ class RayConfig {
local_scheduler_fetch_request_size_ = pair.second;
} else if (pair.first == "kill_worker_timeout_milliseconds") {
kill_worker_timeout_milliseconds_ = pair.second;
} else if (pair.first == "manager_timeout_milliseconds") {
manager_timeout_milliseconds_ = pair.second;
} else if (pair.first == "buf_size") {
buf_size_ = pair.second;
} else if (pair.first == "max_time_for_handler_milliseconds") {
max_time_for_handler_milliseconds_ = pair.second;
} else if (pair.first == "size_limit") {
@@ -216,8 +208,6 @@ class RayConfig {
max_num_to_reconstruct_(10000),
local_scheduler_fetch_request_size_(10000),
kill_worker_timeout_milliseconds_(100),
manager_timeout_milliseconds_(1000),
buf_size_(80 * 1024),
max_time_for_handler_milliseconds_(1000),
size_limit_(10000),
num_elements_limit_(10000),
@@ -245,8 +235,7 @@ class RayConfig {
/// warning is logged that the handler is taking too long.
int64_t handler_warning_timeout_ms_;
/// The duration between heartbeats. These are sent by the plasma manager and
/// local scheduler.
/// The duration between heartbeats. These are sent by the raylet.
int64_t heartbeat_timeout_milliseconds_;
/// If a component has not sent a heartbeat in the last num_heartbeats_timeout
/// heartbeat intervals, the global scheduler or monitor process will report
@@ -306,10 +295,6 @@ class RayConfig {
/// the worker SIGKILL.
int64_t kill_worker_timeout_milliseconds_;
/// These are used by the plasma manager.
int64_t manager_timeout_milliseconds_;
int64_t buf_size_;
/// This is a timeout used to cause failures in the plasma manager and local
/// scheduler when certain event loop handlers take too long.
int64_t max_time_for_handler_milliseconds_;
src/ray/raylet/lib/python/config_extension.cc
-19
View File
@@ -69,18 +69,6 @@ PyObject *PyRayConfig_kill_worker_timeout_milliseconds(PyObject *self) {
return PyLong_FromLongLong(RayConfig::instance().kill_worker_timeout_milliseconds());
}
PyObject *PyRayConfig_manager_timeout_milliseconds(PyObject *self) {
return PyLong_FromLongLong(RayConfig::instance().manager_timeout_milliseconds());
}
PyObject *PyRayConfig_buf_size(PyObject *self) {
return PyLong_FromLongLong(RayConfig::instance().buf_size());
}
PyObject *PyRayConfig_max_time_for_handler_milliseconds(PyObject *self) {
return PyLong_FromLongLong(RayConfig::instance().max_time_for_handler_milliseconds());
}
PyObject *PyRayConfig_size_limit(PyObject *self) {
return PyLong_FromLongLong(RayConfig::instance().size_limit());
}
@@ -144,13 +132,6 @@ static PyMethodDef PyRayConfig_methods[] = {
{"kill_worker_timeout_milliseconds",
(PyCFunction)PyRayConfig_kill_worker_timeout_milliseconds, METH_NOARGS,
"Return kill_worker_timeout_milliseconds"},
{"manager_timeout_milliseconds",
(PyCFunction)PyRayConfig_manager_timeout_milliseconds, METH_NOARGS,
"Return manager_timeout_milliseconds"},
{"buf_size", (PyCFunction)PyRayConfig_buf_size, METH_NOARGS, "Return buf_size"},
{"max_time_for_handler_milliseconds",
(PyCFunction)PyRayConfig_max_time_for_handler_milliseconds, METH_NOARGS,
"Return max_time_for_handler_milliseconds"},
{"size_limit", (PyCFunction)PyRayConfig_size_limit, METH_NOARGS, "Return size_limit"},
{"num_elements_limit", (PyCFunction)PyRayConfig_num_elements_limit, METH_NOARGS,
"Return num_elements_limit"},
src/ray/raylet/lib/python/config_extension.h
-3
View File
@@ -28,9 +28,6 @@ PyObject *PyRayConfig_local_scheduler_reconstruction_timeout_milliseconds(PyObje
PyObject *PyRayConfig_max_num_to_reconstruct(PyObject *self);
PyObject *PyRayConfig_local_scheduler_fetch_request_size(PyObject *self);
PyObject *PyRayConfig_kill_worker_timeout_milliseconds(PyObject *self);
PyObject *PyRayConfig_manager_timeout_milliseconds(PyObject *self);
PyObject *PyRayConfig_buf_size(PyObject *self);
PyObject *PyRayConfig_max_time_for_handler_milliseconds(PyObject *self);
PyObject *PyRayConfig_size_limit(PyObject *self);
PyObject *PyRayConfig_num_elements_limit(PyObject *self);
PyObject *PyRayConfig_max_time_for_loop(PyObject *self);
src/ray/raylet/node_manager.cc
+1 -1
View File
@@ -95,7 +95,7 @@ NodeManager::NodeManager(boost::asio::io_service &io_service,
RAY_CHECK_OK(object_manager_.SubscribeObjDeleted(
[this](const ObjectID &object_id) { HandleObjectMissing(object_id); }));
ARROW_CHECK_OK(store_client_.Connect(config.store_socket_name.c_str(), "", 0));
ARROW_CHECK_OK(store_client_.Connect(config.store_socket_name.c_str()));
}
ray::Status NodeManager::RegisterGcs() {
src/ray/raylet/object_manager_integration_test.cc
+2 -2
View File
@@ -74,8 +74,8 @@ class TestObjectManagerBase : public ::testing::Test {
GetNodeManagerConfig("raylet_2", store_sock_2), om_config_2, gcs_client_2));
// connect to stores.
ARROW_CHECK_OK(client1.Connect(store_sock_1, ""));
ARROW_CHECK_OK(client2.Connect(store_sock_2, ""));
ARROW_CHECK_OK(client1.Connect(store_sock_1));
ARROW_CHECK_OK(client2.Connect(store_sock_2));
}
void TearDown() {