wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-13 13:02:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Rename ClientTableData to GcsNodeInfo (#5251)

Browse Source
This commit is contained in:
micafan
2019-07-30 11:22:47 +08:00
committed by Hao Chen
parent 3ba8680963
commit b3bcf59148
21 changed files with 331 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/gcs/redis_gcs_client_test.cc
+33 -34
View File
@@ -1151,17 +1151,16 @@ TEST_F(TestGcsWithAsio, TestSetSubscribeCancel) {
}
void ClientTableNotification(gcs::RedisGcsClient *client, const ClientID &client_id,
const ClientTableData &data, bool is_insertion) {
const GcsNodeInfo &data, bool is_alive) {
ClientID added_id = client->client_table().GetLocalClientId();
ASSERT_EQ(client_id, added_id);
ASSERT_EQ(ClientID::FromBinary(data.client_id()), added_id);
ASSERT_EQ(ClientID::FromBinary(data.client_id()), added_id);
ASSERT_EQ(data.is_insertion(), is_insertion);
ASSERT_EQ(ClientID::FromBinary(data.node_id()), added_id);
ASSERT_EQ(data.state() == GcsNodeInfo::ALIVE, is_alive);
ClientTableData cached_client;
GcsNodeInfo cached_client;
client->client_table().GetClient(added_id, cached_client);
ASSERT_EQ(ClientID::FromBinary(cached_client.client_id()), added_id);
ASSERT_EQ(cached_client.is_insertion(), is_insertion);
ASSERT_EQ(ClientID::FromBinary(cached_client.node_id()), added_id);
ASSERT_EQ(cached_client.state() == GcsNodeInfo::ALIVE, is_alive);
}
void TestClientTableConnect(const JobID &job_id,
@@ -1169,18 +1168,18 @@ void TestClientTableConnect(const JobID &job_id,
// Register callbacks for when a client gets added and removed. The latter
// event will stop the event loop.
client->client_table().RegisterClientAddedCallback(
[](gcs::RedisGcsClient *client, const ClientID &id, const ClientTableData &data) {
[](gcs::RedisGcsClient *client, const ClientID &id, const GcsNodeInfo &data) {
ClientTableNotification(client, id, data, true);
test->Stop();
});
// Connect and disconnect to client table. We should receive notifications
// for the addition and removal of our own entry.
ClientTableData local_client_info = client->client_table().GetLocalClient();
local_client_info.set_node_manager_address("127.0.0.1");
local_client_info.set_node_manager_port(0);
local_client_info.set_object_manager_port(0);
RAY_CHECK_OK(client->client_table().Connect(local_client_info));
GcsNodeInfo local_node_info = client->client_table().GetLocalClient();
local_node_info.set_node_manager_address("127.0.0.1");
local_node_info.set_node_manager_port(0);
local_node_info.set_object_manager_port(0);
RAY_CHECK_OK(client->client_table().Connect(local_node_info));
test->Start();
}
@@ -1194,24 +1193,24 @@ void TestClientTableDisconnect(const JobID &job_id,
// Register callbacks for when a client gets added and removed. The latter
// event will stop the event loop.
client->client_table().RegisterClientAddedCallback(
[](gcs::RedisGcsClient *client, const ClientID &id, const ClientTableData &data) {
[](gcs::RedisGcsClient *client, const ClientID &id, const GcsNodeInfo &data) {
ClientTableNotification(client, id, data, /*is_insertion=*/true);
// Disconnect from the client table. We should receive a notification
// for the removal of our own entry.
RAY_CHECK_OK(client->client_table().Disconnect());
});
client->client_table().RegisterClientRemovedCallback(
[](gcs::RedisGcsClient *client, const ClientID &id, const ClientTableData &data) {
[](gcs::RedisGcsClient *client, const ClientID &id, const GcsNodeInfo &data) {
ClientTableNotification(client, id, data, /*is_insertion=*/false);
test->Stop();
});
// Connect to the client table. We should receive notification for the
// addition of our own entry.
ClientTableData local_client_info = client->client_table().GetLocalClient();
local_client_info.set_node_manager_address("127.0.0.1");
local_client_info.set_node_manager_port(0);
local_client_info.set_object_manager_port(0);
RAY_CHECK_OK(client->client_table().Connect(local_client_info));
GcsNodeInfo local_node_info = client->client_table().GetLocalClient();
local_node_info.set_node_manager_address("127.0.0.1");
local_node_info.set_node_manager_port(0);
local_node_info.set_object_manager_port(0);
RAY_CHECK_OK(client->client_table().Connect(local_node_info));
test->Start();
}
@@ -1225,21 +1224,21 @@ void TestClientTableImmediateDisconnect(const JobID &job_id,
// Register callbacks for when a client gets added and removed. The latter
// event will stop the event loop.
client->client_table().RegisterClientAddedCallback(
[](gcs::RedisGcsClient *client, const ClientID &id, const ClientTableData &data) {
[](gcs::RedisGcsClient *client, const ClientID &id, const GcsNodeInfo &data) {
ClientTableNotification(client, id, data, true);
});
client->client_table().RegisterClientRemovedCallback(
[](gcs::RedisGcsClient *client, const ClientID &id, const ClientTableData &data) {
[](gcs::RedisGcsClient *client, const ClientID &id, const GcsNodeInfo &data) {
ClientTableNotification(client, id, data, false);
test->Stop();
});
// Connect to then immediately disconnect from the client table. We should
// receive notifications for the addition and removal of our own entry.
ClientTableData local_client_info = client->client_table().GetLocalClient();
local_client_info.set_node_manager_address("127.0.0.1");
local_client_info.set_node_manager_port(0);
local_client_info.set_object_manager_port(0);
RAY_CHECK_OK(client->client_table().Connect(local_client_info));
GcsNodeInfo local_node_info = client->client_table().GetLocalClient();
local_node_info.set_node_manager_address("127.0.0.1");
local_node_info.set_node_manager_port(0);
local_node_info.set_object_manager_port(0);
RAY_CHECK_OK(client->client_table().Connect(local_node_info));
RAY_CHECK_OK(client->client_table().Disconnect());
test->Start();
}
@@ -1251,12 +1250,12 @@ TEST_F(TestGcsWithAsio, TestClientTableImmediateDisconnect) {
void TestClientTableMarkDisconnected(const JobID &job_id,
std::shared_ptr<gcs::RedisGcsClient> client) {
ClientTableData local_client_info = client->client_table().GetLocalClient();
local_client_info.set_node_manager_address("127.0.0.1");
local_client_info.set_node_manager_port(0);
local_client_info.set_object_manager_port(0);
GcsNodeInfo local_node_info = client->client_table().GetLocalClient();
local_node_info.set_node_manager_address("127.0.0.1");
local_node_info.set_node_manager_port(0);
local_node_info.set_object_manager_port(0);
// Connect to the client table to start receiving notifications.
RAY_CHECK_OK(client->client_table().Connect(local_client_info));
RAY_CHECK_OK(client->client_table().Connect(local_node_info));
// Mark a different client as dead.
ClientID dead_client_id = ClientID::FromRandom();
RAY_CHECK_OK(client->client_table().MarkDisconnected(dead_client_id));
@@ -1264,8 +1263,8 @@ void TestClientTableMarkDisconnected(const JobID &job_id,
// marked as dead.
client->client_table().RegisterClientRemovedCallback(
[dead_client_id](gcs::RedisGcsClient *client, const UniqueID &id,
const ClientTableData &data) {
ASSERT_EQ(ClientID::FromBinary(data.client_id()), dead_client_id);
const GcsNodeInfo &data) {
ASSERT_EQ(ClientID::FromBinary(data.node_id()), dead_client_id);
test->Stop();
});
test->Start();
src/ray/gcs/tables.cc
+68 -72
View File
@@ -494,8 +494,8 @@ Status JobTable::AppendJobData(const JobID &job_id, bool is_dead, int64_t timest
void ClientTable::RegisterClientAddedCallback(const ClientTableCallback &callback) {
client_added_callback_ = callback;
// Call the callback for any added clients that are cached.
for (const auto &entry : client_cache_) {
if (!entry.first.IsNil() && (entry.second.is_insertion())) {
for (const auto &entry : node_cache_) {
if (!entry.first.IsNil() && (entry.second.state() == GcsNodeInfo::ALIVE)) {
client_added_callback_(client_, entry.first, entry.second);
}
}
@@ -504,114 +504,111 @@ void ClientTable::RegisterClientAddedCallback(const ClientTableCallback &callbac
void ClientTable::RegisterClientRemovedCallback(const ClientTableCallback &callback) {
client_removed_callback_ = callback;
// Call the callback for any removed clients that are cached.
for (const auto &entry : client_cache_) {
if (!entry.first.IsNil() && !entry.second.is_insertion()) {
for (const auto &entry : node_cache_) {
if (!entry.first.IsNil() && (entry.second.state() == GcsNodeInfo::DEAD)) {
client_removed_callback_(client_, entry.first, entry.second);
}
}
}
void ClientTable::HandleNotification(RedisGcsClient *client,
const ClientTableData &data) {
ClientID client_id = ClientID::FromBinary(data.client_id());
const GcsNodeInfo &node_info) {
ClientID node_id = ClientID::FromBinary(node_info.node_id());
bool is_alive = (node_info.state() == GcsNodeInfo::ALIVE);
// It's possible to get duplicate notifications from the client table, so
// check whether this notification is new.
auto entry = client_cache_.find(client_id);
auto entry = node_cache_.find(node_id);
bool is_notif_new;
if (entry == client_cache_.end()) {
if (entry == node_cache_.end()) {
// If the entry is not in the cache, then the notification is new.
is_notif_new = true;
} else {
// If the entry is in the cache, then the notification is new if the client
// was alive and is now dead or resources have been updated.
bool was_not_deleted = entry->second.is_insertion();
bool is_deleted = !data.is_insertion();
is_notif_new = was_not_deleted && is_deleted;
bool was_alive = (entry->second.state() == GcsNodeInfo::ALIVE);
is_notif_new = was_alive && !is_alive;
// Once a client with a given ID has been removed, it should never be added
// again. If the entry was in the cache and the client was deleted, check
// that this new notification is not an insertion.
if (!entry->second.is_insertion()) {
RAY_CHECK(!data.is_insertion())
<< "Notification for addition of a client that was already removed:"
<< client_id;
if (!was_alive) {
RAY_CHECK(!is_alive)
<< "Notification for addition of a client that was already removed:" << node_id;
}
}
// Add the notification to our cache. Notifications are idempotent.
RAY_LOG(DEBUG) << "[ClientTableNotification] ClientTable Insertion/Deletion "
"notification for client id "
<< client_id << ". IsInsertion: " << data.is_insertion()
<< node_id << ". IsAlive: " << is_alive
<< ". Setting the client cache to data.";
client_cache_[client_id] = data;
node_cache_[node_id] = node_info;
// If the notification is new, call any registered callbacks.
ClientTableData &cache_data = client_cache_[client_id];
GcsNodeInfo &cache_data = node_cache_[node_id];
if (is_notif_new) {
if (data.is_insertion()) {
if (is_alive) {
if (client_added_callback_ != nullptr) {
client_added_callback_(client, client_id, cache_data);
client_added_callback_(client, node_id, cache_data);
}
RAY_CHECK(removed_clients_.find(client_id) == removed_clients_.end());
RAY_CHECK(removed_nodes_.find(node_id) == removed_nodes_.end());
} else {
// NOTE(swang): The client should be added to this data structure before
// NOTE(swang): The node should be added to this data structure before
// the callback gets called, in case the callback depends on the data
// structure getting updated.
removed_clients_.insert(client_id);
removed_nodes_.insert(node_id);
if (client_removed_callback_ != nullptr) {
client_removed_callback_(client, client_id, cache_data);
client_removed_callback_(client, node_id, cache_data);
}
}
}
}
void ClientTable::HandleConnected(RedisGcsClient *client, const ClientTableData &data) {
auto connected_client_id = ClientID::FromBinary(data.client_id());
RAY_CHECK(client_id_ == connected_client_id)
<< connected_client_id << " " << client_id_;
void ClientTable::HandleConnected(RedisGcsClient *client, const GcsNodeInfo &node_info) {
auto connected_node_id = ClientID::FromBinary(node_info.node_id());
RAY_CHECK(node_id_ == connected_node_id) << connected_node_id << " " << node_id_;
}
const ClientID &ClientTable::GetLocalClientId() const { return client_id_; }
const ClientID &ClientTable::GetLocalClientId() const { return node_id_; }
const ClientTableData &ClientTable::GetLocalClient() const { return local_client_; }
const GcsNodeInfo &ClientTable::GetLocalClient() const { return local_node_info_; }
bool ClientTable::IsRemoved(const ClientID &client_id) const {
return removed_clients_.count(client_id) == 1;
bool ClientTable::IsRemoved(const ClientID &node_id) const {
return removed_nodes_.count(node_id) == 1;
}
Status ClientTable::Connect(const ClientTableData &local_client) {
Status ClientTable::Connect(const GcsNodeInfo &local_node_info) {
RAY_CHECK(!disconnected_) << "Tried to reconnect a disconnected client.";
RAY_CHECK(local_client.client_id() == local_client_.client_id());
local_client_ = local_client;
RAY_CHECK(local_node_info.node_id() == local_node_info_.node_id());
local_node_info_ = local_node_info;
// Construct the data to add to the client table.
auto data = std::make_shared<ClientTableData>(local_client_);
data->set_is_insertion(true);
auto data = std::make_shared<GcsNodeInfo>(local_node_info_);
data->set_state(GcsNodeInfo::ALIVE);
// Callback to handle our own successful connection once we've added
// ourselves.
auto add_callback = [this](RedisGcsClient *client, const UniqueID &log_key,
const ClientTableData &data) {
const GcsNodeInfo &data) {
RAY_CHECK(log_key == client_log_key_);
HandleConnected(client, data);
// Callback for a notification from the client table.
auto notification_callback = [this](
RedisGcsClient *client, const UniqueID &log_key,
const std::vector<ClientTableData> &notifications) {
auto notification_callback = [this](RedisGcsClient *client, const UniqueID &log_key,
const std::vector<GcsNodeInfo> &notifications) {
RAY_CHECK(log_key == client_log_key_);
std::unordered_map<std::string, ClientTableData> connected_nodes;
std::unordered_map<std::string, ClientTableData> disconnected_nodes;
std::unordered_map<std::string, GcsNodeInfo> connected_nodes;
std::unordered_map<std::string, GcsNodeInfo> disconnected_nodes;
for (auto &notification : notifications) {
// This is temporary fix for Issue 4140 to avoid connect to dead nodes.
// TODO(yuhguo): remove this temporary fix after GCS entry is removable.
if (notification.is_insertion()) {
connected_nodes.emplace(notification.client_id(), notification);
if (notification.state() == GcsNodeInfo::ALIVE) {
connected_nodes.emplace(notification.node_id(), notification);
} else {
auto iter = connected_nodes.find(notification.client_id());
auto iter = connected_nodes.find(notification.node_id());
if (iter != connected_nodes.end()) {
connected_nodes.erase(iter);
}
disconnected_nodes.emplace(notification.client_id(), notification);
disconnected_nodes.emplace(notification.node_id(), notification);
}
}
for (const auto &pair : connected_nodes) {
@@ -624,52 +621,51 @@ Status ClientTable::Connect(const ClientTableData &local_client) {
// Callback to request notifications from the client table once we've
// successfully subscribed.
auto subscription_callback = [this](RedisGcsClient *c) {
RAY_CHECK_OK(RequestNotifications(JobID::Nil(), client_log_key_, client_id_));
RAY_CHECK_OK(RequestNotifications(JobID::Nil(), client_log_key_, node_id_));
};
// Subscribe to the client table.
RAY_CHECK_OK(Subscribe(JobID::Nil(), client_id_, notification_callback,
subscription_callback));
RAY_CHECK_OK(
Subscribe(JobID::Nil(), node_id_, notification_callback, subscription_callback));
};
return Append(JobID::Nil(), client_log_key_, data, add_callback);
}
Status ClientTable::Disconnect(const DisconnectCallback &callback) {
auto data = std::make_shared<ClientTableData>(local_client_);
data->set_is_insertion(false);
auto node_info = std::make_shared<GcsNodeInfo>(local_node_info_);
node_info->set_state(GcsNodeInfo::DEAD);
auto add_callback = [this, callback](RedisGcsClient *client, const ClientID &id,
const ClientTableData &data) {
const GcsNodeInfo &data) {
HandleConnected(client, data);
RAY_CHECK_OK(CancelNotifications(JobID::Nil(), client_log_key_, id));
if (callback != nullptr) {
callback();
}
};
RAY_RETURN_NOT_OK(Append(JobID::Nil(), client_log_key_, data, add_callback));
RAY_RETURN_NOT_OK(Append(JobID::Nil(), client_log_key_, node_info, add_callback));
// We successfully added the deletion entry. Mark ourselves as disconnected.
disconnected_ = true;
return Status::OK();
}
ray::Status ClientTable::MarkDisconnected(const ClientID &dead_client_id) {
auto data = std::make_shared<ClientTableData>();
data->set_client_id(dead_client_id.Binary());
data->set_is_insertion(false);
return Append(JobID::Nil(), client_log_key_, data, nullptr);
ray::Status ClientTable::MarkDisconnected(const ClientID &dead_node_id) {
auto node_info = std::make_shared<GcsNodeInfo>();
node_info->set_node_id(dead_node_id.Binary());
node_info->set_state(GcsNodeInfo::DEAD);
return Append(JobID::Nil(), client_log_key_, node_info, nullptr);
}
void ClientTable::GetClient(const ClientID &client_id,
ClientTableData &client_info) const {
RAY_CHECK(!client_id.IsNil());
auto entry = client_cache_.find(client_id);
if (entry != client_cache_.end()) {
client_info = entry->second;
void ClientTable::GetClient(const ClientID &node_id, GcsNodeInfo &node_info) const {
RAY_CHECK(!node_id.IsNil());
auto entry = node_cache_.find(node_id);
if (entry != node_cache_.end()) {
node_info = entry->second;
} else {
client_info.set_client_id(ClientID::Nil().Binary());
node_info.set_node_id(ClientID::Nil().Binary());
}
}
const std::unordered_map<ClientID, ClientTableData> &ClientTable::GetAllClients() const {
return client_cache_;
const std::unordered_map<ClientID, GcsNodeInfo> &ClientTable::GetAllClients() const {
return node_cache_;
}
Status ClientTable::Lookup(const Callback &lookup) {
@@ -679,9 +675,9 @@ Status ClientTable::Lookup(const Callback &lookup) {
std::string ClientTable::DebugString() const {
std::stringstream result;
result << Log<ClientID, ClientTableData>::DebugString();
result << ", cache size: " << client_cache_.size()
<< ", num removed: " << removed_clients_.size();
result << Log<ClientID, GcsNodeInfo>::DebugString();
result << ", cache size: " << node_cache_.size()
<< ", num removed: " << removed_nodes_.size();
return result.str();
}
@@ -728,7 +724,7 @@ template class Table<TaskID, TaskLeaseData>;
template class Table<ClientID, HeartbeatTableData>;
template class Table<ClientID, HeartbeatBatchTableData>;
template class Log<JobID, ErrorTableData>;
template class Log<ClientID, ClientTableData>;
template class Log<ClientID, GcsNodeInfo>;
template class Log<JobID, JobTableData>;
template class Log<UniqueID, ProfileTableData>;
template class Table<ActorCheckpointID, ActorCheckpointData>;
src/ray/gcs/tables.h
+29 -29
View File
@@ -23,10 +23,10 @@ namespace gcs {
using rpc::ActorCheckpointData;
using rpc::ActorCheckpointIdData;
using rpc::ActorTableData;
using rpc::ClientTableData;
using rpc::ErrorTableData;
using rpc::GcsChangeMode;
using rpc::GcsEntry;
using rpc::GcsNodeInfo;
using rpc::HeartbeatBatchTableData;
using rpc::HeartbeatTableData;
using rpc::JobTableData;
@@ -820,34 +820,34 @@ class ProfileTable : private Log<UniqueID, ProfileTableData> {
/// it should append an entry to the log indicating that it is dead. A client
/// that is marked as dead should never again be marked as alive; if it needs
/// to reconnect, it must connect with a different ClientID.
class ClientTable : public Log<ClientID, ClientTableData> {
class ClientTable : public Log<ClientID, GcsNodeInfo> {
public:
using ClientTableCallback = std::function<void(
RedisGcsClient *client, const ClientID &id, const ClientTableData &data)>;
RedisGcsClient *client, const ClientID &id, const GcsNodeInfo &data)>;
using DisconnectCallback = std::function<void(void)>;
ClientTable(const std::vector<std::shared_ptr<RedisContext>> &contexts,
RedisGcsClient *client, const ClientID &client_id)
RedisGcsClient *client, const ClientID &node_id)
: Log(contexts, client),
// We set the client log's key equal to nil so that all instances of
// ClientTable have the same key.
client_log_key_(),
disconnected_(false),
client_id_(client_id),
local_client_() {
node_id_(node_id),
local_node_info_() {
pubsub_channel_ = TablePubsub::CLIENT_PUBSUB;
prefix_ = TablePrefix::CLIENT;
// Set the local client's ID.
local_client_.set_client_id(client_id.Binary());
// Set the local node's ID.
local_node_info_.set_node_id(node_id.Binary());
};
/// Connect as a client to the GCS. This registers us in the client table
/// and begins subscription to client table notifications.
///
/// \param Information about the connecting client. This must have the
/// same client_id as the one set in the client table.
/// \param local_node_info Information about the connecting client. This must have the
/// same id as the one set in the client table.
/// \return Status
ray::Status Connect(const ClientTableData &local_client);
ray::Status Connect(const GcsNodeInfo &local_node_info);
/// Disconnect the client from the GCS. The client ID assigned during
/// registration should never be reused after disconnecting.
@@ -858,9 +858,9 @@ class ClientTable : public Log<ClientID, ClientTableData> {
/// Mark a different client as disconnected. The client ID should never be
/// reused for a new client.
///
/// \param dead_client_id The ID of the client to mark as dead.
/// \param dead_node_id The ID of the client to mark as dead.
/// \return Status
ray::Status MarkDisconnected(const ClientID &dead_client_id);
ray::Status MarkDisconnected(const ClientID &dead_node_id);
/// Register a callback to call when a new client is added.
///
@@ -876,11 +876,11 @@ class ClientTable : public Log<ClientID, ClientTableData> {
/// information for clients that we've heard a notification for.
///
/// \param client The client to get information about.
/// \param A reference to the client information. If we have information
/// \param node_info A reference to the client information. If we have information
/// about the client in the cache, then the reference will be modified to
/// contain that information. Else, the reference will be updated to contain
/// a nil client ID.
void GetClient(const ClientID &client, ClientTableData &client_info) const;
void GetClient(const ClientID &client, GcsNodeInfo &node_info) const;
/// Get the local client's ID.
///
@@ -890,18 +890,18 @@ class ClientTable : public Log<ClientID, ClientTableData> {
/// Get the local client's information.
///
/// \return The local client's information.
const ClientTableData &GetLocalClient() const;
const GcsNodeInfo &GetLocalClient() const;
/// Check whether the given client is removed.
///
/// \param client_id The ID of the client to check.
/// \param node_id The ID of the client to check.
/// \return Whether the client with ID client_id is removed.
bool IsRemoved(const ClientID &client_id) const;
bool IsRemoved(const ClientID &node_id) const;
/// Get the information of all clients.
///
/// \return The client ID to client information map.
const std::unordered_map<ClientID, ClientTableData> &GetAllClients() const;
const std::unordered_map<ClientID, GcsNodeInfo> &GetAllClients() const;
/// Lookup the client data in the client table.
///
@@ -922,23 +922,23 @@ class ClientTable : public Log<ClientID, ClientTableData> {
private:
/// Handle a client table notification.
void HandleNotification(RedisGcsClient *client, const ClientTableData &notifications);
void HandleNotification(RedisGcsClient *client, const GcsNodeInfo &node_info);
/// Handle this client's successful connection to the GCS.
void HandleConnected(RedisGcsClient *client, const ClientTableData &client_data);
void HandleConnected(RedisGcsClient *client, const GcsNodeInfo &node_info);
/// Whether this client has called Disconnect().
bool disconnected_;
/// This client's ID.
const ClientID client_id_;
/// Information about this client.
ClientTableData local_client_;
/// This node's ID.
const ClientID node_id_;
/// Information about this node.
GcsNodeInfo local_node_info_;
/// The callback to call when a new client is added.
ClientTableCallback client_added_callback_;
/// The callback to call when a client is removed.
ClientTableCallback client_removed_callback_;
/// A cache for information about all clients.
std::unordered_map<ClientID, ClientTableData> client_cache_;
/// The set of removed clients.
std::unordered_set<ClientID> removed_clients_;
/// A cache for information about all nodes.
std::unordered_map<ClientID, GcsNodeInfo> node_cache_;
/// The set of removed nodes.
std::unordered_set<ClientID> removed_nodes_;
};
} // namespace gcs
src/ray/object_manager/object_directory.cc
+7 -7
View File
@@ -8,8 +8,8 @@ ObjectDirectory::ObjectDirectory(boost::asio::io_service &io_service,
namespace {
using ray::rpc::ClientTableData;
using ray::rpc::GcsChangeMode;
using ray::rpc::GcsNodeInfo;
using ray::rpc::ObjectTableData;
/// Process a notification of the object table entries and store the result in
@@ -106,14 +106,14 @@ ray::Status ObjectDirectory::ReportObjectRemoved(
void ObjectDirectory::LookupRemoteConnectionInfo(
RemoteConnectionInfo &connection_info) const {
ClientTableData client_data;
gcs_client_->client_table().GetClient(connection_info.client_id, client_data);
ClientID result_client_id = ClientID::FromBinary(client_data.client_id());
GcsNodeInfo node_info;
gcs_client_->client_table().GetClient(connection_info.client_id, node_info);
ClientID result_client_id = ClientID::FromBinary(node_info.node_id());
if (!result_client_id.IsNil()) {
RAY_CHECK(result_client_id == connection_info.client_id);
if (client_data.is_insertion()) {
connection_info.ip = client_data.node_manager_address();
connection_info.port = static_cast<uint16_t>(client_data.object_manager_port());
if (node_info.state() == GcsNodeInfo::ALIVE) {
connection_info.ip = node_info.node_manager_address();
connection_info.port = static_cast<uint16_t>(node_info.object_manager_port());
}
}
}
src/ray/object_manager/test/object_manager_stress_test.cc
+12 -13
View File
@@ -11,7 +11,7 @@
namespace ray {
using rpc::ClientTableData;
using rpc::GcsNodeInfo;
std::string store_executable;
@@ -45,12 +45,12 @@ class MockServer {
private:
ray::Status RegisterGcs(boost::asio::io_service &io_service) {
auto object_manager_port = config_.object_manager_port;
ClientTableData client_info = gcs_client_->client_table().GetLocalClient();
client_info.set_node_manager_address("127.0.0.1");
client_info.set_node_manager_port(object_manager_port);
client_info.set_object_manager_port(object_manager_port);
GcsNodeInfo node_info = gcs_client_->client_table().GetLocalClient();
node_info.set_node_manager_address("127.0.0.1");
node_info.set_node_manager_port(object_manager_port);
node_info.set_object_manager_port(object_manager_port);
ray::Status status = gcs_client_->client_table().Connect(client_info);
ray::Status status = gcs_client_->client_table().Connect(node_info);
object_manager_.RegisterGcs();
return status;
}
@@ -215,9 +215,8 @@ class StressTestObjectManager : public TestObjectManagerBase {
client_id_1 = gcs_client_1->client_table().GetLocalClientId();
client_id_2 = gcs_client_2->client_table().GetLocalClientId();
gcs_client_1->client_table().RegisterClientAddedCallback(
[this](gcs::RedisGcsClient *client, const ClientID &id,
const ClientTableData &data) {
ClientID parsed_id = ClientID::FromBinary(data.client_id());
[this](gcs::RedisGcsClient *client, const ClientID &id, const GcsNodeInfo &data) {
ClientID parsed_id = ClientID::FromBinary(data.node_id());
if (parsed_id == client_id_1 || parsed_id == client_id_2) {
num_connected_clients += 1;
}
@@ -412,14 +411,14 @@ class StressTestObjectManager : public TestObjectManagerBase {
RAY_LOG(DEBUG) << "\n"
<< "All connected clients:"
<< "\n";
ClientTableData data;
GcsNodeInfo data;
gcs_client_1->client_table().GetClient(client_id_1, data);
RAY_LOG(DEBUG) << "ClientID=" << ClientID::FromBinary(data.client_id()) << "\n"
RAY_LOG(DEBUG) << "ClientID=" << ClientID::FromBinary(data.node_id()) << "\n"
<< "ClientIp=" << data.node_manager_address() << "\n"
<< "ClientPort=" << data.node_manager_port();
ClientTableData data2;
GcsNodeInfo data2;
gcs_client_1->client_table().GetClient(client_id_2, data2);
RAY_LOG(DEBUG) << "ClientID=" << ClientID::FromBinary(data2.client_id()) << "\n"
RAY_LOG(DEBUG) << "ClientID=" << ClientID::FromBinary(data2.node_id()) << "\n"
<< "ClientIp=" << data2.node_manager_address() << "\n"
<< "ClientPort=" << data2.node_manager_port();
}
src/ray/object_manager/test/object_manager_test.cc
+15 -16
View File
@@ -14,7 +14,7 @@ int64_t wait_timeout_ms;
namespace ray {
using rpc::ClientTableData;
using rpc::GcsNodeInfo;
static inline void flushall_redis(void) {
redisContext *context = redisConnect("127.0.0.1", 6379);
@@ -39,12 +39,12 @@ class MockServer {
private:
ray::Status RegisterGcs(boost::asio::io_service &io_service) {
auto object_manager_port = config_.object_manager_port;
ClientTableData client_info = gcs_client_->client_table().GetLocalClient();
client_info.set_node_manager_address("127.0.0.1");
client_info.set_node_manager_port(object_manager_port);
client_info.set_object_manager_port(object_manager_port);
GcsNodeInfo node_info = gcs_client_->client_table().GetLocalClient();
node_info.set_node_manager_address("127.0.0.1");
node_info.set_node_manager_port(object_manager_port);
node_info.set_object_manager_port(object_manager_port);
ray::Status status = gcs_client_->client_table().Connect(client_info);
ray::Status status = gcs_client_->client_table().Connect(node_info);
object_manager_.RegisterGcs();
return status;
}
@@ -196,9 +196,8 @@ class TestObjectManager : public TestObjectManagerBase {
client_id_1 = gcs_client_1->client_table().GetLocalClientId();
client_id_2 = gcs_client_2->client_table().GetLocalClientId();
gcs_client_1->client_table().RegisterClientAddedCallback(
[this](gcs::RedisGcsClient *client, const ClientID &id,
const ClientTableData &data) {
ClientID parsed_id = ClientID::FromBinary(data.client_id());
[this](gcs::RedisGcsClient *client, const ClientID &id, const GcsNodeInfo &data) {
ClientID parsed_id = ClientID::FromBinary(data.node_id());
if (parsed_id == client_id_1 || parsed_id == client_id_2) {
num_connected_clients += 1;
}
@@ -434,19 +433,19 @@ class TestObjectManager : public TestObjectManagerBase {
RAY_LOG(DEBUG) << "\n"
<< "Server client ids:"
<< "\n";
ClientTableData data;
GcsNodeInfo data;
gcs_client_1->client_table().GetClient(client_id_1, data);
RAY_LOG(DEBUG) << (ClientID::FromBinary(data.client_id()).IsNil());
RAY_LOG(DEBUG) << "Server 1 ClientID=" << ClientID::FromBinary(data.client_id());
RAY_LOG(DEBUG) << (ClientID::FromBinary(data.node_id()).IsNil());
RAY_LOG(DEBUG) << "Server 1 ClientID=" << ClientID::FromBinary(data.node_id());
RAY_LOG(DEBUG) << "Server 1 ClientIp=" << data.node_manager_address();
RAY_LOG(DEBUG) << "Server 1 ClientPort=" << data.node_manager_port();
ASSERT_EQ(client_id_1, ClientID::FromBinary(data.client_id()));
ClientTableData data2;
ASSERT_EQ(client_id_1, ClientID::FromBinary(data.node_id()));
GcsNodeInfo data2;
gcs_client_1->client_table().GetClient(client_id_2, data2);
RAY_LOG(DEBUG) << "Server 2 ClientID=" << ClientID::FromBinary(data2.client_id());
RAY_LOG(DEBUG) << "Server 2 ClientID=" << ClientID::FromBinary(data2.node_id());
RAY_LOG(DEBUG) << "Server 2 ClientIp=" << data2.node_manager_address();
RAY_LOG(DEBUG) << "Server 2 ClientPort=" << data2.node_manager_port();
ASSERT_EQ(client_id_2, ClientID::FromBinary(data2.client_id()));
ASSERT_EQ(client_id_2, ClientID::FromBinary(data2.node_id()));
}
};
src/ray/protobuf/gcs.proto
+18 -14
View File
@@ -151,28 +151,32 @@ message ResourceTableData {
double resource_capacity = 1;
}
message ClientTableData {
// The client ID of the client that the message is about.
bytes client_id = 1;
// The IP address of the client's node manager.
message GcsNodeInfo {
// State of a node.
enum GcsNodeState {
// Node is alive.
ALIVE = 0;
// Node is dead.
DEAD = 1;
}
// The ID of node.
bytes node_id = 1;
// The IP address of the node manager.
string node_manager_address = 2;
// The IPC socket name of the client's raylet.
// The IPC socket name of raylet.
string raylet_socket_name = 3;
// The IPC socket name of the client's plasma store.
// The IPC socket name of the node's plasma store.
string object_store_socket_name = 4;
// The port at which the client's node manager is listening for TCP
// The port at which the node manager is listening for TCP
// connections from other node managers.
int32 node_manager_port = 5;
// The port at which the client's object manager is listening for TCP
// The port at which the object manager is listening for TCP
// connections from other object managers.
int32 object_manager_port = 6;
// True if the message is about the addition of a client and false if it is
// about the deletion of a client.
bool is_insertion = 7;
// TODO(hchen): Define the following resources in map format.
repeated string resources_total_label = 8;
repeated double resources_total_capacity = 9;
// Current state of this node.
GcsNodeState state = 7;
}
message HeartbeatTableData {
src/ray/raylet/monitor.cc
+4 -3
View File
@@ -49,10 +49,11 @@ void Monitor::Tick() {
RAY_LOG(WARNING) << "Client timed out: " << client_id;
auto lookup_callback = [this, client_id](
gcs::RedisGcsClient *client, const ClientID &id,
const std::vector<ClientTableData> &all_data) {
const std::vector<GcsNodeInfo> &all_node) {
bool marked = false;
for (const auto &data : all_data) {
if (client_id.Binary() == data.client_id() && !data.is_insertion()) {
for (const auto &node : all_node) {
if (client_id.Binary() == node.node_id() &&
node.state() == GcsNodeInfo::DEAD) {
// The node has been marked dead by itself.
marked = true;
}
src/ray/raylet/monitor.h
+1 -1
View File
@@ -11,7 +11,7 @@ namespace ray {
namespace raylet {
using rpc::ClientTableData;
using rpc::GcsNodeInfo;
using rpc::HeartbeatBatchTableData;
using rpc::HeartbeatTableData;
src/ray/raylet/node_manager.cc
+11 -12
View File
@@ -176,14 +176,13 @@ ray::Status NodeManager::RegisterGcs() {
// Register a callback on the client table for new clients.
auto node_manager_client_added = [this](gcs::RedisGcsClient *client, const UniqueID &id,
const ClientTableData &data) {
ClientAdded(data);
};
const GcsNodeInfo &data) { ClientAdded(data); };
gcs_client_->client_table().RegisterClientAddedCallback(node_manager_client_added);
// Register a callback on the client table for removed clients.
auto node_manager_client_removed =
[this](gcs::RedisGcsClient *client, const UniqueID &id,
const ClientTableData &data) { ClientRemoved(data); };
auto node_manager_client_removed = [this](gcs::RedisGcsClient *client,
const UniqueID &id, const GcsNodeInfo &data) {
ClientRemoved(data);
};
gcs_client_->client_table().RegisterClientRemovedCallback(node_manager_client_removed);
// Subscribe to resource changes.
@@ -381,8 +380,8 @@ void NodeManager::GetObjectManagerProfileInfo() {
}
}
void NodeManager::ClientAdded(const ClientTableData &client_data) {
const ClientID client_id = ClientID::FromBinary(client_data.client_id());
void NodeManager::ClientAdded(const GcsNodeInfo &node_info) {
const ClientID client_id = ClientID::FromBinary(node_info.node_id());
RAY_LOG(DEBUG) << "[ClientAdded] Received callback from client id " << client_id;
if (client_id == gcs_client_->client_table().GetLocalClientId()) {
@@ -401,8 +400,8 @@ void NodeManager::ClientAdded(const ClientTableData &client_data) {
// Initialize a rpc client to the new node manager.
std::unique_ptr<rpc::NodeManagerClient> client(
new rpc::NodeManagerClient(client_data.node_manager_address(),
client_data.node_manager_port(), client_call_manager_));
new rpc::NodeManagerClient(node_info.node_manager_address(),
node_info.node_manager_port(), client_call_manager_));
remote_node_manager_clients_.emplace(client_id, std::move(client));
// Fetch resource info for the remote client and update cluster resource map.
@@ -420,10 +419,10 @@ void NodeManager::ClientAdded(const ClientTableData &client_data) {
}));
}
void NodeManager::ClientRemoved(const ClientTableData &client_data) {
void NodeManager::ClientRemoved(const GcsNodeInfo &node_info) {
// TODO(swang): If we receive a notification for our own death, clean up and
// exit immediately.
const ClientID client_id = ClientID::FromBinary(client_data.client_id());
const ClientID client_id = ClientID::FromBinary(node_info.node_id());
RAY_LOG(DEBUG) << "[ClientRemoved] Received callback from client id " << client_id;
RAY_CHECK(client_id != gcs_client_->client_table().GetLocalClientId())
src/ray/raylet/node_manager.h
+4 -4
View File
@@ -29,8 +29,8 @@ namespace ray {
namespace raylet {
using rpc::ActorTableData;
using rpc::ClientTableData;
using rpc::ErrorType;
using rpc::GcsNodeInfo;
using rpc::HeartbeatBatchTableData;
using rpc::HeartbeatTableData;
using rpc::JobTableData;
@@ -175,12 +175,12 @@ class NodeManager : public rpc::NodeManagerServiceHandler,
///
/// \param data Data associated with the new client.
/// \return Void.
void ClientAdded(const ClientTableData &data);
void ClientAdded(const GcsNodeInfo &data);
/// Handler for the removal of a GCS client.
/// \param client_data Data associated with the removed client.
/// \param node_info Data associated with the removed client.
/// \return Void.
void ClientRemoved(const ClientTableData &client_data);
void ClientRemoved(const GcsNodeInfo &node_info);
/// Handler for the addition or updation of a resource in the GCS
/// \param client_id ID of the node that created or updated resources.
src/ray/raylet/object_manager_integration_test.cc
+11 -11
View File
@@ -139,8 +139,8 @@ class TestObjectManagerIntegration : public TestObjectManagerBase {
client_id_2 = gcs_client_2->client_table().GetLocalClientId();
gcs_client_1->client_table().RegisterClientAddedCallback(
[this](gcs::RedisGcsClient *client, const ClientID &id,
const ClientTableDataT &data) {
ClientID parsed_id = ClientID::FromBinary(data.client_id);
const rpc::GcsNodeInfo &data) {
ClientID parsed_id = ClientID::FromBinary(data.node_id);
if (parsed_id == client_id_1 || parsed_id == client_id_2) {
num_connected_clients += 1;
}
@@ -208,17 +208,17 @@ class TestObjectManagerIntegration : public TestObjectManagerBase {
RAY_LOG(INFO) << "\n"
<< "All connected clients:"
<< "\n";
ClientTableDataT data;
rpc::GcsNodeInfo data;
gcs_client_2->client_table().GetClient(client_id_1, data);
RAY_LOG(INFO) << (ClientID::FromBinary(data.client_id).IsNil());
RAY_LOG(INFO) << "ClientID=" << ClientID::FromBinary(data.client_id);
RAY_LOG(INFO) << "ClientIp=" << data.node_manager_address;
RAY_LOG(INFO) << "ClientPort=" << data.node_manager_port;
ClientTableDataT data2;
RAY_LOG(INFO) << (ClientID::FromBinary(data.node_id()).IsNil());
RAY_LOG(INFO) << "ClientID=" << ClientID::FromBinary(data.node_id());
RAY_LOG(INFO) << "ClientIp=" << data.node_manager_address();
RAY_LOG(INFO) << "ClientPort=" << data.node_manager_port();
rpc::GcsNodeInfo data2;
gcs_client_1->client_table().GetClient(client_id_2, data2);
RAY_LOG(INFO) << "ClientID=" << ClientID::FromBinary(data2.client_id);
RAY_LOG(INFO) << "ClientIp=" << data2.node_manager_address;
RAY_LOG(INFO) << "ClientPort=" << data2.node_manager_port;
RAY_LOG(INFO) << "ClientID=" << ClientID::FromBinary(data2.node_id());
RAY_LOG(INFO) << "ClientIp=" << data2.node_manager_address();
RAY_LOG(INFO) << "ClientPort=" << data2.node_manager_port();
}
};
src/ray/raylet/raylet.cc
+11 -11
View File
@@ -50,20 +50,20 @@ ray::Status Raylet::RegisterGcs(const std::string &node_ip_address,
const std::string &redis_password,
boost::asio::io_service &io_service,
const NodeManagerConfig &node_manager_config) {
ClientTableData client_info = gcs_client_->client_table().GetLocalClient();
client_info.set_node_manager_address(node_ip_address);
client_info.set_raylet_socket_name(raylet_socket_name);
client_info.set_object_store_socket_name(object_store_socket_name);
client_info.set_object_manager_port(object_manager_.GetServerPort());
client_info.set_node_manager_port(node_manager_.GetServerPort());
GcsNodeInfo node_info = gcs_client_->client_table().GetLocalClient();
node_info.set_node_manager_address(node_ip_address);
node_info.set_raylet_socket_name(raylet_socket_name);
node_info.set_object_store_socket_name(object_store_socket_name);
node_info.set_object_manager_port(object_manager_.GetServerPort());
node_info.set_node_manager_port(node_manager_.GetServerPort());
RAY_LOG(DEBUG) << "Node manager " << gcs_client_->client_table().GetLocalClientId()
<< " started on " << client_info.node_manager_address() << ":"
<< client_info.node_manager_port() << " object manager at "
<< client_info.node_manager_address() << ":"
<< client_info.object_manager_port();
<< " started on " << node_info.node_manager_address() << ":"
<< node_info.node_manager_port() << " object manager at "
<< node_info.node_manager_address() << ":"
<< node_info.object_manager_port();
;
RAY_RETURN_NOT_OK(gcs_client_->client_table().Connect(client_info));
RAY_RETURN_NOT_OK(gcs_client_->client_table().Connect(node_info));
// Add resource information.
std::unordered_map<std::string, std::shared_ptr<gcs::ResourceTableData>> resources;
src/ray/raylet/raylet.h
+1 -1
View File
@@ -16,7 +16,7 @@ namespace ray {
namespace raylet {
using rpc::ClientTableData;
using rpc::GcsNodeInfo;
class NodeManager;