From 82be14f943687c935733d1adfd305fd3c3a0cdb4 Mon Sep 17 00:00:00 2001 From: Mitchell Stern Date: Tue, 5 Nov 2019 00:47:15 -0800 Subject: [PATCH] Move gRPC calls outside of Raylet stats lock (#6090) --- python/ray/dashboard/dashboard.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/ray/dashboard/dashboard.py b/python/ray/dashboard/dashboard.py index 6d7cc8e05..c7337cf82 100644 --- a/python/ray/dashboard/dashboard.py +++ b/python/ray/dashboard/dashboard.py @@ -404,14 +404,15 @@ class RayletStats(threading.Thread): counter = 0 while True: time.sleep(1.0) + replies = {} + for node in self.nodes: + node_id = node["NodeID"] + stub = self.stubs[node_id] + reply = stub.GetNodeStats(node_manager_pb2.NodeStatsRequest()) + replies[node["NodeManagerAddress"]] = reply with self._raylet_stats_lock: - for node in self.nodes: - node_id = node["NodeID"] - stub = self.stubs[node_id] - reply = stub.GetNodeStats( - node_manager_pb2.NodeStatsRequest()) - self._raylet_stats[node[ - "NodeManagerAddress"]] = MessageToDict(reply) + for address, reply in replies.items(): + self._raylet_stats[address] = MessageToDict(reply) counter += 1 # From time to time, check if new nodes have joined the cluster # and update self.nodes