wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-29 06:08:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

GCS server use worker table to handle RegisterWorker instead of redis accessor (#9168)

Browse Source
This commit is contained in:
ChenZhilei
2020-07-06 10:37:25 +08:00
committed by GitHub
parent dcf989292e
commit 6f3d993681
40 changed files with 653 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/state.py
+42 -16
View File
@@ -602,26 +602,52 @@ class GlobalState:
"""Get a dictionary mapping worker ID to worker information."""
self._check_connected()
worker_keys = self.redis_client.keys("Worker*")
# Get all data in worker table
worker_table = self.global_state_accessor.get_worker_table()
workers_data = {}
for i in range(len(worker_table)):
worker_table_data = gcs_utils.WorkerTableData.FromString(
worker_table[i])
if worker_table_data.is_alive and \
worker_table_data.worker_type == gcs_utils.WORKER:
worker_id = binary_to_hex(
worker_table_data.worker_address.worker_id)
worker_info = worker_table_data.worker_info
for worker_key in worker_keys:
worker_info = self.redis_client.hgetall(worker_key)
worker_id = binary_to_hex(worker_key[len("Workers:"):])
workers_data[worker_id] = {
"node_ip_address": decode(worker_info[b"node_ip_address"]),
"plasma_store_socket": decode(
worker_info[b"plasma_store_socket"])
}
if b"stderr_file" in worker_info:
workers_data[worker_id]["stderr_file"] = decode(
worker_info[b"stderr_file"])
if b"stdout_file" in worker_info:
workers_data[worker_id]["stdout_file"] = decode(
worker_info[b"stdout_file"])
workers_data[worker_id] = {
"node_ip_address": decode(worker_info[b"node_ip_address"]),
"plasma_store_socket": decode(
worker_info[b"plasma_store_socket"])
}
if b"stderr_file" in worker_info:
workers_data[worker_id]["stderr_file"] = decode(
worker_info[b"stderr_file"])
if b"stdout_file" in worker_info:
workers_data[worker_id]["stdout_file"] = decode(
worker_info[b"stdout_file"])
return workers_data
def add_worker(self, worker_id, worker_type, worker_info):
""" Add a worker to the cluster.
Args:
worker_id: ID of this worker. Type is bytes.
worker_type: Type of this worker. Value is ray.gcs_utils.DRIVER or
ray.gcs_utils.WORKER.
worker_info: Info of this worker. Type is dict{str: str}.
Returns:
Is operation success
"""
worker_data = ray.gcs_utils.WorkerTableData()
worker_data.is_alive = True
worker_data.worker_address.worker_id = worker_id
worker_data.worker_type = worker_type
for k, v in worker_info.items():
worker_data.worker_info[k] = bytes(v, encoding="utf-8")
return self.global_state_accessor.add_worker_info(
worker_data.SerializeToString())
def _job_length(self):
event_log_sets = self.redis_client.keys("event_log*")
overall_smallest = sys.maxsize