mirror of
https://github.com/wassname/ray.git
synced 2026-07-08 03:46:18 +08:00
Treat actor creation like a regular task. (#1668)
* Treat actor creation like a regular task. * Small cleanups. * Change semantics of actor resource handling. * Bug fix. * Minor linting * Bug fix * Fix jenkins test. * Fix actor tests * Some cleanups * Bug fix * Fix bug. * Remove cached actor tasks when a driver is removed. * Add more info to taskspec in global state API. * Fix cyclic import bug in tune. * Fix * Fix linting. * Fix linting. * Don't schedule any tasks (especially actor creaiton tasks) on local schedulers with 0 CPUs. * Bug fix. * Add test for 0 CPU case * Fix linting * Address comments. * Fix typos and add comment. * Add assertion and fix test.
This commit is contained in:
Robert Nishihara
committed by
Stephanie Wang
Stephanie Wang
parent
3c080f4baa
commit
96913be939
@@ -4,7 +4,6 @@ from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
@@ -115,43 +114,6 @@ class Monitor(object):
|
||||
self.subscribe_client.subscribe(channel)
|
||||
self.subscribed[channel] = False
|
||||
|
||||
def cleanup_actors(self):
|
||||
"""Recreate any live actors whose corresponding local scheduler died.
|
||||
|
||||
For any live actor whose local scheduler just died, we choose a new
|
||||
local scheduler and broadcast a notification to create that actor.
|
||||
"""
|
||||
actor_info = self.state.actors()
|
||||
for actor_id, info in actor_info.items():
|
||||
if (not info["removed"] and
|
||||
info["local_scheduler_id"] in self.dead_local_schedulers):
|
||||
# Choose a new local scheduler to run the actor.
|
||||
local_scheduler_id = ray.utils.select_local_scheduler(
|
||||
info["driver_id"],
|
||||
self.state.local_schedulers(), info["num_gpus"],
|
||||
self.redis)
|
||||
import sys
|
||||
sys.stdout.flush()
|
||||
# The new local scheduler should not be the same as the old
|
||||
# local scheduler. TODO(rkn): This should not be an assert, it
|
||||
# should be something more benign.
|
||||
assert (binary_to_hex(local_scheduler_id) !=
|
||||
info["local_scheduler_id"])
|
||||
# Announce to all of the local schedulers that the actor should
|
||||
# be recreated on this new local scheduler.
|
||||
ray.utils.publish_actor_creation(
|
||||
hex_to_binary(actor_id),
|
||||
hex_to_binary(info["driver_id"]), local_scheduler_id, True,
|
||||
self.redis)
|
||||
log.info("Actor {} for driver {} was on dead local scheduler "
|
||||
"{}. It is being recreated on local scheduler {}"
|
||||
.format(actor_id, info["driver_id"],
|
||||
info["local_scheduler_id"],
|
||||
binary_to_hex(local_scheduler_id)))
|
||||
# Update the actor info in Redis.
|
||||
self.redis.hset(b"Actor:" + hex_to_binary(actor_id),
|
||||
"local_scheduler_id", local_scheduler_id)
|
||||
|
||||
def cleanup_task_table(self):
|
||||
"""Clean up global state for failed local schedulers.
|
||||
|
||||
@@ -473,58 +435,8 @@ class Monitor(object):
|
||||
log.info(
|
||||
"Driver {} has been removed.".format(binary_to_hex(driver_id)))
|
||||
|
||||
# Get a list of the local schedulers that have not been deleted.
|
||||
local_schedulers = ray.global_state.local_schedulers()
|
||||
|
||||
self._clean_up_entries_for_driver(driver_id)
|
||||
|
||||
# Release any GPU resources that have been reserved for this driver in
|
||||
# Redis.
|
||||
for local_scheduler in local_schedulers:
|
||||
if local_scheduler.get("GPU", 0) > 0:
|
||||
local_scheduler_id = local_scheduler["DBClientID"]
|
||||
|
||||
num_gpus_returned = 0
|
||||
|
||||
# Perform a transaction to return the GPUs.
|
||||
with self.redis.pipeline() as pipe:
|
||||
while True:
|
||||
try:
|
||||
# If this key is changed before the transaction
|
||||
# below (the multi/exec block), then the
|
||||
# transaction will not take place.
|
||||
pipe.watch(local_scheduler_id)
|
||||
|
||||
result = pipe.hget(local_scheduler_id,
|
||||
"gpus_in_use")
|
||||
gpus_in_use = (dict() if result is None else
|
||||
json.loads(result.decode("ascii")))
|
||||
|
||||
driver_id_hex = binary_to_hex(driver_id)
|
||||
if driver_id_hex in gpus_in_use:
|
||||
num_gpus_returned = gpus_in_use.pop(
|
||||
driver_id_hex)
|
||||
|
||||
pipe.multi()
|
||||
|
||||
pipe.hset(local_scheduler_id, "gpus_in_use",
|
||||
json.dumps(gpus_in_use))
|
||||
|
||||
pipe.execute()
|
||||
# If a WatchError is not raise, then the operations
|
||||
# should have gone through atomically.
|
||||
break
|
||||
except redis.WatchError:
|
||||
# Another client must have changed the watched key
|
||||
# between the time we started WATCHing it and the
|
||||
# pipeline's execution. We should just retry.
|
||||
continue
|
||||
|
||||
log.info("Driver {} is returning GPU IDs {} to local "
|
||||
"scheduler {}.".format(
|
||||
binary_to_hex(driver_id), num_gpus_returned,
|
||||
local_scheduler_id))
|
||||
|
||||
def process_messages(self):
|
||||
"""Process all messages ready in the subscription channels.
|
||||
|
||||
@@ -592,7 +504,6 @@ class Monitor(object):
|
||||
# state in the state tables.
|
||||
if len(self.dead_local_schedulers) > 0:
|
||||
self.cleanup_task_table()
|
||||
self.cleanup_actors()
|
||||
if len(self.dead_plasma_managers) > 0:
|
||||
self.cleanup_object_table()
|
||||
log.debug("{} dead local schedulers, {} plasma managers total, {} "
|
||||
@@ -617,7 +528,6 @@ class Monitor(object):
|
||||
# dead in this round, clean up the associated state.
|
||||
if len(self.dead_local_schedulers) > num_dead_local_schedulers:
|
||||
self.cleanup_task_table()
|
||||
self.cleanup_actors()
|
||||
if len(self.dead_plasma_managers) > num_dead_plasma_managers:
|
||||
self.cleanup_object_table()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user