wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-21 12:50:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Ray scheduler spillback plumbing + mechanism (#1362)

Browse Source 
* spillback mechanism and plumbing : adding spillback counter + timestamp

* linting fix

* documentation

* Fix argument name.
This commit is contained in:
Alexey Tumanov
2018-01-23 20:18:12 -08:00
committed by Robert Nishihara
parent 21a916009e
commit f1303291b4
11 changed files with 153 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/common/redis_module/runtest.py
+7 -5
View File
@@ -322,16 +322,18 @@ class TestGlobalStateStore(unittest.TestCase):
def check_task_reply(message, task_args, updated=False):
(task_status, local_scheduler_id, execution_dependencies_string,
task_spec) = task_args
spillback_count, task_spec) = task_args
task_reply_object = TaskReply.GetRootAsTaskReply(message, 0)
self.assertEqual(task_reply_object.State(), task_status)
self.assertEqual(task_reply_object.LocalSchedulerId(),
local_scheduler_id)
self.assertEqual(task_reply_object.SpillbackCount(),
spillback_count)
self.assertEqual(task_reply_object.TaskSpec(), task_spec)
self.assertEqual(task_reply_object.Updated(), updated)
# Check that task table adds, updates, and lookups work correctly.
task_args = [TASK_STATUS_WAITING, b"node_id", b"", b"task_spec"]
task_args = [TASK_STATUS_WAITING, b"node_id", b"", 0, b"task_spec"]
response = self.redis.execute_command("RAY.TASK_TABLE_ADD", "task_id",
*task_args)
response = self.redis.execute_command("RAY.TASK_TABLE_GET", "task_id")
@@ -339,7 +341,7 @@ class TestGlobalStateStore(unittest.TestCase):
task_args[0] = TASK_STATUS_SCHEDULED
self.redis.execute_command("RAY.TASK_TABLE_UPDATE", "task_id",
*task_args[:3])
*task_args[:4])
response = self.redis.execute_command("RAY.TASK_TABLE_GET", "task_id")
check_task_reply(response, task_args)
@@ -408,7 +410,7 @@ class TestGlobalStateStore(unittest.TestCase):
def check_task_subscription(self, p, scheduling_state, local_scheduler_id):
task_args = [b"task_id", scheduling_state,
local_scheduler_id.encode("ascii"), b"", b"task_spec"]
local_scheduler_id.encode("ascii"), b"", 0, b"task_spec"]
self.redis.execute_command("RAY.TASK_TABLE_ADD", *task_args)
# Receive the data.
message = get_next_message(p)["data"]
@@ -420,7 +422,7 @@ class TestGlobalStateStore(unittest.TestCase):
task_args[2])
self.assertEqual(notification_object.ExecutionDependencies(),
task_args[3])
self.assertEqual(notification_object.TaskSpec(), task_args[4])
self.assertEqual(notification_object.TaskSpec(), task_args[-1])
def testTaskTableSubscribe(self):
scheduling_state = 1
python/ray/experimental/state.py
+2
View File
@@ -273,6 +273,8 @@ class GlobalState(object):
task_table_message.LocalSchedulerId()),
"ExecutionDependenciesString":
task_table_message.ExecutionDependencies(),
"SpillbackCount":
task_table_message.SpillbackCount(),
"TaskSpec": task_spec_info}
def task_table(self, task_id=None):
python/ray/monitor.py
+2 -1
View File
@@ -198,7 +198,8 @@ class Monitor(object):
key, "RAY.TASK_TABLE_UPDATE",
hex_to_binary(task_id),
ray.experimental.state.TASK_STATUS_LOST, NIL_ID,
task["ExecutionDependenciesString"])
task["ExecutionDependenciesString"],
task["SpillbackCount"])
if ok != b"OK":
log.warn("Failed to update lost task for dead scheduler.")
num_tasks_updated += 1
python/ray/worker.py
+1
View File
@@ -1906,6 +1906,7 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
TASK_STATUS_RUNNING,
NIL_LOCAL_SCHEDULER_ID,
driver_task.execution_dependencies_string(),
0,
ray.local_scheduler.task_to_string(driver_task))
# Set the driver's current task ID to the task ID assigned to the
# driver task.