Render tasks that are not schedulable on the dashboard. (#7034)

This commit is contained in:
SangBin Cho
2020-02-10 14:23:06 -08:00
committed by GitHub
parent 3f99be8dad
commit 1e690673d8
7 changed files with 181 additions and 20 deletions
+3 -1
View File
@@ -103,7 +103,7 @@ export interface RayletInfoResponse {
actorTitle: string;
averageTaskExecutionSpeed: number;
children: RayletInfoResponse["actors"];
currentTaskFuncDesc: string[];
// currentTaskFuncDesc: string[];
ipAddress: string;
isDirectCall: boolean;
jobId: string;
@@ -124,8 +124,10 @@ export interface RayletInfoResponse {
}
| {
actorId: string;
actorTitle: string;
requiredResources: { [key: string]: number };
state: -1;
invalidStateType?: 'infeasibleActor' | 'pendingActor';
};
};
}
@@ -13,6 +13,7 @@ import {
} from "../../../api";
import Actors from "./Actors";
import Collapse from "@material-ui/core/Collapse";
import orange from '@material-ui/core/colors/orange';
const styles = (theme: Theme) =>
createStyles({
@@ -34,9 +35,12 @@ const styles = (theme: Theme) =>
cursor: "pointer"
}
},
infeasible: {
invalidStateTypeInfeasible: {
color: theme.palette.error.main
},
invalidStateTypePendingActor: {
color: orange[500]
},
information: {
fontSize: "0.875rem"
},
@@ -161,11 +165,11 @@ class Actor extends React.Component<Props & WithStyles<typeof styles>, State> {
{
label: "UsedLocalObjectMemory",
value: actor.usedObjectStoreMemory.toLocaleString()
},
{
label: "Task",
value: actor.currentTaskFuncDesc.join(".")
}
// {
// label: "Task",
// value: actor.currentTaskFuncDesc.join(".")
// }
]
: [
{
@@ -285,8 +289,16 @@ class Actor extends React.Component<Props & WithStyles<typeof styles>, State> {
)
)}
</React.Fragment>
) : actor.invalidStateType === 'infeasibleActor' ? (
<span className={classes.invalidStateTypeInfeasible}>
{actor.actorTitle} is infeasible.
(Infeasible actor means an actor cannot be created because
Ray cluster cannot satisfy resources requirement).
</span>
) : (
<span className={classes.infeasible}>Infeasible actor</span>
<span className={classes.invalidStateTypePendingActor}>
{actor.actorTitle} is pending until resources are available.
</span>
)}
</Typography>
<Typography className={classes.information}>
+26 -10
View File
@@ -201,8 +201,13 @@ class Dashboard(object):
}
infeasible_tasks = sum(
(data.get("infeasibleTasks", []) for data in D.values()), [])
# ready_tasks are used to render tasks that are not schedulable
# due to resource limitations.
# (e.g., Actor requires 2 GPUs but there is only 1 gpu available).
ready_tasks = sum(
(data.get("readyTasks", []) for data in D.values()), [])
actor_tree = self.node_stats.get_actor_tree(
workers_info_by_node, infeasible_tasks)
workers_info_by_node, infeasible_tasks, ready_tasks)
for address, data in D.items():
# process view data
measures_dicts = {}
@@ -418,7 +423,8 @@ class NodeStats(threading.Thread):
"error_counts": self.calculate_error_counts(),
}
def get_actor_tree(self, workers_info_by_node, infeasible_tasks) -> Dict:
def get_actor_tree(self, workers_info_by_node, infeasible_tasks,
ready_tasks) -> Dict:
now = time.time()
# construct flattened actor tree
flattened_tree = {"root": {"children": {}}}
@@ -449,17 +455,27 @@ class NodeStats(threading.Thread):
actor_info["nodeId"] = node_id
actor_info["pid"] = worker_info["pid"]
for infeasible_task in infeasible_tasks:
def _update_flatten_tree(task, task_spec_type, invalid_state_type):
actor_id = ray.utils.binary_to_hex(
b64decode(
infeasible_task["actorCreationTaskSpec"]["actorId"]))
caller_addr = (infeasible_task["callerAddress"]["ipAddress"],
str(infeasible_task["callerAddress"]["port"]))
b64decode(task[task_spec_type]["actorId"]))
caller_addr = (task["callerAddress"]["ipAddress"],
str(task["callerAddress"]["port"]))
caller_id = self._addr_to_actor_id.get(caller_addr, "root")
child_to_parent[actor_id] = caller_id
infeasible_task["state"] = -1
format_reply_id(infeasible_tasks)
flattened_tree[actor_id] = infeasible_task
task["state"] = -1
task["invalidStateType"] = invalid_state_type
task["actorTitle"] = task["functionDescriptor"][
"pythonFunctionDescriptor"]["className"]
format_reply_id(task)
flattened_tree[actor_id] = task
for infeasible_task in infeasible_tasks:
_update_flatten_tree(infeasible_task, "actorCreationTaskSpec",
"infeasibleActor")
for ready_task in ready_tasks:
_update_flatten_tree(ready_task, "actorCreationTaskSpec",
"pendingActor")
# construct actor tree
actor_tree = flattened_tree