diff --git a/python/ray/dashboard/client/src/api.ts b/python/ray/dashboard/client/src/api.ts index ee4d46c1e..1719da764 100644 --- a/python/ray/dashboard/client/src/api.ts +++ b/python/ray/dashboard/client/src/api.ts @@ -150,3 +150,31 @@ export interface LogsResponse { export const getLogs = (hostname: string, pid: string | undefined) => get("/api/logs", { hostname, pid: pid || "" }); + +export type LaunchProfilingResponse = string; + +export const launchProfiling = ( + nodeId: string, + pid: number, + duration: number +) => + get("/api/launch_profiling", { + node_id: nodeId, + pid: pid, + duration: duration + }); + +export type CheckProfilingStatusResponse = + | { status: "pending" } + | { status: "finished" } + | { status: "error"; error: string }; + +export const checkProfilingStatus = (profilingId: string) => + get("/api/check_profiling_status", { + profiling_id: profilingId + }); + +export const getProfilingResultURL = (profilingId: string) => + `${base}/speedscope/index.html#profileURL=${encodeURIComponent( + `${base}/api/get_profiling_info?profiling_id=${profilingId}` + )}`; diff --git a/python/ray/dashboard/client/src/pages/dashboard/logical-view/Actor.tsx b/python/ray/dashboard/client/src/pages/dashboard/logical-view/Actor.tsx index c42409cae..2475aed14 100644 --- a/python/ray/dashboard/client/src/pages/dashboard/logical-view/Actor.tsx +++ b/python/ray/dashboard/client/src/pages/dashboard/logical-view/Actor.tsx @@ -1,11 +1,17 @@ -import Typography from "@material-ui/core/Typography"; +import Collapse from "@material-ui/core/Collapse"; import { Theme } from "@material-ui/core/styles/createMuiTheme"; import createStyles from "@material-ui/core/styles/createStyles"; import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles"; +import Typography from "@material-ui/core/Typography"; import React from "react"; -import { RayletInfoResponse } from "../../../api"; +import { + checkProfilingStatus, + CheckProfilingStatusResponse, + getProfilingResultURL, + launchProfiling, + RayletInfoResponse +} from "../../../api"; import Actors from "./Actors"; -import Collapse from "@material-ui/core/Collapse"; const styles = (theme: Theme) => createStyles({ @@ -20,6 +26,13 @@ const styles = (theme: Theme) => color: theme.palette.text.secondary, fontSize: "0.75rem" }, + action: { + color: theme.palette.primary.main, + textDecoration: "none", + "&:hover": { + cursor: "pointer" + } + }, infeasible: { color: theme.palette.error.main }, @@ -33,12 +46,6 @@ const styles = (theme: Theme) => }, webuiDisplay: { fontSize: "0.875rem" - }, - expandCollapseButton: { - color: theme.palette.primary.main, - "&:hover": { - cursor: "pointer" - } } }); @@ -48,71 +55,104 @@ interface Props { interface State { expanded: boolean; + profiling: { + [profilingId: string]: { + startTime: number; + latestResponse: CheckProfilingStatusResponse | null; + }; + }; } class Actor extends React.Component, State> { state: State = { - expanded: true + expanded: true, + profiling: {} }; setExpanded = (expanded: boolean) => () => { this.setState({ expanded }); }; + handleProfilingClick = (duration: number) => async () => { + const actor = this.props.actor; + if (actor.state !== -1) { + const profilingId = await launchProfiling( + actor.nodeId, + actor.pid, + duration + ); + this.setState(state => ({ + profiling: { + ...state.profiling, + [profilingId]: { startTime: Date.now(), latestResponse: null } + } + })); + const checkProfilingStatusLoop = async () => { + const response = await checkProfilingStatus(profilingId); + this.setState(state => ({ + profiling: { + ...state.profiling, + [profilingId]: { + ...state.profiling[profilingId], + latestResponse: response + } + } + })); + if (response.status === "pending") { + setTimeout(checkProfilingStatusLoop, 1000); + } + }; + await checkProfilingStatusLoop(); + } + }; + render() { const { classes, actor } = this.props; - const { expanded } = this.state; + const { expanded, profiling } = this.state; const information = actor.state !== -1 ? [ { label: "ActorTitle", - value: - actor.actorTitle + value: actor.actorTitle }, { label: "State", - value: - actor.state.toLocaleString() + value: actor.state.toLocaleString() }, { label: "Resources", value: Object.entries(actor.usedResources).length > 0 && Object.entries(actor.usedResources) + .sort((a, b) => a[0].localeCompare(b[0])) .map(([key, value]) => `${value.toLocaleString()} ${key}`) .join(", ") }, { label: "Pending", - value: - actor.taskQueueLength.toLocaleString() + value: actor.taskQueueLength.toLocaleString() }, { label: "Executed", - value: - actor.numExecutedTasks.toLocaleString() + value: actor.numExecutedTasks.toLocaleString() }, { label: "NumObjectIdsInScope", - value: - actor.numObjectIdsInScope.toLocaleString() + value: actor.numObjectIdsInScope.toLocaleString() }, { label: "NumLocalObjects", - value: - actor.numLocalObjects.toLocaleString() + value: actor.numLocalObjects.toLocaleString() }, { label: "UsedLocalObjectMemory", - value: - actor.usedObjectStoreMemory.toLocaleString() + value: actor.usedObjectStoreMemory.toLocaleString() }, { label: "Task", - value: - actor.currentTaskFuncDesc.join(".") + value: actor.currentTaskFuncDesc.join(".") } ] : [ @@ -125,6 +165,7 @@ class Actor extends React.Component, State> { value: Object.entries(actor.requiredResources).length > 0 && Object.entries(actor.requiredResources) + .sort((a, b) => a[0].localeCompare(b[0])) .map(([key, value]) => `${value.toLocaleString()} ${key}`) .join(", ") } @@ -140,13 +181,53 @@ class Actor extends React.Component, State> { ( {expanded ? "Collapse" : "Expand"} ) + )}{" "} + (Profile for + {[10, 30, 60].map(duration => ( + + {" "} + + {duration}s + + + ))} + ){" "} + {Object.entries(profiling).map( + ([profilingId, { startTime, latestResponse }]) => + latestResponse !== null && ( + + ( + {latestResponse.status === "pending" ? ( + `Profiling for ${Math.round( + (Date.now() - startTime) / 1000 + )}s...` + ) : latestResponse.status === "finished" ? ( + + Profiling result + + ) : latestResponse.status === "error" ? ( + `Profiling error: ${latestResponse.error.trim()}` + ) : ( + undefined + )} + ){" "} + + ) )} ) : ( diff --git a/python/ray/dashboard/client/src/pages/dashboard/logical-view/LogicalView.tsx b/python/ray/dashboard/client/src/pages/dashboard/logical-view/LogicalView.tsx index f528274ca..949d8997f 100644 --- a/python/ray/dashboard/client/src/pages/dashboard/logical-view/LogicalView.tsx +++ b/python/ray/dashboard/client/src/pages/dashboard/logical-view/LogicalView.tsx @@ -2,12 +2,23 @@ import { Theme } from "@material-ui/core/styles/createMuiTheme"; import createStyles from "@material-ui/core/styles/createStyles"; import withStyles, { WithStyles } from "@material-ui/core/styles/withStyles"; import Typography from "@material-ui/core/Typography"; +import WarningRoundedIcon from "@material-ui/icons/WarningRounded"; import React from "react"; import { connect } from "react-redux"; import { StoreState } from "../../../store"; import Actors from "./Actors"; -const styles = (theme: Theme) => createStyles({}); +const styles = (theme: Theme) => + createStyles({ + warning: { + fontSize: "0.8125rem", + marginBottom: theme.spacing(2) + }, + warningIcon: { + fontSize: "1.25em", + verticalAlign: "text-bottom" + } + }); const mapStateToProps = (state: StoreState) => ({ rayletInfo: state.dashboard.rayletInfo @@ -17,19 +28,20 @@ class LogicalView extends React.Component< WithStyles & ReturnType > { render() { - const { rayletInfo } = this.props; - - if (rayletInfo === null) { - return Loading...; - } - - if (Object.entries(rayletInfo.actors).length === 0) { - return No actors found.; - } - + const { classes, rayletInfo } = this.props; return (
- + + Note: This tab + is experimental. + + {rayletInfo === null ? ( + Loading... + ) : Object.entries(rayletInfo.actors).length === 0 ? ( + No actors found. + ) : ( + + )}
); } diff --git a/python/ray/dashboard/dashboard.py b/python/ray/dashboard/dashboard.py index 86c9715aa..2cb86cef5 100644 --- a/python/ray/dashboard/dashboard.py +++ b/python/ray/dashboard/dashboard.py @@ -257,11 +257,11 @@ class Dashboard(object): duration = int(req.query.get("duration")) profiling_id = self.raylet_stats.launch_profiling( node_id=node_id, pid=pid, duration=duration) - return aiohttp.web.json_response(str(profiling_id)) + return await json_response(str(profiling_id)) async def check_profiling_status(req) -> aiohttp.web.Response: profiling_id = req.query.get("profiling_id") - return aiohttp.web.json_response( + return await json_response( self.raylet_stats.check_profiling_status(profiling_id)) async def get_profiling_info(req) -> aiohttp.web.Response: diff --git a/python/ray/tests/test_metrics.py b/python/ray/tests/test_metrics.py index 42368059d..b61c5adc9 100644 --- a/python/ray/tests/test_metrics.py +++ b/python/ray/tests/test_metrics.py @@ -190,21 +190,23 @@ def test_raylet_info_endpoint(shutdown_only): "node_id": ray.nodes()[0]["NodeID"], "pid": actor_pid, "duration": 5 - }).json() + }).json()["result"] start_time = time.time() while True: - time.sleep(1) - try: - profiling_info = requests.get( - webui_url + "/api/check_profiling_status", - params={ - "profiling_id": profiling_id, - }).json() - assert profiling_info["status"] in ("finished", "pending", "error") + # Sometimes some startup time is required + if time.time() - start_time > 30: + raise RayTestTimeoutException( + "Timed out while collecting profiling stats.") + profiling_info = requests.get( + webui_url + "/api/check_profiling_status", + params={ + "profiling_id": profiling_id, + }).json() + status = profiling_info["result"]["status"] + assert status in ("finished", "pending", "error") + if status in ("finished", "error"): break - except AssertionError: - if time.time() - start_time + 10: - raise Exception("Timed out while collecting profiling stats.") + time.sleep(1) def test_profiling_info_endpoint(shutdown_only):