[Dashboard] CPU/GPU usage details in actor pane (#11269)

2026-08-04 13:14:14 +08:00 · 2020-10-13 20:23:23 -05:00
parent 933cf6675c
commit cd9dcfca0d
6 changed files with 175 additions and 109 deletions
@@ -39,10 +39,8 @@ export type RayConfigResponse = {

 export const getRayConfig = () => get<RayConfigResponse>("/api/ray_config", {});

-export type Worker = {
+type ProcessStats = {
  pid: number;
-  workerId: string;
-  createTime: number;
  memoryInfo: {
    rss: number;
    vms: number;
@@ -52,6 +50,7 @@ export type Worker = {
    data: number;
    dirty: Number;
  };
+  createTime: number;
  cmdline: string[];
  cpuTimes: {
    user: number;
@@ -61,12 +60,17 @@ export type Worker = {
    iowait: number;
  };
  cpuPercent: number;
+}
+
+export type Worker = {
+  pid: number;
+  workerId: string;
  logCount: number;
  errorCount: number;
  language: string;
  jobId: string;
  coreWorkerStats: CoreWorkerStats[];
-};
+} & ProcessStats;

 export type CoreWorkerStats = {
  ipAddress: string;
@@ -220,12 +224,14 @@ export type FullActorInfo = {
    | ActorState.DependenciesUnready
    | ActorState.PendingCreation;
  taskQueueLength?: number;
+  gpus: GPUStats[]; // Contains info about any GPUs the actor is using
  timestamp: number;
  usedObjectStoreMemory?: number;
  usedResources: { [key: string]: ResourceAllocations };
  currentTaskDesc?: string;
  numPendingTasks?: number;
  webuiDisplay?: Record<string, string>;
+  processStats?: ProcessStats;
 };

 export type ActorTaskInfo = {
@@ -10,12 +10,8 @@ import {
  launchKillActor,
  launchProfiling,
 } from "../../../api";
-import { sum } from "../../../common/util";
 import ActorDetailsPane from "./ActorDetailsPane";

-const memoryDebuggingDocLink =
-  "https://docs.ray.io/en/latest/memory-management.html#debugging-using-ray-memory";
-
 const useActorStyles = makeStyles((theme: Theme) =>
  createStyles({
    root: {
@@ -103,80 +99,6 @@ const Actor: React.FC<ActorProps> = ({ actor }) => {
    }
  };

-  const information = isFullActorInfo(actor)
-    ? [
-        {
-          label: "Resources",
-          value:
-            actor.usedResources &&
-            Object.entries(actor.usedResources).length > 0 &&
-            Object.entries(actor.usedResources)
-              .sort((a, b) => a[0].localeCompare(b[0]))
-              .map(
-                ([key, value]) =>
-                  `${sum(
-                    value.resourceSlots.map((slot) => slot.allocation),
-                  )} ${key}`,
-              )
-              .join(", "),
-        },
-        {
-          label: "Number of pending tasks",
-          value: actor.taskQueueLength?.toLocaleString() ?? "0",
-          tooltip:
-            "The number of tasks that are currently pending to execute on this actor. If this number " +
-            "remains consistently high, it may indicate that this actor is a bottleneck in your application.",
-        },
-        {
-          label: "Number of executed tasks",
-          value: actor.numExecutedTasks?.toLocaleString() ?? "0",
-          tooltip:
-            "The number of tasks this actor has executed throughout its lifetimes.",
-        },
-        {
-          label: "Number of ObjectRefs in scope",
-          value: actor.numObjectRefsInScope?.toLocaleString() ?? "0",
-          tooltip:
-            "The number of ObjectRefs that this actor is keeping in scope via its internal state. " +
-            "This does not imply that the objects are in active use or colocated on the node with the actor " +
-            `currently. This can be useful for debugging memory leaks. See the docs at ${memoryDebuggingDocLink} ` +
-            "for more information.",
-        },
-        {
-          label: "Number of local objects",
-          value: actor.numLocalObjects?.toLocaleString() ?? "0",
-          tooltip:
-            "The number of small objects that this actor has stored in its local in-process memory store. This can be useful for " +
-            `debugging memory leaks. See the docs at ${memoryDebuggingDocLink} for more information`,
-        },
-        {
-          label: "Object store memory used (MiB)",
-          value: actor.usedObjectStoreMemory?.toLocaleString() ?? "0",
-          tooltip:
-            "The total amount of memory that this actor is occupying in the Ray object store. " +
-            "If this number is increasing without bounds, you might have a memory leak. See " +
-            `the docs at: ${memoryDebuggingDocLink} for more information.`,
-        },
-      ]
-    : [
-        {
-          label: "Actor ID",
-          value: actor.actorId,
-          tooltip: "",
-        },
-        {
-          label: "Required resources",
-          value:
-            actor.requiredResources &&
-            Object.entries(actor.requiredResources).length > 0 &&
-            Object.entries(actor.requiredResources)
-              .sort((a, b) => a[0].localeCompare(b[0]))
-              .map(([key, value]) => `${value.toLocaleString()} ${key}`)
-              .join(", "),
-          tooltip: "",
-        },
-      ];
-
  // Construct the custom message from the actor.
  let actorCustomDisplay: JSX.Element[] = [];
  if (isFullActorInfo(actor) && actor.webuiDisplay) {
@@ -274,9 +196,7 @@ const Actor: React.FC<ActorProps> = ({ actor }) => {
        )}
      </Typography>
      <ActorDetailsPane
-        actorDetails={information}
-        actorClass={actor.actorClass}
-        actorState={actor.state}
+        actor={actor}
      />
      {isFullActorInfo(actor) && (
        <React.Fragment>
@@ -1,17 +1,98 @@
-import { Divider, Grid, makeStyles, Theme } from "@material-ui/core";
+import { Divider, Grid, makeStyles, Theme, Typography } from "@material-ui/core";
 import React from "react";
-import { ActorState } from "../../../api";
+import { ActorInfo, isFullActorInfo } from "../../../api";
+import { sum } from "../../../common/util";
 import LabeledDatum from "../../../common/LabeledDatum";
 import ActorStateRepr from "./ActorStateRepr";
+import UsageBar from '../../../common/UsageBar';
+
+const memoryDebuggingDocLink =
+  "https://docs.ray.io/en/latest/memory-management.html#debugging-using-ray-memory";
+
+type ActorDatum = {
+  label: string;
+  value: any;
+  tooltip?: string;
+}
+
+const labeledActorData = (actor: ActorInfo) => (
+  isFullActorInfo(actor)
+    ? [
+      {
+        label: "Resources",
+        value:
+          actor.usedResources &&
+          Object.entries(actor.usedResources).length > 0 &&
+          Object.entries(actor.usedResources)
+            .sort((a, b) => a[0].localeCompare(b[0]))
+            .map(
+              ([key, value]) =>
+                `${sum(
+                  value.resourceSlots.map((slot) => slot.allocation),
+                )} ${key}`,
+            )
+            .join(", "),
+      },
+      {
+        label: "Number of pending tasks",
+        value: actor.taskQueueLength?.toLocaleString() ?? "0",
+        tooltip:
+          "The number of tasks that are currently pending to execute on this actor. If this number " +
+          "remains consistently high, it may indicate that this actor is a bottleneck in your application.",
+      },
+      {
+        label: "Number of executed tasks",
+        value: actor.numExecutedTasks?.toLocaleString() ?? "0",
+        tooltip:
+          "The number of tasks this actor has executed throughout its lifetimes.",
+      },
+      {
+        label: "Number of ObjectRefs in scope",
+        value: actor.numObjectRefsInScope?.toLocaleString() ?? "0",
+        tooltip:
+          "The number of ObjectRefs that this actor is keeping in scope via its internal state. " +
+          "This does not imply that the objects are in active use or colocated on the node with the actor " +
+          `currently. This can be useful for debugging memory leaks. See the docs at ${memoryDebuggingDocLink} ` +
+          "for more information.",
+      },
+      {
+        label: "Number of local objects",
+        value: actor.numLocalObjects?.toLocaleString() ?? "0",
+        tooltip:
+          "The number of small objects that this actor has stored in its local in-process memory store. This can be useful for " +
+          `debugging memory leaks. See the docs at ${memoryDebuggingDocLink} for more information`,
+      },
+      {
+        label: "Object store memory used (MiB)",
+        value: actor.usedObjectStoreMemory?.toLocaleString() ?? "0",
+        tooltip:
+          "The total amount of memory that this actor is occupying in the Ray object store. " +
+          "If this number is increasing without bounds, you might have a memory leak. See " +
+          `the docs at: ${memoryDebuggingDocLink} for more information.`,
+      },
+    ]
+    : [
+      {
+        label: "Actor ID",
+        value: actor.actorId,
+        tooltip: "",
+      },
+      {
+        label: "Required resources",
+        value:
+          actor.requiredResources &&
+          Object.entries(actor.requiredResources).length > 0 &&
+          Object.entries(actor.requiredResources)
+            .sort((a, b) => a[0].localeCompare(b[0]))
+            .map(([key, value]) => `${value.toLocaleString()} ${key}`)
+            .join(", "),
+        tooltip: "",
+      },
+    ]);
+

 type ActorDetailsPaneProps = {
-  actorClass: string;
-  actorState: ActorState;
-  actorDetails: {
-    label: string;
-    value: any;
-    tooltip?: string;
-  }[];
+  actor: ActorInfo;
 };

 const useStyles = makeStyles((theme: Theme) => ({
@@ -31,20 +112,55 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));

 const ActorDetailsPane: React.FC<ActorDetailsPaneProps> = ({
-  actorDetails,
-  actorClass,
-  actorState,
+  actor
 }) => {
  const classes = useStyles();
+  const actorData: ActorDatum[] = labeledActorData(actor);
  return (
    <React.Fragment>
      <div className={classes.actorTitleWrapper}>
-        <div>{actorClass}</div>
-        <ActorStateRepr state={actorState} />
+        <div>{actor.actorClass}</div>
+        <ActorStateRepr state={actor.state} />
      </div>
+      {isFullActorInfo(actor) &&
+        <Grid container className={classes.detailsPane}>
+          <Grid container item xs={6}>
+            <Grid item xs={4}>
+              <Typography>CPU Usage</Typography>
+            </Grid>
+            <Grid item xs={4}>
+              <UsageBar
+                percent={actor.processStats?.cpuPercent ?? 0}
+                text={`${actor.processStats?.cpuPercent ?? 0}%`}
+              />
+            </Grid>
+            <Grid item xs={4} />
+          </Grid>
+        { actor.gpus.length > 0 &&
+          <Grid container item xs={6}>
+            <Grid item xs={12}>
+              <Typography>GPU Usage</Typography>
+            </Grid>
+            {actor.gpus.map(gpu => (
+              <React.Fragment key={gpu.uuid}>
+                <Grid item xs={4}>
+                  {`[${gpu.name}]`}
+                </Grid>
+                <Grid item xs={4}>
+                  <UsageBar
+                    percent={gpu.utilizationGpu * 100}
+                    text={`${gpu.utilizationGpu * 100}%`}
+                  />
+                </Grid>
+                <Grid item xs={4} />
+              </React.Fragment>
+            ))}
+          </Grid>
+        }
+        </Grid>}
      <Divider className={classes.divider} />
      <Grid container className={classes.detailsPane}>
-        {actorDetails.map(
+        {actorData.map(
          ({ label, value, tooltip }) =>
            value &&
            value.length > 0 && (