wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-29 11:01:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Dashboard] Fix null gpu utilization (#11650)

Browse Source 
* update dashboard to work if GPU utilization field is missing from GPU payload

* lint

* lint
This commit is contained in:
Max Fitton
2020-10-29 23:11:50 -07:00
committed by GitHub
parent e022d12dc3
commit d6628cdbfb
4 changed files with 56 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dashboard/client/src/api.ts
+1 -1
View File
@@ -96,7 +96,7 @@ export type GPUStats = {
name: string;
temperatureGpu: number;
fanSpeed: number;
utilizationGpu: number;
utilizationGpu?: number;
powerDraw: number;
enforcedPowerLimit: number;
memoryUsed: number;
dashboard/client/src/pages/dashboard/logical-view/ActorDetailsPane.tsx
+27 -14
View File
@@ -144,20 +144,33 @@ const ActorDetailsPane: React.FC<ActorDetailsPaneProps> = ({ actor }) => {
<Grid item xs={12}>
<Typography>GPU Usage</Typography>
</Grid>
{actor.gpus.map((gpu) => (
<React.Fragment key={gpu.uuid}>
<Grid item xs={4}>
{`[${gpu.name}]`}
</Grid>
<Grid item xs={4}>
<UsageBar
percent={gpu.utilizationGpu * 100}
text={`${gpu.utilizationGpu * 100}%`}
/>
</Grid>
<Grid item xs={4} />
</React.Fragment>
))}
{actor.gpus.map((gpu) => {
const gpuUtilization = gpu.utilizationGpu ? (
<UsageBar
percent={gpu.utilizationGpu * 100}
text={`${gpu.utilizationGpu * 100}%`}
/>
) : (
<Typography
color="textSecondary"
component="span"
variant="inherit"
>
N/A
</Typography>
);
return (
<React.Fragment key={gpu.uuid}>
<Grid item xs={4}>
{`[${gpu.name}]`}
</Grid>
<Grid item xs={4}>
{gpuUtilization}
</Grid>
<Grid item xs={4} />
</React.Fragment>
);
})}
</Grid>
)}
</Grid>
dashboard/client/src/pages/dashboard/node-info/features/GPU.tsx
+20 -7
View File
@@ -32,11 +32,18 @@ const clusterGPUUtilization = (nodes: Array<Node>): number => {
};
const nodeGPUUtilization = (node: Node): number => {
if (!node.gpus || node.gpus.length === 0) {
if (node.gpus === null) {
return NaN;
}
const utilizationSum = sum(node.gpus.map((gpu) => gpu.utilizationGpu));
const avgUtilization = utilizationSum / node.gpus.length;
const gpusWithUtilInfo = node.gpus.filter((gpu) => gpu.utilizationGpu);
if (gpusWithUtilInfo.length === 0) {
return NaN;
}
const utilizationSum = sum(
gpusWithUtilInfo.map((gpu) => gpu.utilizationGpu ?? 0),
);
const avgUtilization = utilizationSum / gpusWithUtilInfo.length;
return avgUtilization;
};
@@ -87,10 +94,16 @@ const NodeGPUEntry: React.FC<NodeGPUEntryProps> = ({ gpu, slot }) => {
<Tooltip title={gpu.name}>
<RightPaddedTypography variant="body1">[{slot}]:</RightPaddedTypography>
</Tooltip>
<UsageBar
percent={gpu.utilizationGpu}
text={`${gpu.utilizationGpu.toFixed(1)}%`}
/>
{gpu.utilizationGpu ? (
<UsageBar
percent={gpu.utilizationGpu}
text={`${gpu.utilizationGpu.toFixed(1)}%`}
/>
) : (
<Typography color="textSecondary" component="span" variant="inherit">
N/A
</Typography>
)}
</Box>
);
};
dashboard/client/src/pages/dashboard/node-info/features/GRAM.tsx
+8 -2
View File
@@ -19,8 +19,14 @@ import {
const GRAM_COL_WIDTH = 120;
const nodeGRAMUtilization = (node: Node) => {
const utilization = (gpu: GPUStats) => gpu.memoryUsed / gpu.memoryTotal;
if (node.gpus.length === 0) {
const utilization = (gpu: GPUStats) => {
if (!gpu.memoryUsed || !gpu.memoryTotal) {
return NaN;
}
return gpu.memoryUsed / gpu.memoryTotal;
};
const gramUtils = node.gpus.map(utilization).filter((util) => !!util);
if (gramUtils.length === 0) {
return NaN;
}
const utilizationSum = sum(node.gpus.map((gpu) => utilization(gpu)));