From d6628cdbfb5ade209b060a79bb83fa460d9c8e7b Mon Sep 17 00:00:00 2001
From: Max Fitton <maxfitton@anyscale.com>
Date: Thu, 29 Oct 2020 23:11:50 -0700
Subject: [PATCH] [Dashboard] Fix null gpu utilization (#11650)

* update dashboard to work if GPU utilization field is missing from GPU payload

* lint

* lint
---
 dashboard/client/src/api.ts                   |  2 +-
 .../logical-view/ActorDetailsPane.tsx         | 41 ++++++++++++-------
 .../dashboard/node-info/features/GPU.tsx      | 27 ++++++++----
 .../dashboard/node-info/features/GRAM.tsx     | 10 ++++-
 4 files changed, 56 insertions(+), 24 deletions(-)

diff --git a/dashboard/client/src/api.ts b/dashboard/client/src/api.ts
index be0798118..d8a979c58 100644
--- a/dashboard/client/src/api.ts
+++ b/dashboard/client/src/api.ts
@@ -96,7 +96,7 @@ export type GPUStats = {
   name: string;
   temperatureGpu: number;
   fanSpeed: number;
-  utilizationGpu: number;
+  utilizationGpu?: number;
   powerDraw: number;
   enforcedPowerLimit: number;
   memoryUsed: number;
diff --git a/dashboard/client/src/pages/dashboard/logical-view/ActorDetailsPane.tsx b/dashboard/client/src/pages/dashboard/logical-view/ActorDetailsPane.tsx
index f03901ada..83f30d1ff 100644
--- a/dashboard/client/src/pages/dashboard/logical-view/ActorDetailsPane.tsx
+++ b/dashboard/client/src/pages/dashboard/logical-view/ActorDetailsPane.tsx
@@ -144,20 +144,33 @@ const ActorDetailsPane: React.FC<ActorDetailsPaneProps> = ({ actor }) => {
               <Grid item xs={12}>
                 <Typography>GPU Usage</Typography>
               </Grid>
-              {actor.gpus.map((gpu) => (
-                <React.Fragment key={gpu.uuid}>
-                  <Grid item xs={4}>
-                    {`[${gpu.name}]`}
-                  </Grid>
-                  <Grid item xs={4}>
-                    <UsageBar
-                      percent={gpu.utilizationGpu * 100}
-                      text={`${gpu.utilizationGpu * 100}%`}
-                    />
-                  </Grid>
-                  <Grid item xs={4} />
-                </React.Fragment>
-              ))}
+              {actor.gpus.map((gpu) => {
+                const gpuUtilization = gpu.utilizationGpu ? (
+                  <UsageBar
+                    percent={gpu.utilizationGpu * 100}
+                    text={`${gpu.utilizationGpu * 100}%`}
+                  />
+                ) : (
+                  <Typography
+                    color="textSecondary"
+                    component="span"
+                    variant="inherit"
+                  >
+                    N/A
+                  </Typography>
+                );
+                return (
+                  <React.Fragment key={gpu.uuid}>
+                    <Grid item xs={4}>
+                      {`[${gpu.name}]`}
+                    </Grid>
+                    <Grid item xs={4}>
+                      {gpuUtilization}
+                    </Grid>
+                    <Grid item xs={4} />
+                  </React.Fragment>
+                );
+              })}
             </Grid>
           )}
         </Grid>
diff --git a/dashboard/client/src/pages/dashboard/node-info/features/GPU.tsx b/dashboard/client/src/pages/dashboard/node-info/features/GPU.tsx
index f00b43149..c76941f70 100644
--- a/dashboard/client/src/pages/dashboard/node-info/features/GPU.tsx
+++ b/dashboard/client/src/pages/dashboard/node-info/features/GPU.tsx
@@ -32,11 +32,18 @@ const clusterGPUUtilization = (nodes: Array<Node>): number => {
 };
 
 const nodeGPUUtilization = (node: Node): number => {
-  if (!node.gpus || node.gpus.length === 0) {
+  if (node.gpus === null) {
     return NaN;
   }
-  const utilizationSum = sum(node.gpus.map((gpu) => gpu.utilizationGpu));
-  const avgUtilization = utilizationSum / node.gpus.length;
+  const gpusWithUtilInfo = node.gpus.filter((gpu) => gpu.utilizationGpu);
+  if (gpusWithUtilInfo.length === 0) {
+    return NaN;
+  }
+
+  const utilizationSum = sum(
+    gpusWithUtilInfo.map((gpu) => gpu.utilizationGpu ?? 0),
+  );
+  const avgUtilization = utilizationSum / gpusWithUtilInfo.length;
   return avgUtilization;
 };
 
@@ -87,10 +94,16 @@ const NodeGPUEntry: React.FC<NodeGPUEntryProps> = ({ gpu, slot }) => {
       <Tooltip title={gpu.name}>
         <RightPaddedTypography variant="body1">[{slot}]:</RightPaddedTypography>
       </Tooltip>
-      <UsageBar
-        percent={gpu.utilizationGpu}
-        text={`${gpu.utilizationGpu.toFixed(1)}%`}
-      />
+      {gpu.utilizationGpu ? (
+        <UsageBar
+          percent={gpu.utilizationGpu}
+          text={`${gpu.utilizationGpu.toFixed(1)}%`}
+        />
+      ) : (
+        <Typography color="textSecondary" component="span" variant="inherit">
+          N/A
+        </Typography>
+      )}
     </Box>
   );
 };
diff --git a/dashboard/client/src/pages/dashboard/node-info/features/GRAM.tsx b/dashboard/client/src/pages/dashboard/node-info/features/GRAM.tsx
index 503e623a8..760b5ca7c 100644
--- a/dashboard/client/src/pages/dashboard/node-info/features/GRAM.tsx
+++ b/dashboard/client/src/pages/dashboard/node-info/features/GRAM.tsx
@@ -19,8 +19,14 @@ import {
 const GRAM_COL_WIDTH = 120;
 
 const nodeGRAMUtilization = (node: Node) => {
-  const utilization = (gpu: GPUStats) => gpu.memoryUsed / gpu.memoryTotal;
-  if (node.gpus.length === 0) {
+  const utilization = (gpu: GPUStats) => {
+    if (!gpu.memoryUsed || !gpu.memoryTotal) {
+      return NaN;
+    }
+    return gpu.memoryUsed / gpu.memoryTotal;
+  };
+  const gramUtils = node.gpus.map(utilization).filter((util) => !!util);
+  if (gramUtils.length === 0) {
     return NaN;
   }
   const utilizationSum = sum(node.gpus.map((gpu) => utilization(gpu)));