diff --git a/python/ray/tune/utils/util.py b/python/ray/tune/utils/util.py index 934487cfd..7e6e8125e 100644 --- a/python/ray/tune/utils/util.py +++ b/python/ray/tune/utils/util.py @@ -58,7 +58,12 @@ class UtilMonitor(Thread): self.values["ram_util_percent"].append( float(getattr(psutil.virtual_memory(), "percent"))) if GPUtil is not None: - for gpu in GPUtil.getGPUs(): + gpu_list = [] + try: + gpu_list = GPUtil.getGPUs() + except Exception: + logger.debug("GPUtil failed to retrieve GPUs.") + for gpu in gpu_list: self.values["gpu_util_percent" + str(gpu.id)].append( float(gpu.load)) self.values["vram_util_percent" + str(gpu.id)].append(