wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-28 13:19:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Dashboard] Fix Issue #10319 - Dashboard autoscaler crash (#10323)

Browse Source 
* Patch error that occurred when there was an entry in the dashboard logs or errors internal data structures, and a worker was removed from the cluster. This would crash the cluster with a KeyError.

* lint

Co-authored-by: Max Fitton <max@semprehealth.com>
This commit is contained in:
Max Fitton
2020-08-29 23:18:23 -07:00
committed by GitHub
parent 8c753818ab
commit 63ad2e3340
1 changed files with 12 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/dashboard/node_stats.py
+12 -4
View File
@@ -60,14 +60,22 @@ class NodeStats(threading.Thread):
def _insert_log_counts(self):
for ip, logs_by_pid in self._logs.items():
hostname = self._ip_to_hostname[ip]
logs_by_pid = {pid: len(logs) for pid, logs in logs_by_pid.items()}
self._node_stats[hostname]["log_count"] = logs_by_pid
if hostname in self._node_stats:
logs_by_pid = {
pid: len(logs)
for pid, logs in logs_by_pid.items()
}
self._node_stats[hostname]["log_count"] = logs_by_pid
def _insert_error_counts(self):
for ip, errs_by_pid in self._errors.items():
hostname = self._ip_to_hostname[ip]
errs_by_pid = {pid: len(errs) for pid, errs in errs_by_pid.items()}
self._node_stats[hostname]["error_count"] = errs_by_pid
if hostname in self._node_stats:
errs_by_pid = {
pid: len(errs)
for pid, errs in errs_by_pid.items()
}
self._node_stats[hostname]["error_count"] = errs_by_pid
def _purge_outdated_stats(self):
def current(then, now):