Error-checking for a couple of corruption issues (#8059)

* Extra error handling
* Handle connection closed in Redis monitor
Co-authored-by: Mehrdad <noreply@github.com>
This commit is contained in:
mehrdadn
2020-06-07 06:43:00 -07:00
committed by GitHub
parent a0247ffe55
commit f68183d778
4 changed files with 20 additions and 4 deletions
+6 -1
View File
@@ -13,6 +13,7 @@ import ray.utils
import ray.ray_constants as ray_constants
from ray.utils import binary_to_hex, setup_logger
from ray.autoscaler.commands import teardown_cluster
import redis
logger = logging.getLogger(__name__)
@@ -161,7 +162,11 @@ class Monitor:
subscribe_clients = [self.primary_subscribe_client]
for subscribe_client in subscribe_clients:
for _ in range(max_messages):
message = subscribe_client.get_message()
message = None
try:
message = subscribe_client.get_message()
except redis.exceptions.ConnectionError:
pass
if message is None:
# Continue on to the next subscribe client.
break