From 7e4b4822cf7d8db016b38537686ce3898bef015e Mon Sep 17 00:00:00 2001 From: Kristian Hartikainen Date: Fri, 8 Mar 2019 10:59:31 -0800 Subject: [PATCH] [tune] Fix worker recovery by setting `force=False` when calling logger sync_now (#4302) ## What do these changes do? Fixes a tune autoscaling problem where worker recovery causes things to stall. --- python/ray/tune/logger.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/ray/tune/logger.py b/python/ray/tune/logger.py index 7f4bddac8..80da6698c 100644 --- a/python/ray/tune/logger.py +++ b/python/ray/tune/logger.py @@ -226,14 +226,13 @@ class UnifiedLogger(Logger): def close(self): for _logger in self._loggers: _logger.close() - self._log_syncer.sync_now(force=True) + self._log_syncer.sync_now(force=False) self._log_syncer.close() def flush(self): for _logger in self._loggers: _logger.flush() - self._log_syncer.sync_now(force=True) - self._log_syncer.wait() + self._log_syncer.sync_now(force=False) def sync_results_to_new_location(self, worker_ip): """Sends the current log directory to the remote node.