From 920e4b2ef8d084f5a25c205c0bda3cabb13e3384 Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Wed, 21 Oct 2020 14:29:43 -0700 Subject: [PATCH] Try to raise ulimit for file descriptors to max allowed; warn if ulimit is still too low (#11515) --- python/ray/worker.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/python/ray/worker.py b/python/ray/worker.py index 06bf9d66d..0395e61d7 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -622,6 +622,29 @@ def init( arguments is passed in. """ + # Try to increase the file descriptor limit, which is too low by + # default for Ray: https://github.com/ray-project/ray/issues/11239 + try: + import resource + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft < hard: + logger.debug("Automatically increasing RLIMIT_NOFILE to max " + "value of {}".format(hard)) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + except ValueError: + logger.debug("Failed to raise limit.") + soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft < 4096: + logger.warning( + "File descriptor limit {} is too low for production " + "servers and may result in connection errors. " + "At least 8192 is recommended. --- " + "Fix with 'ulimit -n 8192'".format(soft)) + except ImportError: + logger.debug("Could not import resource module (on Windows)") + pass + if "RAY_ADDRESS" in os.environ: if address is None or address == "auto": address = os.environ["RAY_ADDRESS"]