From 0b022c097391bccc21cb07a20933a3b85a053a97 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Sun, 28 Jan 2018 12:55:38 -0800 Subject: [PATCH] Automatically increase redis maxclients if the ulimit is high enough. (#1482) --- python/ray/services.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/python/ray/services.py b/python/ray/services.py index b2c2186d9..9078967cb 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -11,6 +11,7 @@ import psutil import pyarrow import random import redis +import resource import shutil import signal import socket @@ -391,6 +392,7 @@ def start_redis(node_ip_address, """ redis_stdout_file, redis_stderr_file = new_log_files( "redis", redirect_output) + assigned_port, _ = start_redis_instance( node_ip_address=node_ip_address, port=port, redis_max_clients=redis_max_clients, @@ -513,6 +515,23 @@ def start_redis_instance(node_ip_address="127.0.0.1", # number of Redis clients. if redis_max_clients is not None: redis_client.config_set("maxclients", str(redis_max_clients)) + else: + # If redis_max_clients is not provided, determine the current ulimit. + # We will use this to attempt to raise the maximum number of Redis + # clients. + current_max_clients = int( + redis_client.config_get("maxclients")["maxclients"]) + # The below command should be the same as doing ulimit -n. + ulimit_n = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + # The quantity redis_client_buffer appears to be the required buffer + # between the maximum number of redis clients and ulimit -n. That is, + # if ulimit -n returns 10000, then we can set maxclients to + # 10000 - redis_client_buffer. + redis_client_buffer = 32 + if current_max_clients < ulimit_n - redis_client_buffer: + redis_client.config_set("maxclients", + ulimit_n - redis_client_buffer) + # Increase the hard and soft limits for the redis client pubsub buffer to # 128MB. This is a hack to make it less likely for pubsub messages to be # dropped and for pubsub connections to therefore be killed.