From 80f63696ac250aa2b3342ddf26c7e94ade399bdb Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Mon, 5 Nov 2018 18:34:19 -0800 Subject: [PATCH] Cap object store memory to 20GB when size is None (#3243) * Update services.py * Update services.py --- python/ray/services.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/ray/services.py b/python/ray/services.py index 42e65809e..8549879fc 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -36,6 +36,9 @@ PROCESS_TYPE_PLASMA_STORE = "plasma_store" PROCESS_TYPE_REDIS_SERVER = "redis_server" PROCESS_TYPE_WEB_UI = "web_ui" +# Max bytes to allocate to plasma unless overriden by the user +MAX_DEFAULT_MEM = 20 * 1000 * 1000 * 1000 + # This is a dictionary tracking all of the processes of different types that # have been started by this services module. Note that the order of the keys is # important because it determines the order in which these processes will be @@ -1005,6 +1008,14 @@ def determine_plasma_store_config(object_store_memory=None, # Choose a default object store size. if object_store_memory is None: object_store_memory = int(system_memory * 0.4) + # Cap memory to avoid memory waste and perf issues on large nodes + if object_store_memory > MAX_DEFAULT_MEM: + logger.warning( + "Warning: Capping object memory store to {}GB. ".format( + MAX_DEFAULT_MEM // 1e9) + + "To increase this further, specify `object_store_memory` " + "when calling ray.init() or ray start.") + object_store_memory = MAX_DEFAULT_MEM if plasma_directory is not None: plasma_directory = os.path.abspath(plasma_directory)