From 2a9079aef99fcefe8eeddadd98da583970d39fa1 Mon Sep 17 00:00:00 2001 From: Keqiu Hu Date: Tue, 8 Dec 2020 09:36:53 -0800 Subject: [PATCH] [grpc]'ray memory' fails if there are many objects in scope #8502 (#12673) --- python/ray/includes/ray_config.pxd | 2 ++ python/ray/includes/ray_config.pxi | 4 ++++ python/ray/internal/internal_api.py | 10 +++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/python/ray/includes/ray_config.pxd b/python/ray/includes/ray_config.pxd index 1a1122136..f13757816 100644 --- a/python/ray/includes/ray_config.pxd +++ b/python/ray/includes/ray_config.pxd @@ -68,3 +68,5 @@ cdef extern from "ray/common/ray_config.h" nogil: c_bool enable_timeline() const c_bool automatic_object_deletion_enabled() const + + uint32_t max_grpc_message_size() const diff --git a/python/ray/includes/ray_config.pxi b/python/ray/includes/ray_config.pxi index 7db833cbc..09b0ad497 100644 --- a/python/ray/includes/ray_config.pxi +++ b/python/ray/includes/ray_config.pxi @@ -119,3 +119,7 @@ cdef class Config: @staticmethod def automatic_object_deletion_enabled(): return RayConfig.instance().automatic_object_deletion_enabled() + + @staticmethod + def max_grpc_message_size(): + return RayConfig.instance().max_grpc_message_size() diff --git a/python/ray/internal/internal_api.py b/python/ray/internal/internal_api.py index 04c260ffa..d3e25c1ec 100644 --- a/python/ray/internal/internal_api.py +++ b/python/ray/internal/internal_api.py @@ -1,7 +1,9 @@ +import ray import ray.worker from ray import profiling __all__ = ["free", "global_gc"] +MAX_MESSAGE_LENGTH = ray._config.max_grpc_message_size() def global_gc(): @@ -22,7 +24,13 @@ def memory_summary(): raylet = ray.nodes()[0] raylet_address = "{}:{}".format(raylet["NodeManagerAddress"], ray.nodes()[0]["NodeManagerPort"]) - channel = grpc.insecure_channel(raylet_address) + channel = grpc.insecure_channel( + raylet_address, + options=[ + ("grpc.max_send_message_length", MAX_MESSAGE_LENGTH), + ("grpc.max_receive_message_length", MAX_MESSAGE_LENGTH), + ], + ) stub = node_manager_pb2_grpc.NodeManagerServiceStub(channel) reply = stub.FormatGlobalMemoryInfo( node_manager_pb2.FormatGlobalMemoryInfoRequest(), timeout=30.0)