[grpc]'ray memory' fails if there are many objects in scope #8502 (#12673)

python/ray/includes/ray_config.pxd

@@ -68,3 +68,5 @@ cdef extern from "ray/common/ray_config.h" nogil:
         c_bool enable_timeline() const
 
         c_bool automatic_object_deletion_enabled() const
+
+        uint32_t max_grpc_message_size() const

python/ray/includes/ray_config.pxi

@@ -119,3 +119,7 @@ cdef class Config:
     @staticmethod
     def automatic_object_deletion_enabled():
         return RayConfig.instance().automatic_object_deletion_enabled()
+
+    @staticmethod
+    def max_grpc_message_size():
+        return RayConfig.instance().max_grpc_message_size()

python/ray/internal/internal_api.py

@@ -1,7 +1,9 @@
+import ray
 import ray.worker
 from ray import profiling
 
 __all__ = ["free", "global_gc"]
+MAX_MESSAGE_LENGTH = ray._config.max_grpc_message_size()
 
 
 def global_gc():
@@ -22,7 +24,13 @@ def memory_summary():
     raylet = ray.nodes()[0]
     raylet_address = "{}:{}".format(raylet["NodeManagerAddress"],
                                     ray.nodes()[0]["NodeManagerPort"])
-    channel = grpc.insecure_channel(raylet_address)
+    channel = grpc.insecure_channel(
+        raylet_address,
+        options=[
+            ("grpc.max_send_message_length", MAX_MESSAGE_LENGTH),
+            ("grpc.max_receive_message_length", MAX_MESSAGE_LENGTH),
+        ],
+    )
     stub = node_manager_pb2_grpc.NodeManagerServiceStub(channel)
     reply = stub.FormatGlobalMemoryInfo(
         node_manager_pb2.FormatGlobalMemoryInfoRequest(), timeout=30.0)