Experimental: enable automatic GCS flushing with configurable policy. (#2266)

* build_credis.sh: use an up-to-date credis commit.

* build_credis.sh: leveldb is updated, so update build cmds for it

* WIP: make monitor.py issue flush; switch gcs client to use credis

* Experimental: enable automatic GCS flushing with configurable policy.

* Fix linux compilation error

* Fix leveldb build

* Use optimized build for credis

* Address comments

* Attempt to fix tests

python/ray/experimental/__init__.py

@@ -7,6 +7,8 @@ from .features import (
     flush_redis_unsafe, flush_task_and_object_metadata_unsafe,
     flush_finished_tasks_unsafe, flush_evicted_objects_unsafe,
     _flush_finished_tasks_unsafe_shard, _flush_evicted_objects_unsafe_shard)
+from .gcs_flush_policy import (set_flushing_policy, GcsFlushPolicy,
+                               SimpleGcsFlushPolicy)
 from .named_actors import get_actor, register_actor
 from .api import get, wait
 
@@ -15,5 +17,6 @@ __all__ = [
     "flush_task_and_object_metadata_unsafe", "flush_finished_tasks_unsafe",
     "flush_evicted_objects_unsafe", "_flush_finished_tasks_unsafe_shard",
     "_flush_evicted_objects_unsafe_shard", "get_actor", "register_actor",
-    "get", "wait"
+    "get", "wait", "set_flushing_policy", "GcsFlushPolicy",
+    "SimpleGcsFlushPolicy"
 ]

python/ray/experimental/features.py

@@ -10,7 +10,7 @@ OBJECT_LOCATION_PREFIX = b"OL:"
 TASK_PREFIX = b"TT:"
 
 
-def flush_redis_unsafe():
+def flush_redis_unsafe(redis_client=None):
     """This removes some non-critical state from the primary Redis shard.
 
     This removes the log files as well as the event log from Redis. This can
@@ -18,10 +18,14 @@ def flush_redis_unsafe():
     of metadata in Redis. However, it will only partially address the issue as
     much of the data is in the task table (and object table), which are not
     flushed.
-    """
-    ray.worker.global_worker.check_connected()
 
-    redis_client = ray.worker.global_worker.redis_client
+    Args:
+      redis_client: optional, if not provided then ray.init() must have been
+        called.
+    """
+    if redis_client is None:
+        ray.worker.global_worker.check_connected()
+        redis_client = ray.worker.global_worker.redis_client
 
     # Delete the log files from the primary Redis shard.
     keys = redis_client.keys("LOGFILE:*")

python/ray/experimental/gcs_flush_policy.py

@@ -0,0 +1,91 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import time
+
+import ray
+import ray.cloudpickle as pickle
+
+
+class GcsFlushPolicy(object):
+    """Experimental: a policy to control GCS flushing.
+
+    Used by Monitor to enable automatic control of memory usage.
+    """
+
+    def should_flush(self, redis_client):
+        """Returns a bool, whether a flush request should be issued."""
+        pass
+
+    def num_entries_to_flush(self):
+        """Returns an upper bound for number of entries to flush next."""
+        pass
+
+    def record_flush(self):
+        """Must be called after a flush has been performed."""
+        pass
+
+
+class SimpleGcsFlushPolicy(GcsFlushPolicy):
+    """A simple policy with constant flush rate, after a warmup period.
+
+    Example policy values:
+        flush_when_at_least_bytes 2GB
+        flush_period_secs 10s
+        flush_num_entries_each_time 10k
+
+    This means
+    (1) If the GCS shard uses less than 2GB of memory, no flushing would take
+        place.  This should cover most Ray runs.
+    (2) The GCS shard will only honor a flush request, if it's issued after 10
+        seconds since the last processed flush.  In particular this means it's
+        okay for the Monitor to issue requests more frequently than this param.
+    (3) When processing a flush, the shard will flush at most 10k entries.
+        This is to control the latency of each request.
+
+    Note, flush rate == (flush period) * (num entries each time).  So
+    applications that have a heavier GCS load can tune these params.
+    """
+
+    def __init__(self,
+                 flush_when_at_least_bytes=(1 << 31),
+                 flush_period_secs=10,
+                 flush_num_entries_each_time=10000):
+        self.flush_when_at_least_bytes = flush_when_at_least_bytes
+        self.flush_period_secs = flush_period_secs
+        self.flush_num_entries_each_time = flush_num_entries_each_time
+        self.last_flush_timestamp = time.time()
+
+    def should_flush(self, redis_client):
+        if time.time() - self.last_flush_timestamp < self.flush_period_secs:
+            return False
+
+        used_memory = redis_client.info("memory")["used_memory"]
+        assert used_memory > 0
+
+        return used_memory >= self.flush_when_at_least_bytes
+
+    def num_entries_to_flush(self):
+        return self.flush_num_entries_each_time
+
+    def record_flush(self):
+        self.last_flush_timestamp = time.time()
+
+    def serialize(self):
+        return pickle.dumps(self)
+
+
+def set_flushing_policy(flushing_policy):
+    """Serialize this policy for Monitor to pick up."""
+    if "RAY_USE_NEW_GCS" not in os.environ:
+        raise Exception(
+            "set_flushing_policy() is only available when environment "
+            "variable RAY_USE_NEW_GCS is present at both compile and run time."
+        )
+    ray.worker.global_worker.check_connected()
+    redis_client = ray.worker.global_worker.redis_client
+
+    serialized = pickle.dumps(flushing_policy)
+    redis_client.set("gcs_flushing_policy", serialized)

python/ray/monitor.py

@@ -10,6 +10,7 @@ import time
 from collections import Counter, defaultdict
 
 import ray
+import ray.cloudpickle as pickle
 import ray.utils
 import redis
 # Import flatbuffer bindings.
@@ -113,6 +114,19 @@ class Monitor(object):
         else:
             self.autoscaler = None
 
+        # Experimental feature: GCS flushing.
+        self.issue_gcs_flushes = "RAY_USE_NEW_GCS" in os.environ
+        self.gcs_flush_policy = None
+        if self.issue_gcs_flushes:
+            # For now, we take the primary redis server to issue flushes,
+            # because task table entries are stored there under this flag.
+            try:
+                self.redis.execute_command("HEAD.FLUSH 0")
+            except redis.exceptions.ResponseError as e:
+                log.info("Turning off flushing due to exception: {}".format(
+                    str(e)))
+                self.issue_gcs_flushes = False
+
     def subscribe(self, channel):
         """Subscribe to the given channel.
 
@@ -534,6 +548,37 @@ class Monitor(object):
                 or local_scheduler_info["NodeManagerAddress"]).split(":")[0]
             self.local_scheduler_id_to_ip_map[client_id] = ip_address
 
+    def _maybe_flush_gcs(self):
+        """Experimental: issue a flush request to the GCS.
+
+        The purpose of this feature is to control GCS memory usage.
+
+        To activate this feature, Ray must be compiled with the flag
+        RAY_USE_NEW_GCS set, and Ray must be started at run time with the flag
+        as well.
+        """
+        if not self.issue_gcs_flushes:
+            return
+        if self.gcs_flush_policy is None:
+            serialized = self.redis.get("gcs_flushing_policy")
+            if serialized is None:
+                # Client has not set any policy; by default flushing is off.
+                return
+            self.gcs_flush_policy = pickle.loads(serialized)
+
+        if not self.gcs_flush_policy.should_flush(self.redis):
+            return
+
+        max_entries_to_flush = self.gcs_flush_policy.num_entries_to_flush()
+        num_flushed = self.redis.execute_command(
+            "HEAD.FLUSH {}".format(max_entries_to_flush))
+        log.info('num_flushed {}'.format(num_flushed))
+
+        # This flushes event log and log files.
+        ray.experimental.flush_redis_unsafe(self.redis)
+
+        self.gcs_flush_policy.record_flush()
+
     def run(self):
         """Run the monitor.
 
@@ -578,12 +623,16 @@ class Monitor(object):
             if self.autoscaler:
                 self.autoscaler.update()
 
+            self._maybe_flush_gcs()
+
             # Record how many dead local schedulers and plasma managers we had
             # at the beginning of this round.
             num_dead_local_schedulers = len(self.dead_local_schedulers)
             num_dead_plasma_managers = len(self.dead_plasma_managers)
+
             # Process a round of messages.
             self.process_messages()
+
             # If any new local schedulers or plasma managers were marked as
             # dead in this round, clean up the associated state.
             if len(self.dead_local_schedulers) > num_dead_local_schedulers:

python/ray/services.py

@@ -470,7 +470,10 @@ def start_redis(node_ip_address,
             # It is important to load the credis module BEFORE the ray module,
             # as the latter contains an extern declaration that the former
             # supplies.
-            modules=[CREDIS_MASTER_MODULE, REDIS_MODULE])
+            # NOTE: once data entries are all put under the redis shard(s)
+            # instead of the primary server when RAY_USE_NEW_GCS is set, we
+            # should load CREDIS_MASTER_MODULE here.
+            modules=[CREDIS_MEMBER_MODULE, REDIS_MODULE])
     if port is not None:
         assert assigned_port == port
     port = assigned_port
@@ -523,7 +526,10 @@ def start_redis(node_ip_address,
                 # It is important to load the credis module BEFORE the ray
                 # module, as the latter contains an extern declaration that the
                 # former supplies.
-                modules=[CREDIS_MEMBER_MODULE, REDIS_MODULE])
+                # NOTE: once data entries are all put under the redis shard(s)
+                # instead of the primary server when RAY_USE_NEW_GCS is set, we
+                # should load CREDIS_MEMBER_MODULE here.
+                modules=[CREDIS_MASTER_MODULE, REDIS_MODULE])
 
         if redis_shard_ports[i] is not None:
             assert redis_shard_port == redis_shard_ports[i]
@@ -533,13 +539,15 @@ def start_redis(node_ip_address,
         primary_redis_client.rpush("RedisShards", shard_address)
 
     if use_credis:
-        # Configure the chain state.
-        primary_redis_client.execute_command("MASTER.ADD", node_ip_address,
-                                             redis_shard_port)
         shard_client = redis.StrictRedis(
             host=node_ip_address, port=redis_shard_port)
-        shard_client.execute_command("MEMBER.CONNECT_TO_MASTER",
-                                     node_ip_address, port)
+        # Configure the chain state.
+        # NOTE: once data entries are all put under the redis shard(s) instead
+        # of the primary server when RAY_USE_NEW_GCS is set, we should swap the
+        # callers here.
+        shard_client.execute_command("MASTER.ADD", node_ip_address, port)
+        primary_redis_client.execute_command("MEMBER.CONNECT_TO_MASTER",
+                                             node_ip_address, redis_shard_port)
 
     return redis_address, redis_shards
 
@@ -615,8 +623,8 @@ def _start_redis_instance(node_ip_address="127.0.0.1",
         port = new_port()
         counter += 1
     if counter == num_retries:
-        raise Exception(
-            "Couldn't start Redis. Check stdout file {}".format(stdout_file))
+        raise Exception("Couldn't start Redis. Check log files: {} {}".format(
+            stdout_file.name, stderr_file.name))
 
     # Create a Redis client just for configuring Redis.
     redis_client = redis.StrictRedis(host="127.0.0.1", port=port)