Add option to evict keys LRU from the sharded redis tables (#3499)

* wip

* wip

* format

* wip

* note

* lint

* fix

* flag

* typo

* raise timeout

* fix

* optional get

* fix flag

* increase timeout in test

* update docs

* format

src/ray/ray_config.h

@@ -200,7 +200,7 @@ class RayConfig {
       : ray_protocol_version_(0x0000000000000000),
         handler_warning_timeout_ms_(100),
         heartbeat_timeout_milliseconds_(100),
-        num_heartbeats_timeout_(100),
+        num_heartbeats_timeout_(300),
         num_heartbeats_warning_(5),
         debug_dump_period_milliseconds_(10000),
         initial_reconstruction_timeout_milliseconds_(10000),

src/ray/raylet/node_manager.cc

@@ -1543,7 +1543,12 @@ void NodeManager::HandleTaskReconstruction(const TaskID &task_id) {
       [this](ray::gcs::AsyncGcsClient *client, const TaskID &task_id) {
         // The task was not in the GCS task table. It must therefore be in the
         // lineage cache.
-        RAY_CHECK(lineage_cache_.ContainsTask(task_id));
+        RAY_CHECK(lineage_cache_.ContainsTask(task_id))
+            << "Task metadata not found in either GCS or lineage cache. It may have been "
+               "evicted "
+            << "by the redis LRU configuration. Consider increasing the memory "
+               "allocation via "
+            << "ray.init(redis_max_memory=<max_memory_bytes>).";
         // Use a copy of the cached task spec to re-execute the task.
         const Task task = lineage_cache_.GetTask(task_id);
         ResubmitTask(task);