Add option to evict keys LRU from the sharded redis tables (#3499)

* wip

* wip

* format

* wip

* note

* lint

* fix

* flag

* typo

* raise timeout

* fix

* optional get

* fix flag

* increase timeout in test

* update docs

* format

test/failure_test.py

@@ -3,6 +3,7 @@ from __future__ import division
 from __future__ import print_function
 
 import numpy as np
+import json
 import os
 import ray
 import sys
@@ -570,7 +571,13 @@ def test_warning_for_infeasible_zero_cpu_actor(shutdown_only):
 @pytest.fixture
 def ray_start_two_nodes():
     # Start the Ray processes.
-    ray.worker._init(start_ray_local=True, num_local_schedulers=2, num_cpus=0)
+    ray.worker._init(
+        start_ray_local=True,
+        num_local_schedulers=2,
+        num_cpus=0,
+        _internal_config=json.dumps({
+            "num_heartbeats_timeout": 40
+        }))
     yield None
     # The code after the yield will run as teardown code.
     ray.shutdown()
@@ -594,7 +601,7 @@ def test_warning_for_dead_node(ray_start_two_nodes):
     ray.services.all_processes[ray.services.PROCESS_TYPE_RAYLET][0].kill()
 
     # Check that we get warning messages for both raylets.
-    wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 2, timeout=20)
+    wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 2, timeout=40)
 
     # Extract the client IDs from the error messages. This will need to be
     # changed if the error message changes.