[core] Add flag to enable object reconstruction during ray start (#9488)

* Add flag

* doc

* Fix tests

python/ray/tests/test_reconstruction.py

@@ -52,16 +52,22 @@ def test_cached_object(ray_start_cluster):
 @pytest.mark.parametrize("reconstruction_enabled", [False, True])
 def test_reconstruction_cached_dependency(ray_start_cluster,
                                           reconstruction_enabled):
-    config = json.dumps({
+    config = {
         "num_heartbeats_timeout": 10,
         "raylet_heartbeat_timeout_milliseconds": 100,
-        "lineage_pinning_enabled": 1 if reconstruction_enabled else 0,
-        "free_objects_period_milliseconds": -1,
         "initial_reconstruction_timeout_milliseconds": 200,
-    })
+    }
+    # Workaround to reset the config to the default value.
+    if not reconstruction_enabled:
+        config["lineage_pinning_enabled"] = 0
+    config = json.dumps(config)
+
     cluster = ray_start_cluster
     # Head node with no resources.
-    cluster.add_node(num_cpus=0, _internal_config=config)
+    cluster.add_node(
+        num_cpus=0,
+        _internal_config=config,
+        enable_object_reconstruction=reconstruction_enabled)
     ray.init(address=cluster.address)
     # Node to place the initial object.
     node_to_kill = cluster.add_node(
@@ -106,16 +112,22 @@ def test_reconstruction_cached_dependency(ray_start_cluster,
 
 @pytest.mark.parametrize("reconstruction_enabled", [False, True])
 def test_basic_reconstruction(ray_start_cluster, reconstruction_enabled):
-    config = json.dumps({
+    config = {
         "num_heartbeats_timeout": 10,
         "raylet_heartbeat_timeout_milliseconds": 100,
-        "lineage_pinning_enabled": 1 if reconstruction_enabled else 0,
-        "free_objects_period_milliseconds": -1,
         "initial_reconstruction_timeout_milliseconds": 200,
-    })
+    }
+    # Workaround to reset the config to the default value.
+    if not reconstruction_enabled:
+        config["lineage_pinning_enabled"] = 0
+    config = json.dumps(config)
+
     cluster = ray_start_cluster
     # Head node with no resources.
-    cluster.add_node(num_cpus=0, _internal_config=config)
+    cluster.add_node(
+        num_cpus=0,
+        _internal_config=config,
+        enable_object_reconstruction=reconstruction_enabled)
     ray.init(address=cluster.address)
     # Node to place the initial object.
     node_to_kill = cluster.add_node(
@@ -150,16 +162,22 @@ def test_basic_reconstruction(ray_start_cluster, reconstruction_enabled):
 
 @pytest.mark.parametrize("reconstruction_enabled", [False, True])
 def test_basic_reconstruction_put(ray_start_cluster, reconstruction_enabled):
-    config = json.dumps({
+    config = {
         "num_heartbeats_timeout": 10,
         "raylet_heartbeat_timeout_milliseconds": 100,
-        "lineage_pinning_enabled": 1 if reconstruction_enabled else 0,
-        "free_objects_period_milliseconds": -1,
         "initial_reconstruction_timeout_milliseconds": 200,
-    })
+    }
+    # Workaround to reset the config to the default value.
+    if not reconstruction_enabled:
+        config["lineage_pinning_enabled"] = 0
+    config = json.dumps(config)
+
     cluster = ray_start_cluster
     # Head node with no resources.
-    cluster.add_node(num_cpus=0, _internal_config=config)
+    cluster.add_node(
+        num_cpus=0,
+        _internal_config=config,
+        enable_object_reconstruction=reconstruction_enabled)
     ray.init(address=cluster.address)
     # Node to place the initial object.
     node_to_kill = cluster.add_node(
@@ -191,22 +209,32 @@ def test_basic_reconstruction_put(ray_start_cluster, reconstruction_enabled):
     if reconstruction_enabled:
         ray.get(result)
     else:
-        with pytest.raises(ray.exceptions.UnreconstructableError):
+        # The copy that we fetched earlier may still be local or it may have
+        # been evicted.
+        try:
             ray.get(result)
+        except ray.exceptions.UnreconstructableError:
+            pass
 
 
 @pytest.mark.parametrize("reconstruction_enabled", [False, True])
 def test_multiple_downstream_tasks(ray_start_cluster, reconstruction_enabled):
-    config = json.dumps({
+    config = {
         "num_heartbeats_timeout": 10,
         "raylet_heartbeat_timeout_milliseconds": 100,
-        "lineage_pinning_enabled": 1 if reconstruction_enabled else 0,
-        "free_objects_period_milliseconds": -1,
         "initial_reconstruction_timeout_milliseconds": 200,
-    })
+    }
+    # Workaround to reset the config to the default value.
+    if not reconstruction_enabled:
+        config["lineage_pinning_enabled"] = 0
+    config = json.dumps(config)
+
     cluster = ray_start_cluster
     # Head node with no resources.
-    cluster.add_node(num_cpus=0, _internal_config=config)
+    cluster.add_node(
+        num_cpus=0,
+        _internal_config=config,
+        enable_object_reconstruction=reconstruction_enabled)
     ray.init(address=cluster.address)
     # Node to place the initial object.
     node_to_kill = cluster.add_node(
@@ -252,17 +280,23 @@ def test_multiple_downstream_tasks(ray_start_cluster, reconstruction_enabled):
 
 @pytest.mark.parametrize("reconstruction_enabled", [False, True])
 def test_reconstruction_chain(ray_start_cluster, reconstruction_enabled):
-    config = json.dumps({
+    config = {
         "num_heartbeats_timeout": 10,
         "raylet_heartbeat_timeout_milliseconds": 100,
-        "lineage_pinning_enabled": 1 if reconstruction_enabled else 0,
-        "free_objects_period_milliseconds": -1,
         "initial_reconstruction_timeout_milliseconds": 200,
-    })
+    }
+    # Workaround to reset the config to the default value.
+    if not reconstruction_enabled:
+        config["lineage_pinning_enabled"] = 0
+    config = json.dumps(config)
+
     cluster = ray_start_cluster
     # Head node with no resources.
     cluster.add_node(
-        num_cpus=0, _internal_config=config, object_store_memory=10**8)
+        num_cpus=0,
+        _internal_config=config,
+        object_store_memory=10**8,
+        enable_object_reconstruction=reconstruction_enabled)
     ray.init(address=cluster.address)
     node_to_kill = cluster.add_node(num_cpus=1, object_store_memory=10**8)
     cluster.wait_for_nodes()
@@ -300,15 +334,14 @@ def test_reconstruction_stress(ray_start_cluster):
     config = json.dumps({
         "num_heartbeats_timeout": 10,
         "raylet_heartbeat_timeout_milliseconds": 100,
-        "lineage_pinning_enabled": 1,
-        "free_objects_period_milliseconds": -1,
         "max_direct_call_object_size": 100,
         "task_retry_delay_ms": 100,
         "initial_reconstruction_timeout_milliseconds": 200,
     })
     cluster = ray_start_cluster
     # Head node with no resources.
-    cluster.add_node(num_cpus=0, _internal_config=config)
+    cluster.add_node(
+        num_cpus=0, _internal_config=config, enable_object_reconstruction=True)
     ray.init(address=cluster.address)
     # Node to place the initial object.
     node_to_kill = cluster.add_node(