Add failure tests to test_reference_counting (#7400)

src/ray/common/ray_config_def.h

@@ -260,3 +260,6 @@ RAY_CONFIG(int32_t, object_store_full_max_retries, 5)
 /// Duration to sleep after failing to put an object in plasma because it is full.
 /// This will be exponentially increased for each retry.
 RAY_CONFIG(uint32_t, object_store_full_initial_delay_ms, 1000)
+
+/// Duration to wait between retries for failed tasks.
+RAY_CONFIG(uint32_t, task_retry_delay_ms, 5000)

src/ray/core_worker/core_worker.cc

@@ -202,10 +202,11 @@ CoreWorker::CoreWorker(const WorkerType worker_type, const Language language,
       [this](const TaskSpecification &spec) {
         // Retry after a delay to emulate the existing Raylet reconstruction
         // behaviour. TODO(ekl) backoff exponentially.
-        RAY_LOG(ERROR) << "Will resubmit task after a 5 second delay: "
-                       << spec.DebugString();
+        uint32_t delay = RayConfig::instance().task_retry_delay_ms();
+        RAY_LOG(ERROR) << "Will resubmit task after a " << delay
+                       << "ms delay: " << spec.DebugString();
         absl::MutexLock lock(&mutex_);
-        to_resubmit_.push_back(std::make_pair(current_time_ms() + 5000, spec));
+        to_resubmit_.push_back(std::make_pair(current_time_ms() + delay, spec));
       }));
 
   // Create an entry for the driver task in the task table. This task is