Raise application-level exceptions for some failure scenarios. (#2429)

* Raise application level exception for actor methods that can't be executed and failed tasks.

* Retry task forwarding for actor tasks.

* Small cleanups

* Move constant to ray_config.

* Create ForwardTaskOrResubmit method.

* Minor

* Clean up queued tasks for dead actors.

* Some cleanups.

* Linting

* Notify task_dependency_manager_ about failed tasks.

* Manage timer lifetime better.

* Use smart pointers to deallocate the timer.

* Fix

* add comment

src/common/state/ray_config.h

@@ -88,6 +88,10 @@ class RayConfig {
     return actor_creation_num_spillbacks_warning_;
   }
 
+  int node_manager_forward_task_retry_timeout_milliseconds() const {
+    return node_manager_forward_task_retry_timeout_milliseconds_;
+  }
+
   int object_manager_pull_timeout_ms() const {
     return object_manager_pull_timeout_ms_;
   }
@@ -131,6 +135,7 @@ class RayConfig {
         L3_cache_size_bytes_(100000000),
         max_tasks_to_spillback_(10),
         actor_creation_num_spillbacks_warning_(100),
+        node_manager_forward_task_retry_timeout_milliseconds_(1000),
         // TODO: Setting this to large values results in latency, which needs to
         // be addressed. This timeout is often on the critical path for object
         // transfers.
@@ -226,6 +231,10 @@ class RayConfig {
   /// a value of 100 corresponds to a warning every 10 seconds.
   int64_t actor_creation_num_spillbacks_warning_;
 
+  /// If a node manager attempts to forward a task to another node manager and
+  /// the forward fails, then it will resubmit the task after this duration.
+  int64_t node_manager_forward_task_retry_timeout_milliseconds_;
+
   /// Timeout, in milliseconds, to wait before retrying a failed pull in the
   /// ObjectManager.
   int object_manager_pull_timeout_ms_;