Better errors on process deaths (#3252)

python/ray/worker.py

@@ -5,6 +5,7 @@ from __future__ import print_function
 from contextlib import contextmanager
 import atexit
 import colorama
+import faulthandler
 import hashlib
 import inspect
 import logging
@@ -404,7 +405,8 @@ class Worker(object):
                 invalid_error = RayTaskError(
                     "<unknown>", None,
                     "Invalid return value: likely worker died or was killed "
-                    "while executing the task.")
+                    "while executing the task; check previous logs or dmesg "
+                    "for errors.")
                 return [invalid_error] * len(object_ids)
             except pyarrow.DeserializationCallbackError:
                 # Wait a little bit for the import thread to import the class.
@@ -1850,6 +1852,9 @@ def connect(info,
     assert not worker.connected, error_message
     assert worker.cached_functions_to_run is not None, error_message
 
+    # Enable nice stack traces on SIGSEGV etc.
+    faulthandler.enable(all_threads=False)
+
     # Initialize some fields.
     if mode is WORKER_MODE:
         worker.worker_id = random_string()

python/setup.py

@@ -151,6 +151,7 @@ setup(
         "pytest",
         "pyyaml",
         "redis",
+        "faulthandler;python_version<'3'",
         "setproctitle",
         # The six module is required by pyarrow.
         "six >= 1.0.0",