Give error if a worker has a version mismatch for Python Ray, or clou… (#1245)

* Give error if a worker has a version mismatch for Python Ray, or cloudpickle.

* Check version when attaching driver to cluster.

* Only do check if the version info is present.

* Bug fix.

* Fix typo.

test/failure_test.py

@@ -259,6 +259,21 @@ class ActorTest(unittest.TestCase):
 
 class WorkerDeath(unittest.TestCase):
 
+    def testWorkerRaisingException(self):
+        ray.init(num_workers=1, driver_mode=ray.SILENT_MODE)
+
+        @ray.remote
+        def f():
+            ray.worker.global_worker._get_next_task_from_local_scheduler = None
+
+        # Running this task should cause the worker to raise an exception after
+        # the task has successfully completed.
+        f.remote()
+
+        wait_for_errors(b"worker_crash", 1)
+        wait_for_errors(b"worker_died", 1)
+        self.assertEqual(len(ray.error_info()), 2)
+
     def testWorkerDying(self):
         ray.init(num_workers=0, driver_mode=ray.SILENT_MODE)
 
@@ -434,5 +449,20 @@ class PutErrorTest(unittest.TestCase):
         ray.worker.cleanup()
 
 
+class ConfigurationTest(unittest.TestCase):
+
+    def testVersionMismatch(self):
+        import cloudpickle
+        cloudpickle_version = cloudpickle.__version__
+        cloudpickle.__version__ = "fake cloudpickle version"
+
+        ray.init(num_workers=1, driver_mode=ray.SILENT_MODE)
+
+        wait_for_errors(b"version_mismatch", 1)
+
+        cloudpickle.__version__ = cloudpickle_version
+        ray.worker.cleanup()
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)