Bring cloudpickle inside the repository. (#1445)

* Bring cloudpickle version 0.5.2 inside the repo.

* Use internal copy of cloudpickle everywhere.

* Fix linting.

* Import ordering.

* Change __init__.py.

* Set pickler in serialization context.

* Don't check ray location.

python/ray/actor.py

@@ -2,7 +2,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import cloudpickle as pickle
 import copy
 import hashlib
 import inspect
@@ -11,6 +10,7 @@ import numpy as np
 import traceback
 
 import pyarrow.plasma as plasma
+import ray.cloudpickle as pickle
 import ray.local_scheduler
 import ray.signature as signature
 import ray.worker

python/ray/autoscaler/aws/development-example.yaml

@@ -88,7 +88,7 @@ setup_commands:
     - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
     # Build Ray.
     - git clone https://github.com/ray-project/ray || true
-    - pip install -U cloudpickle boto3==1.4.8
+    - pip install boto3==1.4.8
     - cd ray/python; python setup.py develop
 
 # Custom commands that will be run on the head node after common setup.

python/ray/cloudpickle/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import absolute_import
+
+from ray.cloudpickle.cloudpickle import *
+
+__version__ = '0.5.2'

python/ray/services.py

@@ -5,7 +5,6 @@ from __future__ import print_function
 import binascii
 from collections import namedtuple, OrderedDict
 from datetime import datetime
-import cloudpickle
 import json
 import os
 import psutil
@@ -295,25 +294,22 @@ def _autodetect_num_gpus():
 
 
 def _compute_version_info():
-    """Compute the versions of Python, cloudpickle, pyarrow, and Ray.
+    """Compute the versions of Python, pyarrow, and Ray.
 
     Returns:
         A tuple containing the version information.
     """
     ray_version = ray.__version__
-    ray_location = os.path.abspath(ray.__file__)
     python_version = ".".join(map(str, sys.version_info[:3]))
-    cloudpickle_version = cloudpickle.__version__
     pyarrow_version = pyarrow.__version__
-    return (ray_version, ray_location, python_version, cloudpickle_version,
-            pyarrow_version)
+    return (ray_version, python_version, pyarrow_version)
 
 
 def _put_version_info_in_redis(redis_client):
     """Store version information in Redis.
 
     This will be used to detect if workers or drivers are started using
-    different versions of Python, cloudpickle, pyarrow, or Ray.
+    different versions of Python, pyarrow, or Ray.
 
     Args:
         redis_client: A client for the primary Redis shard.
@@ -325,7 +321,7 @@ def check_version_info(redis_client):
     """Check if various version info of this process is correct.
 
     This will be used to detect if workers or drivers are started using
-    different versions of Python, cloudpickle, pyarrow, or Ray. If the version
+    different versions of Python, pyarrow, or Ray. If the version
     information is not present in Redis, then no check is done.
 
     Args:
@@ -347,18 +343,14 @@ def check_version_info(redis_client):
         node_ip_address = ray.services.get_node_ip_address()
         error_message = ("Version mismatch: The cluster was started with:\n"
                          "    Ray: " + true_version_info[0] + "\n"
-                         "    Ray location: " + true_version_info[1] + "\n"
-                         "    Python: " + true_version_info[2] + "\n"
-                         "    Cloudpickle: " + true_version_info[3] + "\n"
-                         "    Pyarrow: " + str(true_version_info[4]) + "\n"
+                         "    Python: " + true_version_info[1] + "\n"
+                         "    Pyarrow: " + str(true_version_info[2]) + "\n"
                          "This process on node " + node_ip_address +
                          " was started with:" + "\n"
                          "    Ray: " + version_info[0] + "\n"
-                         "    Ray location: " + version_info[1] + "\n"
-                         "    Python: " + version_info[2] + "\n"
-                         "    Cloudpickle: " + version_info[3] + "\n"
-                         "    Pyarrow: " + str(version_info[4]))
-        if version_info[:4] != true_version_info[:4]:
+                         "    Python: " + version_info[1] + "\n"
+                         "    Pyarrow: " + str(version_info[2]))
+        if version_info[:2] != true_version_info[:2]:
             raise Exception(error_message)
         else:
             print(error_message)

python/ray/worker.py

@@ -3,7 +3,6 @@ from __future__ import division
 from __future__ import print_function
 
 import atexit
-import cloudpickle as pickle
 import collections
 import colorama
 import copy
@@ -22,6 +21,7 @@ import traceback
 # Ray modules
 import pyarrow
 import pyarrow.plasma as plasma
+import ray.cloudpickle as pickle
 import ray.experimental.state as state
 import ray.serialization as serialization
 import ray.services as services
@@ -1040,6 +1040,9 @@ def _initialize_serialization(worker=global_worker):
     serialize several exception classes that we define for error handling.
     """
     worker.serialization_context = pyarrow.SerializationContext()
+    # Tell the serialization context to use the cloudpickle version that we
+    # ship with Ray.
+    worker.serialization_context.set_pickle(pickle.dumps, pickle.loads)
     pyarrow.register_default_serialization_handlers(
         worker.serialization_context)
 

python/setup.py

@@ -114,7 +114,6 @@ setup(name="ray",
                         "pytest",
                         "pyyaml",
                         "redis",
-                        "cloudpickle == 0.5.2",
                         # The six module is required by pyarrow.
                         "six >= 1.0.0",
                         "flatbuffers"],