From ddba1df802404d635b879002b5268723d887c3f1 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Sun, 11 Dec 2016 12:25:31 -0800 Subject: [PATCH] Start working toward Python3 compatibility. (#117) --- README.md | 2 +- doc/using-ray-with-tensorflow.md | 2 +- examples/alexnet/alexnet.py | 6 ++- examples/alexnet/driver.py | 12 +++-- examples/hyperopt/README.md | 2 +- examples/hyperopt/driver.py | 15 ++++-- examples/hyperopt/hyperopt.py | 6 ++- examples/lbfgs/README.md | 2 +- examples/lbfgs/driver.py | 12 +++-- examples/rl_pong/driver.py | 6 ++- lib/python/ray/__init__.py | 13 ++++-- lib/python/ray/array/distributed/core.py | 10 ++-- lib/python/ray/array/distributed/linalg.py | 8 +++- lib/python/ray/array/distributed/random.py | 6 ++- lib/python/ray/array/remote/core.py | 4 ++ lib/python/ray/array/remote/linalg.py | 4 ++ lib/python/ray/array/remote/random.py | 4 ++ lib/python/ray/config.py | 9 ---- lib/python/ray/pickling.py | 4 ++ lib/python/ray/serialization.py | 5 +- lib/python/ray/services.py | 3 +- lib/python/ray/test/__init__.py | 0 .../python/ray/test}/test_functions.py | 4 ++ lib/python/ray/worker.py | 9 ++-- lib/python/ray/workers/default_worker.py | 2 + lib/python/setup.py | 2 + numbuf/CMakeLists.txt | 6 +-- numbuf/numbuf/__init__.py | 6 ++- numbuf/python/test/runtest.py | 4 ++ numbuf/setup.py | 4 ++ scripts/cluster.py | 14 ++++-- src/common/test/test.py | 2 + .../lib/python/global_scheduler_services.py | 2 + src/global_scheduler/test/test.py | 2 + src/photon/CMakeLists.txt | 6 +-- src/photon/photon/__init__.py | 4 ++ src/photon/photon/photon_services.py | 2 + src/photon/test/test.py | 2 + src/plasma/CMakeLists.txt | 6 +-- src/plasma/lib/python/plasma.py | 7 ++- src/plasma/lib/python/setup.py | 4 ++ src/plasma/test/test.py | 4 +- test/array_test.py | 10 ++-- test/failure_test.py | 6 ++- test/memory_leak_deserialize.py | 14 ------ test/microbenchmarks.py | 46 ++++++++++--------- test/runtest.py | 4 +- test/stress_tests.py | 2 + 48 files changed, 206 insertions(+), 103 deletions(-) delete mode 100644 lib/python/ray/config.py create mode 100644 lib/python/ray/test/__init__.py rename {test => lib/python/ray/test}/test_functions.py (94%) delete mode 100644 test/memory_leak_deserialize.py diff --git a/README.md b/README.md index 4bb175438..013f13761 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ for _ in range(10): # Fetch the results of the tasks and print their average. estimate = np.mean(ray.get(result_ids)) -print "Pi is approximately {}.".format(estimate) +print("Pi is approximately {}.".format(estimate)) ``` Within the for loop, each call to `estimate_pi.remote(100)` sends a message to diff --git a/doc/using-ray-with-tensorflow.md b/doc/using-ray-with-tensorflow.md index eeb596179..a0c27cdb2 100644 --- a/doc/using-ray-with-tensorflow.md +++ b/doc/using-ray-with-tensorflow.md @@ -190,5 +190,5 @@ for iteration in range(NUM_ITERS): # Print the current weights. They should converge to roughly to the values 0.1 # and 0.3 used in generate_fake_x_y_data. if iteration % 20 == 0: - print "Iteration {}: weights are {}".format(iteration, weights) + print("Iteration {}: weights are {}".format(iteration, weights)) ``` diff --git a/examples/alexnet/alexnet.py b/examples/alexnet/alexnet.py index 9f656763f..3c23ea328 100644 --- a/examples/alexnet/alexnet.py +++ b/examples/alexnet/alexnet.py @@ -1,6 +1,10 @@ # The code for AlexNet is copied and adapted from the TensorFlow repository # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/models/image/alexnet/alexnet_benchmark.py. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import ray import numpy as np import tarfile, io @@ -390,7 +394,7 @@ def shuffle(batches): # Randomly permute the order of the batches. permuted_batches = np.random.permutation(batches) new_batches = [] - for i in range(len(batches) / 2): + for i in range(len(batches) // 2): # Swap data between consecutive batches. shuffled_batch1, shuffled_batch2 = shuffle_pair(permuted_batches[2 * i], permuted_batches[2 * i + 1]) new_batches += [shuffled_batch1, shuffled_batch2] diff --git a/examples/alexnet/driver.py b/examples/alexnet/driver.py index 9c5bb092b..8182264c8 100644 --- a/examples/alexnet/driver.py +++ b/examples/alexnet/driver.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np import ray import os @@ -28,7 +32,7 @@ if __name__ == "__main__": imagenet_bucket = s3_resource.Bucket(args.s3_bucket) objects = imagenet_bucket.objects.filter(Prefix=args.key_prefix) image_tar_files = [str(obj.key) for obj in objects.all()] - print "Images will be downloaded from {} files.".format(len(image_tar_files)) + print("Images will be downloaded from {} files.".format(len(image_tar_files))) # Downloading the label file, and create a dictionary mapping the filenames of # the images to their labels. @@ -38,7 +42,7 @@ if __name__ == "__main__": filename_label_pairs = [line.split(" ") for line in filename_label_str] filename_label_dict = dict([(os.path.basename(name), label) for name, label in filename_label_pairs]) filename_label_dict_id = ray.put(filename_label_dict) - print "Labels extracted." + print("Labels extracted.") # Download the imagenet dataset. imagenet_data = alexnet.load_tarfiles_from_s3(args.s3_bucket, image_tar_files, [256, 256]) @@ -60,7 +64,7 @@ if __name__ == "__main__": # Initialize the network and optimizer weights. This is only run once on the # driver. We initialize the weights manually on the workers. sess.run(init_all_variables) - print "Initialized network weights." + print("Initialized network weights.") iteration = 0 while True: @@ -82,7 +86,7 @@ if __name__ == "__main__": gradient_ids.append(alexnet.compute_grad.remote(x_id, y_id, mean_id, weights_id)) # Print the accuracy on a random training batch. - print "Iteration {}: accuracy = {:.3}%".format(iteration, 100 * ray.get(accuracy)) + print("Iteration {}: accuracy = {:.3}%".format(iteration, 100 * ray.get(accuracy))) # Fetch the gradients. This blocks until the gradients have been computed. gradient_sets = ray.get(gradient_ids) diff --git a/examples/hyperopt/README.md b/examples/hyperopt/README.md index b46471d0e..91972fad3 100644 --- a/examples/hyperopt/README.md +++ b/examples/hyperopt/README.md @@ -148,7 +148,7 @@ while len(remaining_ids) > 0: ready_ids, remaining_ids = ray.wait(remaining_ids, num_returns=1) # Get the accuracy corresponding to the ready object ID. accuracy = ray.get(ready_ids[0]) - print "Accuracy {}".format(accuracy) + print("Accuracy {}".format(accuracy)) ``` Note that the above example does not associate the accuracy with the parameters diff --git a/examples/hyperopt/driver.py b/examples/hyperopt/driver.py index 5b389b40d..2d56369b8 100644 --- a/examples/hyperopt/driver.py +++ b/examples/hyperopt/driver.py @@ -1,5 +1,10 @@ # Most of the tensorflow code is adapted from Tensorflow's tutorial on using CNNs to train MNIST # https://www.tensorflow.org/versions/r0.9/tutorials/mnist/pros/index.html#build-a-multilayer-convolutional-network + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np import ray import argparse @@ -24,7 +29,7 @@ if __name__ == "__main__": steps = args.steps # Load the mnist data and turn the data into remote objects. - print "Downloading the MNIST dataset. This may take a minute." + print("Downloading the MNIST dataset. This may take a minute.") mnist = input_data.read_data_sets("MNIST_data", one_hot=True) train_images = ray.put(mnist.train.images) train_labels = ray.put(mnist.train.labels) @@ -65,20 +70,20 @@ if __name__ == "__main__": result_id = ready_ids[0] params = params_mapping[result_id] accuracy = ray.get(result_id) - print """We achieve accuracy {:.3}% with + print("""We achieve accuracy {:.3}% with learning_rate: {:.2} batch_size: {} dropout: {:.2} stddev: {:.2} - """.format(100 * accuracy, params["learning_rate"], params["batch_size"], params["dropout"], params["stddev"]) + """.format(100 * accuracy, params["learning_rate"], params["batch_size"], params["dropout"], params["stddev"])) if accuracy > best_accuracy: best_params = params best_accuracy = accuracy # Record the best performing set of hyperparameters. - print """Best accuracy over {} trials was {:.3} with + print("""Best accuracy over {} trials was {:.3} with learning_rate: {:.2} batch_size: {} dropout: {:.2} stddev: {:.2} - """.format(trials, 100 * best_accuracy, best_params["learning_rate"], best_params["batch_size"], best_params["dropout"], best_params["stddev"]) + """.format(trials, 100 * best_accuracy, best_params["learning_rate"], best_params["batch_size"], best_params["dropout"], best_params["stddev"])) diff --git a/examples/hyperopt/hyperopt.py b/examples/hyperopt/hyperopt.py index 9d3f8e51b..e27005c5d 100644 --- a/examples/hyperopt/hyperopt.py +++ b/examples/hyperopt/hyperopt.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import ray import numpy as np import tensorflow as tf @@ -6,7 +10,7 @@ def get_batch(data, batch_index, batch_size): # This method currently drops data when num_data is not divisible by # batch_size. num_data = data.shape[0] - num_batches = num_data / batch_size + num_batches = num_data // batch_size batch_index %= num_batches return data[(batch_index * batch_size):((batch_index + 1) * batch_size)] diff --git a/examples/lbfgs/README.md b/examples/lbfgs/README.md index 63d6eb2cf..5c6d9791b 100644 --- a/examples/lbfgs/README.md +++ b/examples/lbfgs/README.md @@ -32,7 +32,7 @@ built in methods for loading the data. from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) batch_size = 100 -num_batches = mnist.train.num_examples / batch_size +num_batches = mnist.train.num_examples // batch_size batches = [mnist.train.next_batch(batch_size) for _ in range(num_batches)] ``` diff --git a/examples/lbfgs/driver.py b/examples/lbfgs/driver.py index f1d3643c8..115918f06 100644 --- a/examples/lbfgs/driver.py +++ b/examples/lbfgs/driver.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import ray import numpy as np @@ -115,16 +119,16 @@ if __name__ == "__main__": # algorithm. # Load the mnist data and turn the data into remote objects. - print "Downloading the MNIST dataset. This may take a minute." + print("Downloading the MNIST dataset. This may take a minute.") mnist = input_data.read_data_sets("MNIST_data", one_hot=True) batch_size = 100 - num_batches = mnist.train.num_examples / batch_size + num_batches = mnist.train.num_examples // batch_size batches = [mnist.train.next_batch(batch_size) for _ in range(num_batches)] - print "Putting MNIST in the object store." + print("Putting MNIST in the object store.") batch_ids = [(ray.put(xs), ray.put(ys)) for (xs, ys) in batches] # Initialize the weights for the network to the vector of all zeros. theta_init = 1e-2 * np.random.normal(size=dim) # Use L-BFGS to minimize the loss function. - print "Running L-BFGS." + print("Running L-BFGS.") result = scipy.optimize.fmin_l_bfgs_b(full_loss, theta_init, maxiter=10, fprime=full_grad, disp=True) diff --git a/examples/rl_pong/driver.py b/examples/rl_pong/driver.py index c835ed456..18ee0aa5a 100644 --- a/examples/rl_pong/driver.py +++ b/examples/rl_pong/driver.py @@ -1,6 +1,10 @@ # This code is copied and adapted from Andrej Karpathy's code for learning to # play Pong https://gist.github.com/karpathy/a4166c7fe253700972fcbc77e4ea32c5. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np import cPickle as pickle import ray @@ -135,7 +139,7 @@ if __name__ == "__main__": reward_sum = ray.get(reward_sums[i]) for k in model: grad_buffer[k] += grad[k] # accumulate grad over batch running_reward = reward_sum if running_reward is None else running_reward * 0.99 + reward_sum * 0.01 - print "Batch {}. episode reward total was {}. running mean: {}".format(batch_num, reward_sum, running_reward) + print("Batch {}. episode reward total was {}. running mean: {}".format(batch_num, reward_sum, running_reward)) for k, v in model.iteritems(): g = grad_buffer[k] # gradient rmsprop_cache[k] = decay_rate * rmsprop_cache[k] + (1 - decay_rate) * g ** 2 diff --git a/lib/python/ray/__init__.py b/lib/python/ray/__init__.py index b536dc846..075d41541 100644 --- a/lib/python/ray/__init__.py +++ b/lib/python/ray/__init__.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + # Ray version string __version__ = "0.01" @@ -9,8 +13,7 @@ if hasattr(ctypes, "windll"): # This is done by associating all child processes with a "job" object that imposes this behavior. (lambda kernel32: (lambda job: (lambda n: kernel32.SetInformationJobObject(job, 9, "\0" * 17 + chr(0x8 | 0x4 | 0x20) + "\0" * (n - 18), n))(0x90 if ctypes.sizeof(ctypes.c_void_p) > ctypes.sizeof(ctypes.c_int) else 0x70) and kernel32.AssignProcessToJobObject(job, ctypes.c_void_p(kernel32.GetCurrentProcess())))(ctypes.c_void_p(kernel32.CreateJobObjectW(None, None))) if kernel32 is not None else None)(ctypes.windll.kernel32) -import config -import serialization -from worker import register_class, error_info, init, connect, disconnect, get, put, wait, remote -from worker import Reusable, reusables -from worker import SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, SILENT_MODE +import ray.serialization +from ray.worker import register_class, error_info, init, connect, disconnect, get, put, wait, remote +from ray.worker import Reusable, reusables +from ray.worker import SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, SILENT_MODE diff --git a/lib/python/ray/array/distributed/core.py b/lib/python/ray/array/distributed/core.py index 6481762a6..a307c3953 100644 --- a/lib/python/ray/array/distributed/core.py +++ b/lib/python/ray/array/distributed/core.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np import ray.array.remote as ra import ray @@ -141,10 +145,10 @@ def blockwise_dot(*matrices): n = len(matrices) if n % 2 != 0: raise Exception("blockwise_dot expects an even number of arguments, but len(matrices) is {}.".format(n)) - shape = (matrices[0].shape[0], matrices[n / 2].shape[1]) + shape = (matrices[0].shape[0], matrices[n // 2].shape[1]) result = np.zeros(shape) - for i in range(n / 2): - result += np.dot(matrices[i], matrices[n / 2 + i]) + for i in range(n // 2): + result += np.dot(matrices[i], matrices[n // 2 + i]) return result @ray.remote diff --git a/lib/python/ray/array/distributed/linalg.py b/lib/python/ray/array/distributed/linalg.py index d569cefe4..11b9c3454 100644 --- a/lib/python/ray/array/distributed/linalg.py +++ b/lib/python/ray/array/distributed/linalg.py @@ -1,8 +1,12 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np import ray.array.remote as ra import ray -from core import * +from .core import * __all__ = ["tsqr", "modified_lu", "tsqr_hr", "qr"] @@ -68,7 +72,7 @@ def tsqr(a): else: lower = [a.shape[1], 0] upper = [2 * a.shape[1], BLOCK_SIZE] - ith_index /= 2 + ith_index //= 2 q_block_current = ra.dot.remote(q_block_current, ra.subarray.remote(q_tree[ith_index, j], lower, upper)) q_result.objectids[i] = q_block_current r = current_rs[0] diff --git a/lib/python/ray/array/distributed/random.py b/lib/python/ray/array/distributed/random.py index c7b44a713..ff3343b18 100644 --- a/lib/python/ray/array/distributed/random.py +++ b/lib/python/ray/array/distributed/random.py @@ -1,8 +1,12 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np import ray.array.remote as ra import ray -from core import * +from .core import * @ray.remote def normal(shape): diff --git a/lib/python/ray/array/remote/core.py b/lib/python/ray/array/remote/core.py index c8cd3ecf4..7d8e69e52 100644 --- a/lib/python/ray/array/remote/core.py +++ b/lib/python/ray/array/remote/core.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np import ray diff --git a/lib/python/ray/array/remote/linalg.py b/lib/python/ray/array/remote/linalg.py index ab01788c3..6ddb5fa89 100644 --- a/lib/python/ray/array/remote/linalg.py +++ b/lib/python/ray/array/remote/linalg.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np import ray diff --git a/lib/python/ray/array/remote/random.py b/lib/python/ray/array/remote/random.py index 7fb3def0e..635b9c136 100644 --- a/lib/python/ray/array/remote/random.py +++ b/lib/python/ray/array/remote/random.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import numpy as np import ray diff --git a/lib/python/ray/config.py b/lib/python/ray/config.py deleted file mode 100644 index 6453e0a32..000000000 --- a/lib/python/ray/config.py +++ /dev/null @@ -1,9 +0,0 @@ -import datetime -import os.path -import sys -import tempfile - -def get_log_file_path(name): - return os.path.join( - os.path.join("/tmp" if sys.platform.startswith("darwin") else tempfile.gettempdir(), "raylogs"), - ("{:%Y-%m-%d-%H-%M-%S}-{}").format(datetime.datetime.now(), name.replace(":", "-"))) diff --git a/lib/python/ray/pickling.py b/lib/python/ray/pickling.py index b0befa962..40e63089f 100644 --- a/lib/python/ray/pickling.py +++ b/lib/python/ray/pickling.py @@ -1,5 +1,9 @@ # Note that a little bit of code here is taken and slightly modified from the pickler because it was not possible to change its behavior otherwise. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import sys from ctypes import c_void_p from cloudpickle import pickle, cloudpickle, CloudPickler, load, loads diff --git a/lib/python/ray/serialization.py b/lib/python/ray/serialization.py index 64d708c98..b91ccad6b 100644 --- a/lib/python/ray/serialization.py +++ b/lib/python/ray/serialization.py @@ -1,9 +1,12 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import numpy as np -import pickling import numbuf +import ray.pickling as pickling + def check_serializable(cls): """Throws an exception if Ray cannot serialize this class efficiently. diff --git a/lib/python/ray/services.py b/lib/python/ray/services.py index 5e42fcd63..bfc789b78 100644 --- a/lib/python/ray/services.py +++ b/lib/python/ray/services.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import psutil @@ -10,7 +12,6 @@ import sys import time # Ray modules -import config import photon import plasma import global_scheduler diff --git a/lib/python/ray/test/__init__.py b/lib/python/ray/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/test_functions.py b/lib/python/ray/test/test_functions.py similarity index 94% rename from test/test_functions.py rename to lib/python/ray/test/test_functions.py index b8a242ac0..3f7dec3ac 100644 --- a/test/test_functions.py +++ b/lib/python/ray/test/test_functions.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import ray import numpy as np diff --git a/lib/python/ray/worker.py b/lib/python/ray/worker.py index 9c184eacd..befad89fe 100644 --- a/lib/python/ray/worker.py +++ b/lib/python/ray/worker.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import hashlib @@ -16,10 +18,9 @@ import threading import string # Ray modules -import config -import pickling -import serialization -import services +import ray.pickling as pickling +import ray.serialization as serialization +import ray.services as services import numbuf import photon import plasma diff --git a/lib/python/ray/workers/default_worker.py b/lib/python/ray/workers/default_worker.py index c204f4458..f3a7349f8 100644 --- a/lib/python/ray/workers/default_worker.py +++ b/lib/python/ray/workers/default_worker.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import sys diff --git a/lib/python/setup.py b/lib/python/setup.py index 3e8c180cb..a79a97654 100644 --- a/lib/python/setup.py +++ b/lib/python/setup.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import os diff --git a/numbuf/CMakeLists.txt b/numbuf/CMakeLists.txt index a5f8a1f85..5c2e3e80e 100644 --- a/numbuf/CMakeLists.txt +++ b/numbuf/CMakeLists.txt @@ -12,17 +12,17 @@ message(STATUS "Trying custom approach for finding Python.") # Start off by figuring out which Python executable to use. find_program(CUSTOM_PYTHON_EXECUTABLE python) message(STATUS "Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}") -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print 'python' + sys.version[0:3]" +execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print('python' + sys.version[0:3])" OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "PYTHON_LIBRARY_NAME: " ${PYTHON_LIBRARY_NAME}) # Now find the Python include directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print get_python_inc()" +execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())" OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS}) # Now find the Python libraries. We'll start by looking near the Python # executable. If that fails, then we'll look near the Python include # directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print sys.exec_prefix" +execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print(sys.exec_prefix)" OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "PYTHON_PREFIX: " ${PYTHON_PREFIX}) FIND_LIBRARY(PYTHON_LIBRARIES diff --git a/numbuf/numbuf/__init__.py b/numbuf/numbuf/__init__.py index 21cb259ba..42ad48c78 100644 --- a/numbuf/numbuf/__init__.py +++ b/numbuf/numbuf/__init__.py @@ -1 +1,5 @@ -from libnumbuf import * +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from numbuf.libnumbuf import * diff --git a/numbuf/python/test/runtest.py b/numbuf/python/test/runtest.py index 56a1f0996..59afd9d7a 100644 --- a/numbuf/python/test/runtest.py +++ b/numbuf/python/test/runtest.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import unittest import numbuf import numpy as np diff --git a/numbuf/setup.py b/numbuf/setup.py index 26b4f208a..f6f1932ab 100644 --- a/numbuf/setup.py +++ b/numbuf/setup.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import subprocess from setuptools import setup, find_packages import setuptools.command.install as _install diff --git a/scripts/cluster.py b/scripts/cluster.py index c34c67bd1..6be24e71f 100644 --- a/scripts/cluster.py +++ b/scripts/cluster.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import os import subprocess import socket @@ -63,7 +67,7 @@ class RayCluster(object): raise Exception("Commands run over ssh must not contain the single quote character. This command does: {}".format(command)) full_command = "ssh -o StrictHostKeyChecking=no -i {} {}@{} '{}'".format(self.key_file, self.username, node_ip_address, command) subprocess.call([full_command], shell=True) - print "Finished running command '{}' on {}@{}.".format(command, self.username, node_ip_address) + print("Finished running command '{}' on {}@{}.".format(command, self.username, node_ip_address)) def _run_parallel_functions(self, functions, inputs): """Run functions in parallel. @@ -117,7 +121,7 @@ class RayCluster(object): self._run_command_over_ssh(node_ip_address, command) inputs = zip(node_ip_addresses, commands) self._run_parallel_functions(len(self.node_ip_addresses) * [function], inputs) - print "Finished running commands {} on all nodes.".format(inputs) + print("Finished running commands {} on all nodes.".format(inputs)) def install_ray(self): """Install Ray on every node in the cluster. @@ -172,7 +176,7 @@ class RayCluster(object): self.run_command_over_ssh_on_all_nodes_in_parallel(start_workers_commands) setup_env_path = os.path.join(self.installation_directory, "ray/setup-env.sh") - print """ + print(""" The cluster has been started. You can attach to the cluster by sshing to the head node with the following command. ssh -i {} {}@{} @@ -185,7 +189,7 @@ class RayCluster(object): import ray ray.init(node_ip_address="{}", scheduler_address="{}:10001") - """.format(self.key_file, self.username, self.node_ip_addresses[0], setup_env_path, self.node_private_ip_addresses[0], self.node_private_ip_addresses[0]) + """.format(self.key_file, self.username, self.node_ip_addresses[0], setup_env_path, self.node_private_ip_addresses[0], self.node_private_ip_addresses[0])) def stop_ray(self): """Kill all of the processes in the Ray cluster. @@ -317,7 +321,7 @@ if __name__ == "__main__": # will be expanded locally instead of remotely. echo_home_command = "ssh -o StrictHostKeyChecking=no -i {} {}@{} 'echo $HOME'".format(key_file, username, node_ip_addresses[0]) installation_directory = subprocess.check_output(echo_home_command, shell=True).strip() - print "Using '{}' as the home directory on the cluster.".format(installation_directory) + print("Using '{}' as the home directory on the cluster.".format(installation_directory)) # Create the Raycluster object. cluster = RayCluster(node_ip_addresses, node_private_ip_addresses, username, key_file, installation_directory) # Drop into an IPython shell. diff --git a/src/common/test/test.py b/src/common/test/test.py index b3136de60..e38efd6f3 100644 --- a/src/common/test/test.py +++ b/src/common/test/test.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import numpy as np diff --git a/src/global_scheduler/lib/python/global_scheduler_services.py b/src/global_scheduler/lib/python/global_scheduler_services.py index 5191ffada..86dffff93 100644 --- a/src/global_scheduler/lib/python/global_scheduler_services.py +++ b/src/global_scheduler/lib/python/global_scheduler_services.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import os diff --git a/src/global_scheduler/test/test.py b/src/global_scheduler/test/test.py index 13325871a..405ae4d14 100644 --- a/src/global_scheduler/test/test.py +++ b/src/global_scheduler/test/test.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import numpy as np diff --git a/src/photon/CMakeLists.txt b/src/photon/CMakeLists.txt index fa5b6cc16..168c1d2e4 100644 --- a/src/photon/CMakeLists.txt +++ b/src/photon/CMakeLists.txt @@ -6,17 +6,17 @@ message(STATUS "Trying custom approach for finding Python.") # Start off by figuring out which Python executable to use. find_program(CUSTOM_PYTHON_EXECUTABLE python) message(STATUS "Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}") -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print 'python' + sys.version[0:3]" +execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print('python' + sys.version[0:3])" OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "PYTHON_LIBRARY_NAME: " ${PYTHON_LIBRARY_NAME}) # Now find the Python include directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print get_python_inc()" +execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())" OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS}) # Now find the Python libraries. We'll start by looking near the Python # executable. If that fails, then we'll look near the Python include # directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print sys.exec_prefix" +execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print(sys.exec_prefix)" OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "PYTHON_PREFIX: " ${PYTHON_PREFIX}) FIND_LIBRARY(PYTHON_LIBRARIES diff --git a/src/photon/photon/__init__.py b/src/photon/photon/__init__.py index d34d8d3e1..6a24ca4e0 100644 --- a/src/photon/photon/__init__.py +++ b/src/photon/photon/__init__.py @@ -1,2 +1,6 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + from libphoton import * from photon_services import * diff --git a/src/photon/photon/photon_services.py b/src/photon/photon/photon_services.py index 6707a797e..59db729a4 100644 --- a/src/photon/photon/photon_services.py +++ b/src/photon/photon/photon_services.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import os diff --git a/src/photon/test/test.py b/src/photon/test/test.py index 6df731d2a..e9d326d21 100644 --- a/src/photon/test/test.py +++ b/src/photon/test/test.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import numpy as np diff --git a/src/plasma/CMakeLists.txt b/src/plasma/CMakeLists.txt index 15ab58a15..6c75c3017 100644 --- a/src/plasma/CMakeLists.txt +++ b/src/plasma/CMakeLists.txt @@ -6,17 +6,17 @@ message(STATUS "Trying custom approach for finding Python.") # Start off by figuring out which Python executable to use. find_program(CUSTOM_PYTHON_EXECUTABLE python) message(STATUS "Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}") -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print 'python' + sys.version[0:3]" +execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print('python' + sys.version[0:3])" OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "PYTHON_LIBRARY_NAME: " ${PYTHON_LIBRARY_NAME}) # Now find the Python include directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print get_python_inc()" +execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())" OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS}) # Now find the Python libraries. We'll start by looking near the Python # executable. If that fails, then we'll look near the Python include # directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print sys.exec_prefix" +execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print(sys.exec_prefix)" OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "PYTHON_PREFIX: " ${PYTHON_PREFIX}) FIND_LIBRARY(PYTHON_LIBRARIES diff --git a/src/plasma/lib/python/plasma.py b/src/plasma/lib/python/plasma.py index 1be68ec93..a519abbf3 100644 --- a/src/plasma/lib/python/plasma.py +++ b/src/plasma/lib/python/plasma.py @@ -1,8 +1,13 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import os import random import subprocess import time -import libplasma + +from . import libplasma PLASMA_ID_SIZE = 20 PLASMA_WAIT_TIMEOUT = 2 ** 30 diff --git a/src/plasma/lib/python/setup.py b/src/plasma/lib/python/setup.py index c02f4d220..6c638be24 100644 --- a/src/plasma/lib/python/setup.py +++ b/src/plasma/lib/python/setup.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + from setuptools import setup, find_packages import setuptools.command.install as _install diff --git a/src/plasma/test/test.py b/src/plasma/test/test.py index 17327e8b3..60b7939fb 100644 --- a/src/plasma/test/test.py +++ b/src/plasma/test/test.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import numpy as np @@ -534,7 +536,7 @@ class TestPlasmaManager(unittest.TestCase): # loop slow down the manager so much that some of the asynchronous Redis # commands timeout triggering fatal failure callbacks. n = 40 - for i in range(n * (n + 1) / 2): + for i in range(n * (n + 1) // 2): if i % 2 == 0: object_id, _, _ = create_object(self.client1, 200, 200) else: diff --git a/test/array_test.py b/test/array_test.py index 8e2d1e84d..7c77bd55e 100644 --- a/test/array_test.py +++ b/test/array_test.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import unittest import ray import numpy as np @@ -130,7 +134,7 @@ class DistributedArrayTest(unittest.TestCase): # test da.linalg.modified_lu def test_modified_lu(d1, d2): - print "testing dist_modified_lu with d1 = " + str(d1) + ", d2 = " + str(d2) + print("testing dist_modified_lu with d1 = " + str(d1) + ", d2 = " + str(d2)) assert d1 >= d2 k = min(d1, d2) m = ra.random.normal.remote([d1, d2]) @@ -153,7 +157,7 @@ class DistributedArrayTest(unittest.TestCase): # test dist_tsqr_hr def test_dist_tsqr_hr(d1, d2): - print "testing dist_tsqr_hr with d1 = " + str(d1) + ", d2 = " + str(d2) + print("testing dist_tsqr_hr with d1 = " + str(d1) + ", d2 = " + str(d2)) a = da.random.normal.remote([d1, d2]) y, t, y_top, r = da.linalg.tsqr_hr.remote(a) a_val = ray.get(da.assemble.remote(a)) @@ -171,7 +175,7 @@ class DistributedArrayTest(unittest.TestCase): test_dist_tsqr_hr(d1, d2) def test_dist_qr(d1, d2): - print "testing qr with d1 = {}, and d2 = {}.".format(d1, d2) + print("testing qr with d1 = {}, and d2 = {}.".format(d1, d2)) a = da.random.normal.remote([d1, d2]) K = min(d1, d2) q, r = da.linalg.qr.remote(a) diff --git a/test/failure_test.py b/test/failure_test.py index 2e251767d..7ca31d813 100644 --- a/test/failure_test.py +++ b/test/failure_test.py @@ -1,8 +1,12 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import unittest import ray import time -import test_functions +import ray.test.test_functions as test_functions def wait_for_errors(error_type, num_errors, timeout=10): start_time = time.time() diff --git a/test/memory_leak_deserialize.py b/test/memory_leak_deserialize.py deleted file mode 100644 index 00db7dc36..000000000 --- a/test/memory_leak_deserialize.py +++ /dev/null @@ -1,14 +0,0 @@ -# This code reproduces a memory leak we had in the past - -import os -import numpy as np -import ray - -ray.init(start_ray_local=True, num_workers=1) - -d = {"w": np.zeros(1000000)} - -obj_capsule, contained_objectids = ray.libraylib.serialize_object(ray.worker.global_worker.handle, d) - -while True: - ray.libraylib.deserialize_object(ray.worker.global_worker.handle, obj_capsule) diff --git a/test/microbenchmarks.py b/test/microbenchmarks.py index 04ce08bba..9082aa21e 100644 --- a/test/microbenchmarks.py +++ b/test/microbenchmarks.py @@ -1,9 +1,13 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import unittest import ray import time import numpy as np -import test_functions +import ray.test.test_functions as test_functions class MicroBenchmarkTest(unittest.TestCase): @@ -20,11 +24,11 @@ class MicroBenchmarkTest(unittest.TestCase): elapsed_times.append(end_time - start_time) elapsed_times = np.sort(elapsed_times) average_elapsed_time = sum(elapsed_times) / 1000 - print "Time required to submit an empty function call:" - print " Average: {}".format(average_elapsed_time) - print " 90th percentile: {}".format(elapsed_times[900]) - print " 99th percentile: {}".format(elapsed_times[990]) - print " worst: {}".format(elapsed_times[999]) + print("Time required to submit an empty function call:") + print(" Average: {}".format(average_elapsed_time)) + print(" 90th percentile: {}".format(elapsed_times[900])) + print(" 99th percentile: {}".format(elapsed_times[990])) + print(" worst: {}".format(elapsed_times[999])) # average_elapsed_time should be about 0.00038 # measure the time required to submit a remote task to the scheduler (where the remote task returns one value) @@ -36,11 +40,11 @@ class MicroBenchmarkTest(unittest.TestCase): elapsed_times.append(end_time - start_time) elapsed_times = np.sort(elapsed_times) average_elapsed_time = sum(elapsed_times) / 1000 - print "Time required to submit a trivial function call:" - print " Average: {}".format(average_elapsed_time) - print " 90th percentile: {}".format(elapsed_times[900]) - print " 99th percentile: {}".format(elapsed_times[990]) - print " worst: {}".format(elapsed_times[999]) + print("Time required to submit a trivial function call:") + print(" Average: {}".format(average_elapsed_time)) + print(" 90th percentile: {}".format(elapsed_times[900])) + print(" 99th percentile: {}".format(elapsed_times[990])) + print(" worst: {}".format(elapsed_times[999])) # average_elapsed_time should be about 0.001 # measure the time required to submit a remote task to the scheduler and get the result @@ -53,11 +57,11 @@ class MicroBenchmarkTest(unittest.TestCase): elapsed_times.append(end_time - start_time) elapsed_times = np.sort(elapsed_times) average_elapsed_time = sum(elapsed_times) / 1000 - print "Time required to submit a trivial function call and get the result:" - print " Average: {}".format(average_elapsed_time) - print " 90th percentile: {}".format(elapsed_times[900]) - print " 99th percentile: {}".format(elapsed_times[990]) - print " worst: {}".format(elapsed_times[999]) + print("Time required to submit a trivial function call and get the result:") + print(" Average: {}".format(average_elapsed_time)) + print(" 90th percentile: {}".format(elapsed_times[900])) + print(" 99th percentile: {}".format(elapsed_times[990])) + print(" worst: {}".format(elapsed_times[999])) # average_elapsed_time should be about 0.0013 # measure the time required to do do a put @@ -69,11 +73,11 @@ class MicroBenchmarkTest(unittest.TestCase): elapsed_times.append(end_time - start_time) elapsed_times = np.sort(elapsed_times) average_elapsed_time = sum(elapsed_times) / 1000 - print "Time required to put an int:" - print " Average: {}".format(average_elapsed_time) - print " 90th percentile: {}".format(elapsed_times[900]) - print " 99th percentile: {}".format(elapsed_times[990]) - print " worst: {}".format(elapsed_times[999]) + print("Time required to put an int:") + print(" Average: {}".format(average_elapsed_time)) + print(" 90th percentile: {}".format(elapsed_times[900])) + print(" 99th percentile: {}".format(elapsed_times[990])) + print(" worst: {}".format(elapsed_times[999])) # average_elapsed_time should be about 0.00087 ray.worker.cleanup() diff --git a/test/runtest.py b/test/runtest.py index ad5ce7f51..59d3112b0 100644 --- a/test/runtest.py +++ b/test/runtest.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import unittest @@ -8,7 +10,7 @@ import string import sys from collections import namedtuple -import test_functions +import ray.test.test_functions as test_functions import ray.array.remote as ra import ray.array.distributed as da diff --git a/test/stress_tests.py b/test/stress_tests.py index 1cbb83fe8..85c31b50c 100644 --- a/test/stress_tests.py +++ b/test/stress_tests.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import +from __future__ import division from __future__ import print_function import unittest