diff --git a/.gitignore b/.gitignore index 346c56c5f..666380f61 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # The build output should clearly not be checked in -/python/core +/python/ray/core /src/common/thirdparty/redis /numbuf/thirdparty/arrow diff --git a/.travis.yml b/.travis.yml index 4c9a4866d..591b8b9be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,10 +28,10 @@ matrix: install: [] script: - .travis/check-git-clang-format-output.sh - # Try generating Sphinx documentation. To do this, we need to setup some - # Python stuff. - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - - bash miniconda.sh -b -p $HOME/miniconda + # Try generating Sphinx documentation. To do this, we need to install + # Ray first. + - ./.travis/install-dependencies.sh + - ./.travis/install-ray.sh - export PATH="$HOME/miniconda/bin:$PATH" - cd doc - pip install -r requirements-doc.txt @@ -46,16 +46,16 @@ matrix: - ./.travis/install-dependencies.sh - ./.travis/install-ray.sh - - cd python/core - - bash ../../src/common/test/run_valgrind.sh - - bash ../../src/plasma/test/run_valgrind.sh - - bash ../../src/local_scheduler/test/run_valgrind.sh - - cd ../.. + - cd python/ray/core + - bash ../../../src/common/test/run_valgrind.sh + - bash ../../../src/plasma/test/run_valgrind.sh + - bash ../../../src/local_scheduler/test/run_valgrind.sh + - cd ../../.. script: - - python ./python/plasma/test/test.py valgrind - - python ./python/local_scheduler/test/test.py valgrind - - python ./python/global_scheduler/test/test.py valgrind + - python ./python/ray/plasma/test/test.py valgrind + - python ./python/ray/local_scheduler/test/test.py valgrind + - python ./python/ray/global_scheduler/test/test.py valgrind install: - ./.travis/install-dependencies.sh @@ -63,22 +63,22 @@ install: - if [[ "$PYTHON" == "3.5" ]]; then export PATH="$HOME/miniconda/bin:$PATH"; fi - - cd python/core - - bash ../../src/common/test/run_tests.sh - - bash ../../src/plasma/test/run_tests.sh - - bash ../../src/local_scheduler/test/run_tests.sh - - cd ../.. + - cd python/ray/core + - bash ../../../src/common/test/run_tests.sh + - bash ../../../src/plasma/test/run_tests.sh + - bash ../../../src/local_scheduler/test/run_tests.sh + - cd ../../.. script: - if [[ "$PYTHON" == "3.5" ]]; then export PATH="$HOME/miniconda/bin:$PATH"; fi - python src/numbuf/python/test/runtest.py - - python python/common/test/test.py - - python python/common/redis_module/runtest.py - - python python/plasma/test/test.py - - python python/local_scheduler/test/test.py - - python python/global_scheduler/test/test.py + - python python/ray/common/test/test.py + - python python/ray/common/redis_module/runtest.py + - python python/ray/plasma/test/test.py + - python python/ray/local_scheduler/test/test.py + - python python/ray/global_scheduler/test/test.py - python test/runtest.py - python test/array_test.py diff --git a/.travis/install-dependencies.sh b/.travis/install-dependencies.sh index 041be3950..0507ebe66 100755 --- a/.travis/install-dependencies.sh +++ b/.travis/install-dependencies.sh @@ -58,6 +58,14 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" pip install numpy cloudpickle funcsigs colorama psutil redis tensorflow +elif [[ "$LINT" == "1" ]]; then + sudo apt-get update + sudo apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip + # Install miniconda. + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + pip install numpy cloudpickle funcsigs colorama psutil redis else echo "Unrecognized environment." exit 1 diff --git a/.travis/install-ray.sh b/.travis/install-ray.sh index ae520ceba..a3555248e 100755 --- a/.travis/install-ray.sh +++ b/.travis/install-ray.sh @@ -20,6 +20,13 @@ elif [[ "$PYTHON" == "3.5" ]]; then python setup.py install --user popd +elif [[ "$LINT" == "1" ]]; then + export PATH="$HOME/miniconda/bin:$PATH" + + pushd "$ROOT_DIR/../python" + python setup.py install --user + popd + else echo "Unrecognized Python version." exit 1 diff --git a/build.sh b/build.sh index d5ad48ea9..8ce3e5fd9 100755 --- a/build.sh +++ b/build.sh @@ -24,8 +24,8 @@ bash "$ROOT_DIR/src/numbuf/thirdparty/download_thirdparty.sh" bash "$ROOT_DIR/src/numbuf/thirdparty/build_thirdparty.sh" # Now build everything. -pushd "$ROOT_DIR/python/core" - cmake -DCMAKE_BUILD_TYPE=Release ../.. +pushd "$ROOT_DIR/python/ray/core" + cmake -DCMAKE_BUILD_TYPE=Release ../../.. make clean make popd diff --git a/doc/README.md b/doc/README.md index f001d0d68..153795a9a 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1,6 +1,7 @@ # Ray Documentation To compile the documentation, run the following commands from this directory. +Note that Ray must be installed first. ``` pip install -r requirements-doc.txt diff --git a/doc/source/conf.py b/doc/source/conf.py index 4c4d77cca..2963d0249 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,12 +16,6 @@ import sys import os import shlex -# These lines added to enable Sphinx to work without installing Ray. -import mock -MOCK_MODULES = ["global_scheduler", "numbuf", "local_scheduler", "plasma"] -for mod_name in MOCK_MODULES: - sys.modules[mod_name] = mock.Mock() - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. diff --git a/python/ray/actor.py b/python/ray/actor.py index 1088ce7e2..40dc95492 100644 --- a/python/ray/actor.py +++ b/python/ray/actor.py @@ -6,10 +6,10 @@ import hashlib import inspect import json import numpy as np -import local_scheduler import random import traceback +import ray.local_scheduler as local_scheduler import ray.pickling as pickling import ray.worker import ray.experimental.state as state diff --git a/python/common/__init__.py b/python/ray/common/__init__.py similarity index 100% rename from python/common/__init__.py rename to python/ray/common/__init__.py diff --git a/python/common/redis_module/.gitkeep b/python/ray/common/redis_module/.gitkeep similarity index 100% rename from python/common/redis_module/.gitkeep rename to python/ray/common/redis_module/.gitkeep diff --git a/python/common/redis_module/runtest.py b/python/ray/common/redis_module/runtest.py similarity index 96% rename from python/common/redis_module/runtest.py rename to python/ray/common/redis_module/runtest.py index 690d93fb9..1bc1164af 100644 --- a/python/common/redis_module/runtest.py +++ b/python/ray/common/redis_module/runtest.py @@ -11,15 +11,6 @@ import unittest import redis import ray.services -# Check if the redis-server binary is present. -redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server") -if not os.path.exists(redis_path): - raise Exception("You do not have the redis-server binary. Run `make test` in the plasma directory to get it.") - -# Absolute path of the ray redis module. -module_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/redis_module/libray_redis_module.so") -print("path to the redis module is {}".format(module_path)) - OBJECT_INFO_PREFIX = "OI:" OBJECT_LOCATION_PREFIX = "OL:" OBJECT_SUBSCRIBE_PREFIX = "OS:" diff --git a/python/common/test/test.py b/python/ray/common/test/test.py similarity index 99% rename from python/common/test/test.py rename to python/ray/common/test/test.py index 8bc27daab..05d59eec2 100644 --- a/python/common/test/test.py +++ b/python/ray/common/test/test.py @@ -7,7 +7,7 @@ import pickle import sys import unittest -import local_scheduler +import ray.local_scheduler as local_scheduler ID_SIZE = 20 diff --git a/python/common/thirdparty/redis/src/.gitkeep b/python/ray/common/thirdparty/redis/src/.gitkeep similarity index 100% rename from python/common/thirdparty/redis/src/.gitkeep rename to python/ray/common/thirdparty/redis/src/.gitkeep diff --git a/python/core/__init__.py b/python/ray/core/__init__.py similarity index 100% rename from python/core/__init__.py rename to python/ray/core/__init__.py diff --git a/python/core/src/__init__.py b/python/ray/core/src/__init__.py similarity index 100% rename from python/core/src/__init__.py rename to python/ray/core/src/__init__.py diff --git a/python/core/src/local_scheduler/__init__.py b/python/ray/core/src/local_scheduler/__init__.py similarity index 100% rename from python/core/src/local_scheduler/__init__.py rename to python/ray/core/src/local_scheduler/__init__.py diff --git a/python/core/src/numbuf/__init__.py b/python/ray/core/src/numbuf/__init__.py similarity index 100% rename from python/core/src/numbuf/__init__.py rename to python/ray/core/src/numbuf/__init__.py diff --git a/python/core/src/plasma/__init__.py b/python/ray/core/src/plasma/__init__.py similarity index 100% rename from python/core/src/plasma/__init__.py rename to python/ray/core/src/plasma/__init__.py diff --git a/python/global_scheduler/__init__.py b/python/ray/global_scheduler/__init__.py similarity index 100% rename from python/global_scheduler/__init__.py rename to python/ray/global_scheduler/__init__.py diff --git a/python/global_scheduler/build/.gitkeep b/python/ray/global_scheduler/build/.gitkeep similarity index 100% rename from python/global_scheduler/build/.gitkeep rename to python/ray/global_scheduler/build/.gitkeep diff --git a/python/global_scheduler/global_scheduler_services.py b/python/ray/global_scheduler/global_scheduler_services.py similarity index 100% rename from python/global_scheduler/global_scheduler_services.py rename to python/ray/global_scheduler/global_scheduler_services.py diff --git a/python/global_scheduler/test/test.py b/python/ray/global_scheduler/test/test.py similarity index 97% rename from python/global_scheduler/test/test.py rename to python/ray/global_scheduler/test/test.py index dac0064d9..9be25d6ae 100644 --- a/python/global_scheduler/test/test.py +++ b/python/ray/global_scheduler/test/test.py @@ -13,10 +13,10 @@ import threading import time import unittest -import global_scheduler -import local_scheduler -import plasma -from plasma.utils import random_object_id, generate_metadata, write_to_data_buffer, create_object_with_id, create_object +import ray.global_scheduler as global_scheduler +import ray.local_scheduler as local_scheduler +import ray.plasma as plasma +from ray.plasma.utils import random_object_id, generate_metadata, write_to_data_buffer, create_object_with_id, create_object from ray import services diff --git a/python/local_scheduler/__init__.py b/python/ray/local_scheduler/__init__.py similarity index 68% rename from python/local_scheduler/__init__.py rename to python/ray/local_scheduler/__init__.py index 4cd8b6d03..d3895317f 100644 --- a/python/local_scheduler/__init__.py +++ b/python/ray/local_scheduler/__init__.py @@ -2,5 +2,5 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from core.src.local_scheduler.liblocal_scheduler_library import * +from ray.core.src.local_scheduler.liblocal_scheduler_library import * from .local_scheduler_services import * diff --git a/python/local_scheduler/build/.gitkeep b/python/ray/local_scheduler/build/.gitkeep similarity index 100% rename from python/local_scheduler/build/.gitkeep rename to python/ray/local_scheduler/build/.gitkeep diff --git a/python/local_scheduler/local_scheduler_services.py b/python/ray/local_scheduler/local_scheduler_services.py similarity index 100% rename from python/local_scheduler/local_scheduler_services.py rename to python/ray/local_scheduler/local_scheduler_services.py diff --git a/python/local_scheduler/test/test.py b/python/ray/local_scheduler/test/test.py similarity index 98% rename from python/local_scheduler/test/test.py rename to python/ray/local_scheduler/test/test.py index 6ae557acf..2c3f09bb0 100644 --- a/python/local_scheduler/test/test.py +++ b/python/ray/local_scheduler/test/test.py @@ -12,8 +12,8 @@ import threading import time import unittest -import local_scheduler -import plasma +import ray.local_scheduler as local_scheduler +import ray.plasma as plasma USE_VALGRIND = False ID_SIZE = 20 diff --git a/python/numbuf/__init__.py b/python/ray/numbuf/__init__.py similarity index 95% rename from python/numbuf/__init__.py rename to python/ray/numbuf/__init__.py index bc541516a..b4b58bc1f 100644 --- a/python/numbuf/__init__.py +++ b/python/ray/numbuf/__init__.py @@ -11,7 +11,7 @@ If you are using Anaconda, try fixing this problem by running: """ try: - from core.src.numbuf.libnumbuf import * + from ray.core.src.numbuf.libnumbuf import * except ImportError as e: if hasattr(e, "msg") and isinstance(e.msg, str) and ("libstdc++" in e.msg or "CXX" in e.msg): # This code path should be taken with Python 3. diff --git a/python/plasma/.gitkeep b/python/ray/plasma/.gitkeep similarity index 100% rename from python/plasma/.gitkeep rename to python/ray/plasma/.gitkeep diff --git a/python/plasma/__init__.py b/python/ray/plasma/__init__.py similarity index 77% rename from python/plasma/__init__.py rename to python/ray/plasma/__init__.py index 6fe09d30f..b49ea4b9d 100644 --- a/python/plasma/__init__.py +++ b/python/ray/plasma/__init__.py @@ -2,4 +2,4 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from plasma.plasma import * +from ray.plasma.plasma import * diff --git a/python/plasma/plasma.py b/python/ray/plasma/plasma.py similarity index 98% rename from python/plasma/plasma.py rename to python/ray/plasma/plasma.py index 14fe2e1c8..c67eb4ef6 100644 --- a/python/plasma/plasma.py +++ b/python/ray/plasma/plasma.py @@ -8,9 +8,9 @@ import subprocess import sys import time -import core.src.plasma.libplasma as libplasma -from core.src.plasma.libplasma import plasma_object_exists_error -from core.src.plasma.libplasma import plasma_out_of_memory_error +import ray.core.src.plasma.libplasma as libplasma +from ray.core.src.plasma.libplasma import plasma_object_exists_error +from ray.core.src.plasma.libplasma import plasma_out_of_memory_error PLASMA_ID_SIZE = 20 PLASMA_WAIT_TIMEOUT = 2 ** 30 diff --git a/python/plasma/test/test.py b/python/ray/plasma/test/test.py similarity index 99% rename from python/plasma/test/test.py rename to python/ray/plasma/test/test.py index e87220463..5cc6a7c6e 100644 --- a/python/plasma/test/test.py +++ b/python/ray/plasma/test/test.py @@ -15,8 +15,8 @@ import threading import time import unittest -import plasma -from plasma.utils import random_object_id, generate_metadata, write_to_data_buffer, create_object_with_id, create_object +import ray.plasma as plasma +from ray.plasma.utils import random_object_id, generate_metadata, write_to_data_buffer, create_object_with_id, create_object from ray import services USE_VALGRIND = False diff --git a/python/plasma/utils.py b/python/ray/plasma/utils.py similarity index 100% rename from python/plasma/utils.py rename to python/ray/plasma/utils.py diff --git a/python/ray/serialization.py b/python/ray/serialization.py index c650bbc85..9f5228824 100644 --- a/python/ray/serialization.py +++ b/python/ray/serialization.py @@ -3,8 +3,8 @@ from __future__ import division from __future__ import print_function import numpy as np -import numbuf +import ray.numbuf as numbuf import ray.pickling as pickling def check_serializable(cls): diff --git a/python/ray/services.py b/python/ray/services.py index 5d3ad1ad4..7e91f002d 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -17,9 +17,9 @@ import time import threading # Ray modules -import local_scheduler -import plasma -import global_scheduler +import ray.local_scheduler as local_scheduler +import ray.plasma as plasma +import ray.global_scheduler as global_scheduler PROCESS_TYPE_WORKER = "worker" PROCESS_TYPE_LOCAL_SCHEDULER = "local_scheduler" @@ -217,8 +217,8 @@ def start_redis(port=None, num_retries=20, stdout_file=None, stderr_file=None, Raises: Exception: An exception is raised if Redis could not be started. """ - redis_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/common/thirdparty/redis/src/redis-server") - redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/common/redis_module/libray_redis_module.so") + redis_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "./core/src/common/thirdparty/redis/src/redis-server") + redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "./core/src/common/redis_module/libray_redis_module.so") assert os.path.isfile(redis_filepath) assert os.path.isfile(redis_module) counter = 0 diff --git a/python/ray/worker.py b/python/ray/worker.py index 2369961c6..23fc32ba6 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -25,9 +25,9 @@ import traceback import ray.pickling as pickling import ray.serialization as serialization import ray.services as services -import numbuf -import local_scheduler -import plasma +import ray.numbuf as numbuf +import ray.local_scheduler as local_scheduler +import ray.plasma as plasma SCRIPT_MODE = 0 WORKER_MODE = 1 diff --git a/python/ray/workers/default_worker.py b/python/ray/workers/default_worker.py index c825d407e..c6457357c 100644 --- a/python/ray/workers/default_worker.py +++ b/python/ray/workers/default_worker.py @@ -37,7 +37,7 @@ if __name__ == "__main__": error_explanation = """ This error is unexpected and should not have happened. Somehow a worker crashed in an unanticipated way causing the main_loop to throw an exception, which is -being caught in "lib/python/ray/workers/default_worker.py". +being caught in "python/ray/workers/default_worker.py". """ while True: diff --git a/python/setup.py b/python/setup.py index 0d746cb73..c943ac04d 100644 --- a/python/setup.py +++ b/python/setup.py @@ -19,16 +19,16 @@ class install(_install.install): setup(name="ray", version="0.0.1", packages=find_packages(), - package_data={"core": ["src/common/thirdparty/redis/src/redis-server", - "src/common/redis_module/libray_redis_module.so", - "src/plasma/plasma_store", - "src/plasma/plasma_manager", - "src/plasma/libplasma.so", - "src/local_scheduler/local_scheduler", - "src/local_scheduler/liblocal_scheduler_library.so", - "src/numbuf/libarrow.so", - "src/numbuf/libnumbuf.so", - "src/global_scheduler/global_scheduler"]}, + package_data={"ray": ["core/src/common/thirdparty/redis/src/redis-server", + "core/src/common/redis_module/libray_redis_module.so", + "core/src/plasma/plasma_store", + "core/src/plasma/plasma_manager", + "core/src/plasma/libplasma.so", + "core/src/local_scheduler/local_scheduler", + "core/src/local_scheduler/liblocal_scheduler_library.so", + "core/src/numbuf/libarrow.so", + "core/src/numbuf/libnumbuf.so", + "core/src/global_scheduler/global_scheduler"]}, cmdclass={"install": install}, install_requires=["numpy", "funcsigs", diff --git a/src/local_scheduler/test/local_scheduler_tests.cc b/src/local_scheduler/test/local_scheduler_tests.cc index 0f3773ff3..511125e27 100644 --- a/src/local_scheduler/test/local_scheduler_tests.cc +++ b/src/local_scheduler/test/local_scheduler_tests.cc @@ -74,7 +74,7 @@ LocalSchedulerMock *LocalSchedulerMock_init(int num_workers, UT_string *worker_command; utstring_new(worker_command); utstring_printf(worker_command, - "python ../../python/ray/workers/default_worker.py " + "python ../../../python/ray/workers/default_worker.py " "--node-ip-address=%s --object-store-name=%s " "--object-store-manager-name=%s --local-scheduler-name=%s " "--redis-address=%s:%d", diff --git a/src/numbuf/python/test/runtest.py b/src/numbuf/python/test/runtest.py index 67f59ed08..90080b968 100644 --- a/src/numbuf/python/test/runtest.py +++ b/src/numbuf/python/test/runtest.py @@ -3,7 +3,7 @@ from __future__ import division from __future__ import print_function import unittest -import numbuf +import ray.numbuf as numbuf import numpy as np from numpy.testing import assert_equal import os