diff --git a/.gitignore b/.gitignore index f57b0110e..346c56c5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,7 @@ # The build output should clearly not be checked in -/build -/src/plasma/build -/src/photon/build -/src/common/build +/python/core /src/common/thirdparty/redis -/src/global_scheduler/build -/numbuf/build /numbuf/thirdparty/arrow -/numbuf/dist -/webui/client/public -/webui/node_modules - -# The files copied under lib/python should be ignored -/lib/python/common -/lib/python/global_scheduler -/lib/python/photon -/lib/python/plasma -/lib/python/webui # Files generated by flatcc should be ignored /src/plasma/format/*_builder.h diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index fdeb0c6db..000000000 --- a/.gitmodules +++ /dev/null @@ -1,8 +0,0 @@ -[submodule "src/common/thirdparty/redis"] - path = src/common/thirdparty/redis-windows - url = https://github.com/MSOpenTech/redis.git - ignore = all -[submodule "src/common/thirdparty/python"] - path = src/common/thirdparty/python - url = https://github.com/austinsc/python.git - ignore = all diff --git a/.travis.yml b/.travis.yml index 4e214a81d..cd406694b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,39 +36,39 @@ matrix: - sudo apt-get install -qq valgrind install: - ./.travis/install-dependencies.sh - - - cd src/common - - make valgrind - - cd ../.. - - - cd src/plasma - - make valgrind - - cd ../.. - - - cd src/photon - - make valgrind - - cd ../.. - - ./.travis/install-ray.sh + + - cd python/core + - bash ../../src/common/test/run_valgrind.sh + - bash ../../src/plasma/test/run_valgrind.sh + - bash ../../src/photon/test/run_valgrind.sh + - cd ../.. + script: - - python src/plasma/test/test.py valgrind - - python src/photon/test/test.py valgrind - - python src/global_scheduler/test/test.py valgrind + - python ./python/plasma/test/test.py valgrind + - python ./python/photon/test/test.py valgrind + - python ./python/global_scheduler/test/test.py valgrind install: - ./.travis/install-dependencies.sh - - ./.travis/run-c-tests.sh - ./.travis/install-ray.sh + - cd python/core + - bash ../../src/common/test/run_tests.sh + - bash ../../src/plasma/test/run_tests.sh + - bash ../../src/photon/test/run_tests.sh + - cd ../.. + script: - if [[ "$PYTHON" == "3.5" ]]; then export PATH="$HOME/miniconda/bin:$PATH"; fi - - python numbuf/python/test/runtest.py + - python src/numbuf/python/test/runtest.py - - python src/common/test/test.py - - python src/plasma/test/test.py - - python src/photon/test/test.py - - python src/global_scheduler/test/test.py + - python python/common/test/test.py + - python python/common/redis_module/runtest.py + - python python/plasma/test/test.py + - python python/photon/test/test.py + - python python/global_scheduler/test/test.py - python test/runtest.py - python test/array_test.py @@ -76,11 +76,3 @@ script: - python test/failure_test.py - python test/microbenchmarks.py - python test/stress_tests.py - -after_script: - # Make sure that we can build numbuf as a standalone library. - - pip uninstall -y numbuf - - cd numbuf/build - - rm -rf * - - cmake -DHAS_PLASMA=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g" -DCMAKE_CXX_FLAGS="-g" .. - - make install diff --git a/.travis/install-ray.sh b/.travis/install-ray.sh index 5ce8e52e4..ae520ceba 100755 --- a/.travis/install-ray.sh +++ b/.travis/install-ray.sh @@ -9,36 +9,14 @@ echo "PYTHON is $PYTHON" if [[ "$PYTHON" == "2.7" ]]; then - pushd "$ROOT_DIR/../numbuf" - sudo python setup.py install - popd - - pushd "$ROOT_DIR/../src/common/lib/python" - pushd "$ROOT_DIR/../src/common" - make - popd - sudo python setup.py install - popd - - pushd "$ROOT_DIR/../lib/python" + pushd "$ROOT_DIR/../python" sudo python setup.py install popd elif [[ "$PYTHON" == "3.5" ]]; then export PATH="$HOME/miniconda/bin:$PATH" - pushd "$ROOT_DIR/../numbuf" - python setup.py install --user - popd - - pushd "$ROOT_DIR/../src/common/lib/python" - pushd "$ROOT_DIR/../src/common" - make - popd - python setup.py install --user - popd - - pushd "$ROOT_DIR/../lib/python" + pushd "$ROOT_DIR/../python" python setup.py install --user popd diff --git a/.travis/run-c-tests.sh b/.travis/run-c-tests.sh deleted file mode 100755 index c698919ef..000000000 --- a/.travis/run-c-tests.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Cause the script to exit if a single command fails. -set -e - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -echo "PYTHON is $PYTHON" - -if [[ "$PYTHON" == "3.5" ]]; then export PATH="$HOME/miniconda/bin:$PATH"; fi - -pushd "$ROOT_DIR/../src/common" - make test -popd - -pushd "$ROOT_DIR/../src/plasma" - make test -popd - -pushd "$ROOT_DIR/../src/photon" - make test -popd diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..6b69562aa --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 2.8) + +project(ray) + +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/) +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/) +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/photon/) +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/global_scheduler/) +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/numbuf/) diff --git a/build.sh b/build.sh index 8729bed96..7214c10a3 100755 --- a/build.sh +++ b/build.sh @@ -16,64 +16,16 @@ else exit 1 fi -COMMON_DIR="$ROOT_DIR/src/common" -PLASMA_DIR="$ROOT_DIR/src/plasma" -PHOTON_DIR="$ROOT_DIR/src/photon" -GLOBAL_SCHEDULER_DIR="$ROOT_DIR/src/global_scheduler" +pushd "$ROOT_DIR/src/common/thirdparty/" + bash build-redis.sh +popd -PYTHON_DIR="$ROOT_DIR/lib/python" -PYTHON_COMMON_DIR="$PYTHON_DIR/common" -PYTHON_PLASMA_DIR="$PYTHON_DIR/plasma" -PYTHON_PHOTON_DIR="$PYTHON_DIR/photon" -PYTHON_GLOBAL_SCHEDULER_DIR="$PYTHON_DIR/global_scheduler" - -# First clean up old build files. -pushd "$COMMON_DIR" - make clean -popd -pushd "$PLASMA_DIR" - make clean -popd -pushd "$PHOTON_DIR" - make clean -popd -pushd "$GLOBAL_SCHEDULER_DIR" - make clean -popd +bash "$ROOT_DIR/src/numbuf/thirdparty/download_thirdparty.sh" +bash "$ROOT_DIR/src/numbuf/thirdparty/build_thirdparty.sh" # Now build everything. -pushd "$COMMON_DIR" +pushd "$ROOT_DIR/python/core" + cmake ../.. + make clean make popd -cp "$COMMON_DIR/thirdparty/redis/src/redis-server" "$PYTHON_COMMON_DIR/thirdparty/redis/src/" -cp "$COMMON_DIR/redis_module/ray_redis_module.so" "$PYTHON_COMMON_DIR/redis_module/ray_redis_module.so" - -pushd "$PLASMA_DIR" - make - pushd "$PLASMA_DIR/build" - cmake .. - make install - popd -popd -cp "$PLASMA_DIR/build/plasma_store" "$PYTHON_PLASMA_DIR/" -cp "$PLASMA_DIR/build/plasma_manager" "$PYTHON_PLASMA_DIR/" -cp "$PLASMA_DIR/plasma/plasma.py" "$PYTHON_PLASMA_DIR/" -cp "$PLASMA_DIR/plasma/utils.py" "$PYTHON_PLASMA_DIR/" -cp "$PLASMA_DIR/plasma/libplasma.so" "$PYTHON_PLASMA_DIR/" - -pushd "$PHOTON_DIR" - make - pushd "$PHOTON_DIR/build" - cmake .. - make install - popd -popd -cp "$PHOTON_DIR/build/photon_scheduler" "$PYTHON_PHOTON_DIR/build/" -cp "$PHOTON_DIR/photon/libphoton.so" "$PYTHON_PHOTON_DIR/photon/" -cp "$PHOTON_DIR/photon/photon_services.py" "$PYTHON_PHOTON_DIR/photon/" - -pushd "$GLOBAL_SCHEDULER_DIR" - make -popd -cp "$GLOBAL_SCHEDULER_DIR/build/global_scheduler" "$PYTHON_GLOBAL_SCHEDULER_DIR/build/" -cp "$GLOBAL_SCHEDULER_DIR/lib/python/global_scheduler_services.py" "$PYTHON_GLOBAL_SCHEDULER_DIR/lib/python/" diff --git a/doc/install-on-macosx.md b/doc/install-on-macosx.md index ab333cb7d..10c3f80d8 100644 --- a/doc/install-on-macosx.md +++ b/doc/install-on-macosx.md @@ -13,7 +13,6 @@ brew install cmake automake autoconf libtool boost wget sudo easy_install pip # If you're using Anaconda, then this is unnecessary. pip install numpy cloudpickle funcsigs colorama psutil redis --ignore-installed six -pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=numbuf&subdirectory=numbuf" ``` # Install Ray @@ -26,14 +25,8 @@ cd ray/lib/python python setup.py install --user ``` -Alternatively, Ray can be installed with pip as follows. However, this is -slightly less likely to succeed. - -``` -pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=ray&subdirectory=lib/python" -``` - ## Test if the installation succeeded + To test if the installation was successful, try running some tests. This assumes that you've cloned the git repository. diff --git a/doc/install-on-ubuntu.md b/doc/install-on-ubuntu.md index b5af97e2c..7d46d5c2a 100644 --- a/doc/install-on-ubuntu.md +++ b/doc/install-on-ubuntu.md @@ -13,7 +13,6 @@ sudo apt-get update sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-pip libboost-all-dev unzip # If you're using Anaconda, then python-dev and python-pip are unnecessary. pip install numpy cloudpickle funcsigs colorama psutil redis -pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=numbuf&subdirectory=numbuf" ``` # Install Ray @@ -26,13 +25,6 @@ cd ray/lib/python python setup.py install --user ``` -Alternatively, Ray can be installed with pip as follows. However, this is -slightly less likely to succeed. - -``` -pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=ray&subdirectory=lib/python" -``` - ## Test if the installation succeeded To test if the installation was successful, try running some tests. This assumes diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 310ac5785..ecd8c4a96 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -14,4 +14,3 @@ RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \ ENV PATH "/opt/conda/bin:$PATH" RUN conda install libgcc RUN pip install --upgrade pip -RUN pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=numbuf&subdirectory=numbuf" diff --git a/lib/python/global_scheduler/__init__.py b/lib/python/global_scheduler/__init__.py deleted file mode 100644 index 36b583f9c..000000000 --- a/lib/python/global_scheduler/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from .lib.python.global_scheduler_services import * diff --git a/lib/python/photon/__init__.py b/lib/python/photon/__init__.py deleted file mode 100644 index 08512ce4d..000000000 --- a/lib/python/photon/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .photon import * diff --git a/numbuf/build.sh b/numbuf/build.sh deleted file mode 100755 index 0d3618d61..000000000 --- a/numbuf/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# Cause the script to exit if a single command fails. -set -e - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -# Determine how many parallel jobs to use for make based on the number of cores -unamestr="$(uname)" -if [[ "$unamestr" == "Linux" ]]; then - PARALLEL=$(nproc) -elif [[ "$unamestr" == "Darwin" ]]; then - PARALLEL=$(sysctl -n hw.ncpu) -else - echo "Unrecognized platform." - exit 1 -fi - -# Build plasma. We currently have to do this because we compile numbuf with the -# HAS_PLASMA flag. -PLASMA_DIR="$ROOT_DIR/../src/plasma" -pushd "$PLASMA_DIR" - make clean - make -popd - -mkdir -p "$ROOT_DIR/build" -pushd "$ROOT_DIR/build" - cmake -DHAS_PLASMA=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g" -DCMAKE_CXX_FLAGS="-g" .. - make install -j$PARALLEL -popd diff --git a/numbuf/setup.py b/numbuf/setup.py deleted file mode 100644 index f6f1932ab..000000000 --- a/numbuf/setup.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import subprocess -from setuptools import setup, find_packages -import setuptools.command.install as _install -from sys import platform - -extension = "" -if platform == "linux" or platform == "linux2": - extension = ".so" -elif platform == "darwin": - extension = ".dylib" - -# Because of relative paths, this must be run from inside numbuf/. - -class install(_install.install): - def run(self): - subprocess.check_call(["./setup.sh"]) - subprocess.check_call(["./build.sh"]) - # Calling _install.install.run(self) does not fetch required packages and - # instead performs an old-style install. See command/install.py in - # setuptools. So, calling do_egg_install() manually here. - self.do_egg_install() - -setup(name="numbuf", - version="0.0.1", - packages=find_packages(), - package_data={"numbuf": ["libnumbuf.so", - "libarrow" + extension, - "libarrow_io" + extension, - "libarrow_ipc" + extension]}, - cmdclass={"install": install}, - setup_requires=["numpy"], - include_package_data=True, - zip_safe=False) diff --git a/numbuf/setup.sh b/numbuf/setup.sh deleted file mode 100755 index c807ec84f..000000000 --- a/numbuf/setup.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# Cause the script to exit if a single command fails. -set -e - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -platform="unknown" -unamestr="$(uname)" -if [[ "$unamestr" == "Linux" ]]; then - echo "Platform is linux." - platform="linux" -elif [[ "$unamestr" == "Darwin" ]]; then - echo "Platform is macosx." - platform="macosx" -else - echo "Unrecognized platform." - exit 1 -fi - -pushd "$ROOT_DIR" - ./thirdparty/download_thirdparty.sh - ./thirdparty/build_thirdparty.sh -popd diff --git a/lib/python/common/__init__.py b/python/common/__init__.py similarity index 100% rename from lib/python/common/__init__.py rename to python/common/__init__.py diff --git a/lib/python/common/redis_module/.gitkeep b/python/common/redis_module/.gitkeep similarity index 100% rename from lib/python/common/redis_module/.gitkeep rename to python/common/redis_module/.gitkeep diff --git a/src/common/redis_module/runtest.py b/python/common/redis_module/runtest.py similarity index 99% rename from src/common/redis_module/runtest.py rename to python/common/redis_module/runtest.py index 156bab161..45ed3c4c3 100644 --- a/src/common/redis_module/runtest.py +++ b/python/common/redis_module/runtest.py @@ -11,12 +11,12 @@ import unittest import redis # Check if the redis-server binary is present. -redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../thirdparty/redis/src/redis-server") +redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server") if not os.path.exists(redis_path): raise Exception("You do not have the redis-server binary. Run `make test` in the plasma directory to get it.") # Absolute path of the ray redis module. -module_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "ray_redis_module.so") +module_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/redis_module/libray_redis_module.so") print("path to the redis module is {}".format(module_path)) OBJECT_INFO_PREFIX = "OI:" diff --git a/src/common/test/test.py b/python/common/test/test.py similarity index 85% rename from src/common/test/test.py rename to python/common/test/test.py index d38d1dba5..9a26e8bb4 100644 --- a/src/common/test/test.py +++ b/python/common/test/test.py @@ -7,18 +7,18 @@ import pickle import sys import unittest -import common +import photon ID_SIZE = 20 def random_object_id(): - return common.ObjectID(np.random.bytes(ID_SIZE)) + return photon.ObjectID(np.random.bytes(ID_SIZE)) def random_function_id(): - return common.ObjectID(np.random.bytes(ID_SIZE)) + return photon.ObjectID(np.random.bytes(ID_SIZE)) def random_task_id(): - return common.ObjectID(np.random.bytes(ID_SIZE)) + return photon.ObjectID(np.random.bytes(ID_SIZE)) BASE_SIMPLE_OBJECTS = [ 0, 1, 100000, 0.0, 0.5, 0.9, 100000.1, (), [], {}, @@ -62,9 +62,9 @@ class TestSerialization(unittest.TestCase): def test_serialize_by_value(self): for val in SIMPLE_OBJECTS: - self.assertTrue(common.check_simple_value(val)) + self.assertTrue(photon.check_simple_value(val)) for val in COMPLEX_OBJECTS: - self.assertFalse(common.check_simple_value(val)) + self.assertFalse(photon.check_simple_value(val)) class TestObjectID(unittest.TestCase): @@ -89,17 +89,17 @@ class TestObjectID(unittest.TestCase): self.assertRaises(Exception, lambda : pickling.dumps(h)) def test_equality_comparisons(self): - x1 = common.ObjectID(ID_SIZE * b"a") - x2 = common.ObjectID(ID_SIZE * b"a") - y1 = common.ObjectID(ID_SIZE * b"b") - y2 = common.ObjectID(ID_SIZE * b"b") + x1 = photon.ObjectID(ID_SIZE * b"a") + x2 = photon.ObjectID(ID_SIZE * b"a") + y1 = photon.ObjectID(ID_SIZE * b"b") + y2 = photon.ObjectID(ID_SIZE * b"b") self.assertEqual(x1, x2) self.assertEqual(y1, y2) self.assertNotEqual(x1, y1) random_strings = [np.random.bytes(ID_SIZE) for _ in range(256)] - object_ids1 = [common.ObjectID(random_strings[i]) for i in range(256)] - object_ids2 = [common.ObjectID(random_strings[i]) for i in range(256)] + object_ids1 = [photon.ObjectID(random_strings[i]) for i in range(256)] + object_ids2 = [photon.ObjectID(random_strings[i]) for i in range(256)] self.assertEqual(len(set(object_ids1)), 256) self.assertEqual(len(set(object_ids1 + object_ids2)), 256) self.assertEqual(set(object_ids1), set(object_ids2)) @@ -118,7 +118,7 @@ class TestTask(unittest.TestCase): self.assertEqual(num_return_vals, len(task.returns())) self.assertEqual(len(args), len(retrieved_args)) for i in range(len(retrieved_args)): - if isinstance(retrieved_args[i], common.ObjectID): + if isinstance(retrieved_args[i], photon.ObjectID): self.assertEqual(retrieved_args[i].id(), args[i].id()) else: self.assertEqual(retrieved_args[i], args[i]) @@ -156,10 +156,10 @@ class TestTask(unittest.TestCase): ] for args in args_list: for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: - task = common.Task(function_id, args, num_return_vals, parent_id, 0) + task = photon.Task(function_id, args, num_return_vals, parent_id, 0) self.check_task(task, function_id, num_return_vals, args) - data = common.task_to_string(task) - task2 = common.task_from_string(data) + data = photon.task_to_string(task) + task2 = photon.task_from_string(data) self.check_task(task2, function_id, num_return_vals, args) if __name__ == "__main__": diff --git a/lib/python/common/thirdparty/redis/src/.gitkeep b/python/common/thirdparty/redis/src/.gitkeep similarity index 100% rename from lib/python/common/thirdparty/redis/src/.gitkeep rename to python/common/thirdparty/redis/src/.gitkeep diff --git a/lib/python/global_scheduler/lib/__init__.py b/python/core/__init__.py similarity index 100% rename from lib/python/global_scheduler/lib/__init__.py rename to python/core/__init__.py diff --git a/lib/python/global_scheduler/lib/python/__init__.py b/python/core/src/__init__.py similarity index 100% rename from lib/python/global_scheduler/lib/python/__init__.py rename to python/core/src/__init__.py diff --git a/lib/python/ray/experimental/array/__init__.py b/python/core/src/numbuf/__init__.py similarity index 100% rename from lib/python/ray/experimental/array/__init__.py rename to python/core/src/numbuf/__init__.py diff --git a/lib/python/ray/test/__init__.py b/python/core/src/photon/__init__.py similarity index 100% rename from lib/python/ray/test/__init__.py rename to python/core/src/photon/__init__.py diff --git a/lib/python/ray/workers/__init__.py b/python/core/src/plasma/__init__.py similarity index 100% rename from lib/python/ray/workers/__init__.py rename to python/core/src/plasma/__init__.py diff --git a/src/plasma/plasma/__init__.py b/python/global_scheduler/__init__.py similarity index 72% rename from src/plasma/plasma/__init__.py rename to python/global_scheduler/__init__.py index 6fe09d30f..cde7ce231 100644 --- a/src/plasma/plasma/__init__.py +++ b/python/global_scheduler/__init__.py @@ -2,4 +2,4 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from plasma.plasma import * +from .global_scheduler_services import * diff --git a/lib/python/global_scheduler/build/.gitkeep b/python/global_scheduler/build/.gitkeep similarity index 100% rename from lib/python/global_scheduler/build/.gitkeep rename to python/global_scheduler/build/.gitkeep diff --git a/src/global_scheduler/lib/python/global_scheduler_services.py b/python/global_scheduler/global_scheduler_services.py similarity index 95% rename from src/global_scheduler/lib/python/global_scheduler_services.py rename to python/global_scheduler/global_scheduler_services.py index 505c0da13..d689cbe0c 100644 --- a/src/global_scheduler/lib/python/global_scheduler_services.py +++ b/python/global_scheduler/global_scheduler_services.py @@ -23,7 +23,7 @@ def start_global_scheduler(redis_address, use_valgrind=False, use_profiler=False """ if use_valgrind and use_profiler: raise Exception("Cannot use valgrind and profiler at the same time.") - global_scheduler_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../build/global_scheduler") + global_scheduler_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../core/src/global_scheduler/global_scheduler") command = [global_scheduler_executable, "-r", redis_address] with open(os.devnull, "w") as FNULL: stdout = FNULL if redirect_output else None diff --git a/src/global_scheduler/test/test.py b/python/global_scheduler/test/test.py similarity index 98% rename from src/global_scheduler/test/test.py rename to python/global_scheduler/test/test.py index 508c683d9..1b93871ed 100644 --- a/src/global_scheduler/test/test.py +++ b/python/global_scheduler/test/test.py @@ -46,8 +46,8 @@ class TestGlobalScheduler(unittest.TestCase): def setUp(self): # Start a Redis server. - redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../common/thirdparty/redis/src/redis-server") - redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../common/redis_module/ray_redis_module.so") + redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server") + redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../core/src/common/redis_module/libray_redis_module.so") assert os.path.isfile(redis_path) assert os.path.isfile(redis_module) node_ip_address = "127.0.0.1" diff --git a/python/numbuf/__init__.py b/python/numbuf/__init__.py new file mode 100644 index 000000000..a4d0a82ea --- /dev/null +++ b/python/numbuf/__init__.py @@ -0,0 +1,29 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# See https://github.com/ray-project/ray/issues/131. +helpful_message = """ + +If you are using Anaconda, try fixing this problem by running: + + conda install libgcc +""" + +try: + from core.src.numbuf.libnumbuf import * +except ImportError as e: + if hasattr(e, "msg") and isinstance(e.msg, str) and "GLIBCXX" in e.msg: + # This code path should be taken with Python 3. + e.msg += helpful_message + elif hasattr(e, "message") and isinstance(e.message, str) and "GLIBCXX" in e.message: + # This code path should be taken with Python 2. + if hasattr(e, "args") and isinstance(e.args, tuple) and len(e.args) == 1 and isinstance(e.args[0], str): + e.args = (e.args[0] + helpful_message,) + else: + if not hasattr(e, "args"): + e.args = () + elif not isinstance(e.args, tuple): + e.args = (e.args,) + e.args += (helpful_message,) + raise diff --git a/lib/python/photon/photon/__init__.py b/python/photon/__init__.py similarity index 77% rename from lib/python/photon/photon/__init__.py rename to python/photon/__init__.py index c81d5021f..b2f2a1b11 100644 --- a/lib/python/photon/photon/__init__.py +++ b/python/photon/__init__.py @@ -2,5 +2,5 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from .libphoton import * +from core.src.photon.libphoton import * from .photon_services import * diff --git a/lib/python/photon/build/.gitkeep b/python/photon/build/.gitkeep similarity index 100% rename from lib/python/photon/build/.gitkeep rename to python/photon/build/.gitkeep diff --git a/src/photon/photon/photon_services.py b/python/photon/photon_services.py similarity index 98% rename from src/photon/photon/photon_services.py rename to python/photon/photon_services.py index 5aa629cc2..5452fbdb4 100644 --- a/src/photon/photon/photon_services.py +++ b/python/photon/photon_services.py @@ -40,7 +40,7 @@ def start_local_scheduler(plasma_store_name, plasma_manager_name=None, plasma_ad raise Exception("If one of the plasma_manager_name and the redis_address is provided, then both must be provided.") if use_valgrind and use_profiler: raise Exception("Cannot use valgrind and profiler at the same time.") - local_scheduler_executable = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../build/photon_scheduler") + local_scheduler_executable = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/photon/photon_scheduler") local_scheduler_name = "/tmp/scheduler{}".format(random_name()) command = [local_scheduler_executable, "-s", local_scheduler_name, "-p", plasma_store_name, "-h", node_ip_address] if plasma_manager_name is not None: diff --git a/src/photon/test/test.py b/python/photon/test/test.py similarity index 100% rename from src/photon/test/test.py rename to python/photon/test/test.py diff --git a/lib/python/plasma/.gitkeep b/python/plasma/.gitkeep similarity index 100% rename from lib/python/plasma/.gitkeep rename to python/plasma/.gitkeep diff --git a/lib/python/plasma/__init__.py b/python/plasma/__init__.py similarity index 100% rename from lib/python/plasma/__init__.py rename to python/plasma/__init__.py diff --git a/src/plasma/plasma/plasma.py b/python/plasma/plasma.py similarity index 97% rename from src/plasma/plasma/plasma.py rename to python/plasma/plasma.py index 1dfbe9557..819cebcbe 100644 --- a/src/plasma/plasma/plasma.py +++ b/python/plasma/plasma.py @@ -8,9 +8,9 @@ import subprocess import sys import time -from . import libplasma -from .libplasma import plasma_object_exists_error -from .libplasma import plasma_out_of_memory_error +import core.src.plasma.libplasma as libplasma +from core.src.plasma.libplasma import plasma_object_exists_error +from core.src.plasma.libplasma import plasma_out_of_memory_error PLASMA_ID_SIZE = 20 PLASMA_WAIT_TIMEOUT = 2 ** 30 @@ -280,7 +280,7 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, use_valg """ if use_valgrind and use_profiler: raise Exception("Cannot use valgrind and profiler at the same time.") - plasma_store_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "plasma_store") + plasma_store_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../core/src/plasma/plasma_store") plasma_store_name = "/tmp/plasma_store{}".format(random_name()) command = [plasma_store_executable, "-s", plasma_store_name, "-m", str(plasma_store_memory)] with open(os.devnull, "w") as FNULL: @@ -316,7 +316,7 @@ def start_plasma_manager(store_name, redis_address, node_ip_address="127.0.0.1", Raises: Exception: An exception is raised if the manager could not be started. """ - plasma_manager_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "plasma_manager") + plasma_manager_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../core/src/plasma/plasma_manager") plasma_manager_name = "/tmp/plasma_manager{}".format(random_name()) port = None process = None diff --git a/src/plasma/test/test.py b/python/plasma/test/test.py similarity index 99% rename from src/plasma/test/test.py rename to python/plasma/test/test.py index 041ad0994..7618d571e 100644 --- a/src/plasma/test/test.py +++ b/python/plasma/test/test.py @@ -463,8 +463,8 @@ class TestPlasmaManager(unittest.TestCase): store_name1, self.p2 = plasma.start_plasma_store(use_valgrind=USE_VALGRIND) store_name2, self.p3 = plasma.start_plasma_store(use_valgrind=USE_VALGRIND) # Start a Redis server. - redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../common/thirdparty/redis/src/redis-server") - redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../common/redis_module/ray_redis_module.so") + redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server") + redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../core/src/common/redis_module/libray_redis_module.so") assert os.path.isfile(redis_path) assert os.path.isfile(redis_module) redis_port = 6379 diff --git a/src/plasma/plasma/utils.py b/python/plasma/utils.py similarity index 100% rename from src/plasma/plasma/utils.py rename to python/plasma/utils.py diff --git a/lib/python/ray/__init__.py b/python/ray/__init__.py similarity index 100% rename from lib/python/ray/__init__.py rename to python/ray/__init__.py diff --git a/lib/python/ray/experimental/__init__.py b/python/ray/experimental/__init__.py similarity index 100% rename from lib/python/ray/experimental/__init__.py rename to python/ray/experimental/__init__.py diff --git a/lib/python/webui/.gitkeep b/python/ray/experimental/array/__init__.py similarity index 100% rename from lib/python/webui/.gitkeep rename to python/ray/experimental/array/__init__.py diff --git a/lib/python/ray/experimental/array/distributed/__init__.py b/python/ray/experimental/array/distributed/__init__.py similarity index 100% rename from lib/python/ray/experimental/array/distributed/__init__.py rename to python/ray/experimental/array/distributed/__init__.py diff --git a/lib/python/ray/experimental/array/distributed/core.py b/python/ray/experimental/array/distributed/core.py similarity index 100% rename from lib/python/ray/experimental/array/distributed/core.py rename to python/ray/experimental/array/distributed/core.py diff --git a/lib/python/ray/experimental/array/distributed/linalg.py b/python/ray/experimental/array/distributed/linalg.py similarity index 100% rename from lib/python/ray/experimental/array/distributed/linalg.py rename to python/ray/experimental/array/distributed/linalg.py diff --git a/lib/python/ray/experimental/array/distributed/random.py b/python/ray/experimental/array/distributed/random.py similarity index 100% rename from lib/python/ray/experimental/array/distributed/random.py rename to python/ray/experimental/array/distributed/random.py diff --git a/lib/python/ray/experimental/array/remote/__init__.py b/python/ray/experimental/array/remote/__init__.py similarity index 100% rename from lib/python/ray/experimental/array/remote/__init__.py rename to python/ray/experimental/array/remote/__init__.py diff --git a/lib/python/ray/experimental/array/remote/core.py b/python/ray/experimental/array/remote/core.py similarity index 100% rename from lib/python/ray/experimental/array/remote/core.py rename to python/ray/experimental/array/remote/core.py diff --git a/lib/python/ray/experimental/array/remote/linalg.py b/python/ray/experimental/array/remote/linalg.py similarity index 100% rename from lib/python/ray/experimental/array/remote/linalg.py rename to python/ray/experimental/array/remote/linalg.py diff --git a/lib/python/ray/experimental/array/remote/random.py b/python/ray/experimental/array/remote/random.py similarity index 100% rename from lib/python/ray/experimental/array/remote/random.py rename to python/ray/experimental/array/remote/random.py diff --git a/lib/python/ray/experimental/tfutils.py b/python/ray/experimental/tfutils.py similarity index 100% rename from lib/python/ray/experimental/tfutils.py rename to python/ray/experimental/tfutils.py diff --git a/lib/python/ray/experimental/utils.py b/python/ray/experimental/utils.py similarity index 100% rename from lib/python/ray/experimental/utils.py rename to python/ray/experimental/utils.py diff --git a/lib/python/ray/pickling.py b/python/ray/pickling.py similarity index 100% rename from lib/python/ray/pickling.py rename to python/ray/pickling.py diff --git a/lib/python/ray/serialization.py b/python/ray/serialization.py similarity index 100% rename from lib/python/ray/serialization.py rename to python/ray/serialization.py diff --git a/lib/python/ray/services.py b/python/ray/services.py similarity index 99% rename from lib/python/ray/services.py rename to python/ray/services.py index 1abb24b93..ec76eb8a0 100644 --- a/lib/python/ray/services.py +++ b/python/ray/services.py @@ -154,8 +154,8 @@ def start_redis(node_ip_address, num_retries=20, cleanup=True, redirect_output=F Raises: Exception: An exception is raised if Redis could not be started. """ - redis_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../common/thirdparty/redis/src/redis-server") - redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../common/redis_module/ray_redis_module.so") + redis_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/common/thirdparty/redis/src/redis-server") + redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/common/redis_module/libray_redis_module.so") assert os.path.isfile(redis_filepath) assert os.path.isfile(redis_module) counter = 0 diff --git a/src/plasma/build/.gitkeep b/python/ray/test/__init__.py similarity index 100% rename from src/plasma/build/.gitkeep rename to python/ray/test/__init__.py diff --git a/lib/python/ray/test/test_functions.py b/python/ray/test/test_functions.py similarity index 100% rename from lib/python/ray/test/test_functions.py rename to python/ray/test/test_functions.py diff --git a/lib/python/ray/worker.py b/python/ray/worker.py similarity index 100% rename from lib/python/ray/worker.py rename to python/ray/worker.py diff --git a/python/ray/workers/__init__.py b/python/ray/workers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/python/ray/workers/default_worker.py b/python/ray/workers/default_worker.py similarity index 100% rename from lib/python/ray/workers/default_worker.py rename to python/ray/workers/default_worker.py diff --git a/lib/python/setup.py b/python/setup.py similarity index 58% rename from lib/python/setup.py rename to python/setup.py index 08ab1b10c..42640d441 100644 --- a/lib/python/setup.py +++ b/python/setup.py @@ -10,7 +10,7 @@ import setuptools.command.install as _install class install(_install.install): def run(self): - subprocess.check_call(["../../build.sh"]) + subprocess.check_call(["../build.sh"]) # Calling _install.install.run(self) does not fetch required packages and # instead performs an old-style install. See command/install.py in # setuptools. So, calling do_egg_install() manually here. @@ -19,14 +19,16 @@ class install(_install.install): setup(name="ray", version="0.0.1", packages=find_packages(), - package_data={"common": ["thirdparty/redis/src/redis-server", - "redis_module/ray_redis_module.so"], - "plasma": ["plasma_store", - "plasma_manager", - "libplasma.so"], - "photon": ["build/photon_scheduler", - "photon/libphoton.so"], - "global_scheduler": ["build/global_scheduler"]}, + package_data={"core": ["src/common/thirdparty/redis/src/redis-server", + "src/common/redis_module/libray_redis_module.so", + "src/plasma/plasma_store", + "src/plasma/plasma_manager", + "src/plasma/libplasma.so", + "src/photon/photon_scheduler", + "src/photon/libphoton.so", + "src/numbuf/libarrow.so", + "src/numbuf/libnumbuf.so", + "src/global_scheduler/global_scheduler"]}, cmdclass={"install": install}, install_requires=["numpy", "funcsigs", diff --git a/python/webui/.gitkeep b/python/webui/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index eb47576cc..a03fb42da 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -2,15 +2,60 @@ cmake_minimum_required(VERSION 2.8) project(common) -include(ExternalProject) +include(${CMAKE_CURRENT_LIST_DIR}/cmake/Common.cmake) -set(FLATCC_PREFIX "${CMAKE_CURRENT_LIST_DIR}/build/flatcc-prefix/src/flatcc") +add_subdirectory(redis_module) -ExternalProject_Add(flatcc - URL "https://github.com/dvidelabs/flatcc/archive/v0.4.0.tar.gz" - INSTALL_COMMAND "" - CMAKE_ARGS "-DCMAKE_C_FLAGS=-fPIC") +include_directories(thirdparty/ae) -set(FLATBUFFERS_INCLUDE_DIR "${FLATCC_PREFIX}/include") -set(FLATBUFFERS_STATIC_LIB "${FLATCC_PREFIX}/lib/libflatcc.a") -set(FLATBUFFERS_COMPILER "${FLATCC_PREFIX}/bin/flatcc") +add_custom_target( + hiredis + COMMAND make + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis) + +add_library(common STATIC + event_loop.c + common.c + task.c + io.c + net.c + logging.c + state/redis.c + state/table.c + state/object_table.c + state/task_table.c + state/db_client_table.c + state/local_scheduler_table.c + thirdparty/ae/ae.c + thirdparty/sha256.c) + +target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhiredis.a") + +function(define_test test_name library) + add_executable(${test_name} test/${test_name}.c ${ARGN}) + add_dependencies(${test_name} hiredis flatcc) + target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ${library}) + target_compile_options(${test_name} PUBLIC "-DPLASMA_TEST -DPHOTON_TEST -DCOMMON_TEST -DRAY_COMMON_LOG_LEVEL=4 -DRAY_TIMEOUT=50") +endfunction() + +define_test(common_tests "") +define_test(db_tests "") +define_test(io_tests "") +define_test(task_tests "") +define_test(redis_tests "") +define_test(task_table_tests "") +define_test(object_table_tests "") + +add_custom_target( + redis + COMMAND make + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/thirdparty/redis) + +add_custom_target(copy_redis ALL) +foreach(file "redis-cli" "redis-server") +add_custom_command(TARGET copy_redis POST_BUILD + COMMAND ${CMAKE_COMMAND} -E + copy ${CMAKE_CURRENT_LIST_DIR}/thirdparty/redis/src/${file} + ${CMAKE_BINARY_DIR}/src/common/thirdparty/redis/src/${file}) +endforeach() +add_dependencies(copy_redis redis) diff --git a/src/common/Makefile b/src/common/Makefile deleted file mode 100644 index 9087a6faf..000000000 --- a/src/common/Makefile +++ /dev/null @@ -1,71 +0,0 @@ -CC = gcc -CFLAGS = -g -Wall -Wextra -Werror=implicit-function-declaration -Wno-typedef-redefinition -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -fPIC -I. -Ithirdparty -Ithirdparty/ae -BUILD = build - -all: hiredis redis redismodule $(BUILD)/libcommon.a - -$(BUILD)/libcommon.a: event_loop.o common.o task.o io.o net.o logging.o state/redis.o state/table.o state/object_table.o state/task_table.o state/db_client_table.o state/local_scheduler_table.o thirdparty/ae/ae.o thirdparty/sha256.o - ar rcs $@ $^ - -$(BUILD)/common_tests: test/common_tests.c $(BUILD)/libcommon.a - $(CC) -o $@ test/common_tests.c $(BUILD)/libcommon.a $(CFLAGS) - -$(BUILD)/db_tests: hiredis test/db_tests.c $(BUILD)/libcommon.a - $(CC) -o $@ test/db_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS) - -$(BUILD)/object_table_tests: hiredis test/object_table_tests.c $(BUILD)/libcommon.a - $(CC) -o $@ test/object_table_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS) - -$(BUILD)/task_table_tests: hiredis test/task_table_tests.c $(BUILD)/libcommon.a - $(CC) -o $@ test/task_table_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS) - -$(BUILD)/io_tests: test/io_tests.c $(BUILD)/libcommon.a - $(CC) -o $@ $^ $(CFLAGS) - -$(BUILD)/task_tests: test/task_tests.c $(BUILD)/libcommon.a - $(CC) -o $@ $^ thirdparty/hiredis/libhiredis.a $(CFLAGS) - -$(BUILD)/redis_tests: hiredis test/redis_tests.c $(BUILD)/libcommon.a logging.h - $(CC) -o $@ test/redis_tests.c logging.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS) - -clean: - rm -f *.o state/*.o test/*.o thirdparty/ae/*.o - rm -rf $(BUILD)/* - cd redis_module; make clean - -redis: - cd thirdparty ; bash ./build-redis.sh - -hiredis: - cd thirdparty/hiredis ; make - -redismodule: - cd redis_module && make && cd .. - -test: CFLAGS += -DRAY_COMMON_LOG_LEVEL=4 -test: hiredis redis redismodule $(BUILD)/common_tests $(BUILD)/task_table_tests $(BUILD)/object_table_tests $(BUILD)/db_tests $(BUILD)/io_tests $(BUILD)/task_tests $(BUILD)/redis_tests FORCE - ./thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./redis_module/ray_redis_module.so & - sleep 1s - ./build/common_tests - ./build/db_tests - ./build/io_tests - ./build/task_tests - ./build/redis_tests - ./build/task_table_tests - ./build/object_table_tests - ./thirdparty/redis/src/redis-cli shutdown - python ./redis_module/runtest.py - -valgrind: test - ./thirdparty/redis/src/redis-server --loglevel warning --loadmodule redis_module/ray_redis_module.so & - sleep 1s - valgrind --leak-check=full --error-exitcode=1 ./build/common_tests - valgrind --leak-check=full --error-exitcode=1 ./build/db_tests - valgrind --leak-check=full --error-exitcode=1 ./build/io_tests - valgrind --leak-check=full --error-exitcode=1 ./build/task_tests - valgrind --leak-check=full --error-exitcode=1 ./build/redis_tests - valgrind --leak-check=full --error-exitcode=1 ./build/task_table_tests - valgrind --leak-check=full --error-exitcode=1 ./build/object_table_tests - ./thirdparty/redis/src/redis-cli shutdown - -FORCE: diff --git a/numbuf/CMakeLists.txt b/src/common/cmake/Common.cmake similarity index 50% rename from numbuf/CMakeLists.txt rename to src/common/cmake/Common.cmake index 67ebe4ec1..a85d228c2 100644 --- a/numbuf/CMakeLists.txt +++ b/src/common/cmake/Common.cmake @@ -1,17 +1,28 @@ -cmake_minimum_required(VERSION 2.8) +# Code for compiling flatbuffers -project(numbuf) +include(ExternalProject) -list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) +set(FLATCC_PREFIX "${CMAKE_BINARY_DIR}/src/common/flatcc-prefix/src/flatcc") -option(HAS_PLASMA - "Are we linking with the plasma object store? Recommended if numbuf is used as part of ray." - OFF) - -if(HAS_PLASMA) - add_definitions(-DHAS_PLASMA) +if (NOT TARGET flatcc) + ExternalProject_Add(flatcc + URL "https://github.com/dvidelabs/flatcc/archive/v0.4.0.tar.gz" + INSTALL_COMMAND "" + CMAKE_ARGS "-DCMAKE_C_FLAGS=-fPIC") endif() +set(FLATBUFFERS_INCLUDE_DIR "${FLATCC_PREFIX}/include") +set(FLATBUFFERS_STATIC_LIB "${FLATCC_PREFIX}/lib/libflatcc.a") +set(FLATBUFFERS_COMPILER "${FLATCC_PREFIX}/bin/flatcc") + +include_directories("${FLATBUFFERS_INCLUDE_DIR}") + +# Custom CFLAGS + +set(CMAKE_C_FLAGS "-g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -fPIC -std=c99") + +# Code for finding Python + message(STATUS "Trying custom approach for finding Python.") # Start off by figuring out which Python executable to use. find_program(CUSTOM_PYTHON_EXECUTABLE python) @@ -65,68 +76,11 @@ message(STATUS "Using CUSTOM_PYTHON_EXECUTABLE: " ${CUSTOM_PYTHON_EXECUTABLE}) message(STATUS "Using PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES}) message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS}) -find_package(NumPy REQUIRED) +# Common libraries -if(APPLE) - SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif(APPLE) +set(COMMON_LIB "${CMAKE_BINARY_DIR}/src/common/libcommon.a" + CACHE STRING "Path to libcommon.a") -include_directories("${PYTHON_INCLUDE_DIRS}") -include_directories("${NUMPY_INCLUDE_DIR}") - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") - -if(UNIX AND NOT APPLE) - link_libraries(rt) -endif() - -set(ARROW_DIR "${CMAKE_SOURCE_DIR}/thirdparty/arrow/" - CACHE STRING "Path of the arrow source directory") - -set(ARROW_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a" - CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)") -set(ARROW_IO_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_io.a" - CACHE STRING "Path to libarrow_io.a (needs to be changed if arrow is build in debug mode)") -set(ARROW_IPC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_ipc.a" - CACHE STRING "Path to libarrow_ipc.a (needs to be changed if arrow is build in debug mode)") - -include_directories("${ARROW_DIR}/cpp/src/") -include_directories("cpp/src/") -include_directories("python/src/") - -if(HAS_PLASMA) - include_directories("../src/plasma") - include_directories("../src/common") - include_directories("../src/common/thirdparty") - include_directories("../src/common/build/flatcc-prefix/src/flatcc/include") -endif() - -add_definitions(-fPIC) - -add_library(numbuf SHARED - cpp/src/numbuf/tensor.cc - cpp/src/numbuf/dict.cc - cpp/src/numbuf/sequence.cc - python/src/pynumbuf/numbuf.cc - python/src/pynumbuf/adapters/numpy.cc - python/src/pynumbuf/adapters/python.cc) - -get_filename_component(PYTHON_SHARED_LIBRARY ${PYTHON_LIBRARIES} NAME) -if(APPLE) - add_custom_command(TARGET numbuf - POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libnumbuf.so) -endif(APPLE) - -if(APPLE) - target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES}) -else() - target_link_libraries(numbuf -Wl,--whole-archive ${ARROW_LIB} -Wl,--no-whole-archive ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES}) -endif() - -if(HAS_PLASMA) - target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES} "${CMAKE_SOURCE_DIR}/../src/plasma/build/libplasma_client.a" "${CMAKE_SOURCE_DIR}/../src/common/build/libcommon.a" "${CMAKE_SOURCE_DIR}/../src/common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a") -else() - target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES}) -endif() - -install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/) +include_directories("${CMAKE_CURRENT_LIST_DIR}/..") +include_directories("${CMAKE_CURRENT_LIST_DIR}/../thirdparty/") +include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python") diff --git a/src/common/lib/python/common_module.c b/src/common/lib/python/common_module.c deleted file mode 100644 index 3b204f4a2..000000000 --- a/src/common/lib/python/common_module.c +++ /dev/null @@ -1,82 +0,0 @@ -#include - -#include "common_extension.h" - -static PyMethodDef common_methods[] = { - {"check_simple_value", check_simple_value, METH_VARARGS, - "Should the object be passed by value?"}, - {"compute_put_id", compute_put_id, METH_VARARGS, - "Return the object ID for a put call within a task."}, - {"task_from_string", PyTask_from_string, METH_VARARGS, - "Creates a Python PyTask object from a string representation of " - "task_spec."}, - {"task_to_string", PyTask_to_string, METH_VARARGS, - "Translates a PyTask python object to a byte string."}, - {NULL} /* Sentinel */ -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "common", /* m_name */ - "A module for common types. This is used for testing.", /* m_doc */ - 0, /* m_size */ - common_methods, /* m_methods */ - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL, /* m_free */ -}; -#endif - -#if PY_MAJOR_VERSION >= 3 -#define INITERROR return NULL -#else -#define INITERROR return -#endif - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif - -#if PY_MAJOR_VERSION >= 3 -#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) -#else -#define MOD_INIT(name) PyMODINIT_FUNC init##name(void) -#endif - -MOD_INIT(common) { - PyObject *m; - - if (PyType_Ready(&PyTaskType) < 0) { - INITERROR; - } - - if (PyType_Ready(&PyObjectIDType) < 0) { - INITERROR; - } - -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("common", common_methods, - "A module for common types. This is used for testing."); -#endif - - init_pickle_module(); - - Py_INCREF(&PyTaskType); - PyModule_AddObject(m, "Task", (PyObject *) &PyTaskType); - - Py_INCREF(&PyObjectIDType); - PyModule_AddObject(m, "ObjectID", (PyObject *) &PyObjectIDType); - - char common_error[] = "common.error"; - CommonError = PyErr_NewException(common_error, NULL, NULL); - Py_INCREF(CommonError); - PyModule_AddObject(m, "common_error", CommonError); - -#if PY_MAJOR_VERSION >= 3 - return m; -#endif -} diff --git a/src/common/lib/python/setup.py b/src/common/lib/python/setup.py deleted file mode 100644 index db861593b..000000000 --- a/src/common/lib/python/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import setup, find_packages, Extension - -common_module = Extension("common", - sources=["common_module.c", "common_extension.c"], - include_dirs=["../../", "../../thirdparty"], - extra_objects=["../../build/libcommon.a"], - extra_compile_args=["--std=c99", "-Werror"]) - -setup(name="Common", - version="0.0.1", - description="Common library for Ray", - ext_modules=[common_module]) diff --git a/src/common/redis_module/CMakeLists.txt b/src/common/redis_module/CMakeLists.txt new file mode 100644 index 000000000..2b9fe7c33 --- /dev/null +++ b/src/common/redis_module/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 2.8) + +project(ray_redis_module) + +if(APPLE) + set(REDIS_MODULE_CFLAGS -W -Wall -dynamic -fno-common -g -ggdb -std=c99 -O2) + set(REDIS_MODULE_LDFLAGS "-undefined dynamic_lookup") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +else() + set(REDIS_MODULE_CFLAGS -W -Wall -fno-common -g -ggdb -std=c99 -O2) + set(REDIS_MODULE_LDFLAGS -shared) +endif() + +add_library(ray_redis_module SHARED ray_redis_module.c) + +target_compile_options(ray_redis_module PUBLIC ${REDIS_MODULE_CFLAGS} -fPIC) +target_link_libraries(ray_redis_module ${REDIS_MODULE_LDFLAGS}) diff --git a/src/common/redis_module/Makefile b/src/common/redis_module/Makefile deleted file mode 100644 index 61e2332af..000000000 --- a/src/common/redis_module/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -# find the OS -uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') - -# Compile flags for linux / osx -ifeq ($(uname_S),Linux) - SHOBJ_CFLAGS ?= -W -Wall -fno-common -g -ggdb -std=c99 -O2 - SHOBJ_LDFLAGS ?= -shared -else - SHOBJ_CFLAGS ?= -W -Wall -dynamic -fno-common -g -ggdb -std=c99 -O2 - SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup -endif - -SHOBJ_CFLAGS += -I../thirdparty - -.SUFFIXES: .c .so .xo .o - -all: ray_redis_module.so - -.c.xo: - $(CC) -I. -I.. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ - -ray_redis_module.xo: redismodule.h - -ray_redis_module.so: ray_redis_module.xo - $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc - -clean: - rm -rf *.xo *.so diff --git a/src/common/test/run_tests.sh b/src/common/test/run_tests.sh new file mode 100644 index 000000000..4fd2f1d4c --- /dev/null +++ b/src/common/test/run_tests.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# This needs to be run in the build tree, which is normally ray/build + +# Cause the script to exit if a single command fails. +set -e + +./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so & +sleep 1s +./src/common/common_tests +./src/common/db_tests +./src/common/io_tests +./src/common/task_tests +./src/common/redis_tests +./src/common/task_table_tests +./src/common/object_table_tests +./src/common/thirdparty/redis/src/redis-cli shutdown diff --git a/src/common/test/run_valgrind.sh b/src/common/test/run_valgrind.sh new file mode 100644 index 000000000..51edf5f5b --- /dev/null +++ b/src/common/test/run_valgrind.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# This needs to be run in the build tree, which is normally ray/build + +# Cause the script to exit if a single command fails. +set -e + +./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so & +sleep 1s +valgrind --leak-check=full --error-exitcode=1 ./src/common/common_tests +valgrind --leak-check=full --error-exitcode=1 ./src/common/db_tests +valgrind --leak-check=full --error-exitcode=1 ./src/common/io_tests +valgrind --leak-check=full --error-exitcode=1 ./src/common/task_tests +valgrind --leak-check=full --error-exitcode=1 ./src/common/redis_tests +valgrind --leak-check=full --error-exitcode=1 ./src/common/task_table_tests +valgrind --leak-check=full --error-exitcode=1 ./src/common/object_table_tests +./src/common/thirdparty/redis/src/redis-cli shutdown diff --git a/src/global_scheduler/CMakeLists.txt b/src/global_scheduler/CMakeLists.txt new file mode 100644 index 000000000..f4a2e8496 --- /dev/null +++ b/src/global_scheduler/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 2.8) + +project(global_scheduler) + +include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake) + +add_executable(global_scheduler global_scheduler.c global_scheduler_algorithm.c) +target_link_libraries(global_scheduler common ${HIREDIS_LIB}) diff --git a/src/global_scheduler/Makefile b/src/global_scheduler/Makefile deleted file mode 100644 index 890fd1094..000000000 --- a/src/global_scheduler/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -CC = gcc -CFLAGS = -g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -I. -I../common -I../common/thirdparty -I../common/state -BUILD = build - -all: $(BUILD)/global_scheduler - -$(BUILD)/global_scheduler: global_scheduler.c global_scheduler_algorithm.c - $(CC) $(CFLAGS) -o $@ global_scheduler.c global_scheduler_algorithm.c ../common/build/libcommon.a ../common/thirdparty/hiredis/libhiredis.a - -clean: - rm -f *.o - rm -rf $(BUILD)/* diff --git a/src/global_scheduler/global_scheduler_algorithm.c b/src/global_scheduler/global_scheduler_algorithm.c index 5c6f09c83..97cd5c3a2 100644 --- a/src/global_scheduler/global_scheduler_algorithm.c +++ b/src/global_scheduler/global_scheduler_algorithm.c @@ -2,7 +2,7 @@ #include "object_info.h" #include "task.h" -#include "task_table.h" +#include "state/task_table.h" #include "global_scheduler_algorithm.h" diff --git a/numbuf/.clang-format b/src/numbuf/.clang-format similarity index 100% rename from numbuf/.clang-format rename to src/numbuf/.clang-format diff --git a/src/numbuf/CMakeLists.txt b/src/numbuf/CMakeLists.txt new file mode 100644 index 000000000..a7e6bf5fb --- /dev/null +++ b/src/numbuf/CMakeLists.txt @@ -0,0 +1,81 @@ +cmake_minimum_required(VERSION 2.8) + +project(numbuf) + +include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake) + +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) + +option(HAS_PLASMA + "Are we linking with the plasma object store? Recommended if numbuf is used as part of ray." + ON) + +if(HAS_PLASMA) + add_definitions(-DHAS_PLASMA) +endif() + +find_package(NumPy REQUIRED) + +if(APPLE) + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif(APPLE) + +include_directories("${PYTHON_INCLUDE_DIRS}") +include_directories("${NUMPY_INCLUDE_DIR}") + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + +if(UNIX AND NOT APPLE) + link_libraries(rt) +endif() + +set(ARROW_DIR "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/" + CACHE STRING "Path of the arrow source directory") + +set(ARROW_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a" + CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)") +set(ARROW_IO_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow_io.a" + CACHE STRING "Path to libarrow_io.a (needs to be changed if arrow is build in debug mode)") +set(ARROW_IPC_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow_ipc.a" + CACHE STRING "Path to libarrow_ipc.a (needs to be changed if arrow is build in debug mode)") + +include_directories("${ARROW_DIR}/cpp/src/") +include_directories("cpp/src/") +include_directories("python/src/") + +if(HAS_PLASMA) + include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma") + include_directories("${CMAKE_CURRENT_LIST_DIR}/../common") + include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty") + include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/build/flatcc-prefix/src/flatcc/include") +endif() + +add_definitions(-fPIC) + +add_library(numbuf SHARED + cpp/src/numbuf/tensor.cc + cpp/src/numbuf/dict.cc + cpp/src/numbuf/sequence.cc + python/src/pynumbuf/numbuf.cc + python/src/pynumbuf/adapters/numpy.cc + python/src/pynumbuf/adapters/python.cc) + +get_filename_component(PYTHON_SHARED_LIBRARY ${PYTHON_LIBRARIES} NAME) +if(APPLE) + add_custom_command(TARGET numbuf + POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libnumbuf.so) +endif(APPLE) + +if(APPLE) + target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES}) +else() + target_link_libraries(numbuf -Wl,--whole-archive ${ARROW_LIB} -Wl,--no-whole-archive ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES}) +endif() + +if(HAS_PLASMA) + target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES} plasma_lib common ${FLATBUFFERS_STATIC_LIB}) +else() + target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES}) +endif() + +install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/) diff --git a/numbuf/cmake/Modules/FindNumPy.cmake b/src/numbuf/cmake/Modules/FindNumPy.cmake similarity index 100% rename from numbuf/cmake/Modules/FindNumPy.cmake rename to src/numbuf/cmake/Modules/FindNumPy.cmake diff --git a/numbuf/cpp/src/numbuf/dict.cc b/src/numbuf/cpp/src/numbuf/dict.cc similarity index 100% rename from numbuf/cpp/src/numbuf/dict.cc rename to src/numbuf/cpp/src/numbuf/dict.cc diff --git a/numbuf/cpp/src/numbuf/dict.h b/src/numbuf/cpp/src/numbuf/dict.h similarity index 100% rename from numbuf/cpp/src/numbuf/dict.h rename to src/numbuf/cpp/src/numbuf/dict.h diff --git a/numbuf/cpp/src/numbuf/sequence.cc b/src/numbuf/cpp/src/numbuf/sequence.cc similarity index 100% rename from numbuf/cpp/src/numbuf/sequence.cc rename to src/numbuf/cpp/src/numbuf/sequence.cc diff --git a/numbuf/cpp/src/numbuf/sequence.h b/src/numbuf/cpp/src/numbuf/sequence.h similarity index 100% rename from numbuf/cpp/src/numbuf/sequence.h rename to src/numbuf/cpp/src/numbuf/sequence.h diff --git a/numbuf/cpp/src/numbuf/tensor.cc b/src/numbuf/cpp/src/numbuf/tensor.cc similarity index 100% rename from numbuf/cpp/src/numbuf/tensor.cc rename to src/numbuf/cpp/src/numbuf/tensor.cc diff --git a/numbuf/cpp/src/numbuf/tensor.h b/src/numbuf/cpp/src/numbuf/tensor.h similarity index 100% rename from numbuf/cpp/src/numbuf/tensor.h rename to src/numbuf/cpp/src/numbuf/tensor.h diff --git a/numbuf/numbuf/__init__.py b/src/numbuf/numbuf/__init__.py similarity index 96% rename from numbuf/numbuf/__init__.py rename to src/numbuf/numbuf/__init__.py index 41cefb779..a692edb79 100644 --- a/numbuf/numbuf/__init__.py +++ b/src/numbuf/numbuf/__init__.py @@ -11,7 +11,7 @@ If you are using Anaconda, try fixing this problem by running: """ try: - from numbuf.libnumbuf import * + from .libnumbuf import * except ImportError as e: if hasattr(e, "msg") and isinstance(e.msg, str) and "GLIBCXX" in e.msg: # This code path should be taken with Python 3. diff --git a/numbuf/python/src/pynumbuf/adapters/numpy.cc b/src/numbuf/python/src/pynumbuf/adapters/numpy.cc similarity index 100% rename from numbuf/python/src/pynumbuf/adapters/numpy.cc rename to src/numbuf/python/src/pynumbuf/adapters/numpy.cc diff --git a/numbuf/python/src/pynumbuf/adapters/numpy.h b/src/numbuf/python/src/pynumbuf/adapters/numpy.h similarity index 100% rename from numbuf/python/src/pynumbuf/adapters/numpy.h rename to src/numbuf/python/src/pynumbuf/adapters/numpy.h diff --git a/numbuf/python/src/pynumbuf/adapters/python.cc b/src/numbuf/python/src/pynumbuf/adapters/python.cc similarity index 100% rename from numbuf/python/src/pynumbuf/adapters/python.cc rename to src/numbuf/python/src/pynumbuf/adapters/python.cc diff --git a/numbuf/python/src/pynumbuf/adapters/python.h b/src/numbuf/python/src/pynumbuf/adapters/python.h similarity index 100% rename from numbuf/python/src/pynumbuf/adapters/python.h rename to src/numbuf/python/src/pynumbuf/adapters/python.h diff --git a/numbuf/python/src/pynumbuf/adapters/scalars.h b/src/numbuf/python/src/pynumbuf/adapters/scalars.h similarity index 100% rename from numbuf/python/src/pynumbuf/adapters/scalars.h rename to src/numbuf/python/src/pynumbuf/adapters/scalars.h diff --git a/numbuf/python/src/pynumbuf/memory.h b/src/numbuf/python/src/pynumbuf/memory.h similarity index 100% rename from numbuf/python/src/pynumbuf/memory.h rename to src/numbuf/python/src/pynumbuf/memory.h diff --git a/numbuf/python/src/pynumbuf/numbuf.cc b/src/numbuf/python/src/pynumbuf/numbuf.cc similarity index 100% rename from numbuf/python/src/pynumbuf/numbuf.cc rename to src/numbuf/python/src/pynumbuf/numbuf.cc diff --git a/numbuf/python/test/runtest.py b/src/numbuf/python/test/runtest.py similarity index 100% rename from numbuf/python/test/runtest.py rename to src/numbuf/python/test/runtest.py diff --git a/numbuf/thirdparty/build_thirdparty.sh b/src/numbuf/thirdparty/build_thirdparty.sh similarity index 100% rename from numbuf/thirdparty/build_thirdparty.sh rename to src/numbuf/thirdparty/build_thirdparty.sh diff --git a/numbuf/thirdparty/download_thirdparty.sh b/src/numbuf/thirdparty/download_thirdparty.sh similarity index 100% rename from numbuf/thirdparty/download_thirdparty.sh rename to src/numbuf/thirdparty/download_thirdparty.sh diff --git a/numbuf/vsprojects/numbuf.vcxproj b/src/numbuf/vsprojects/numbuf.vcxproj similarity index 100% rename from numbuf/vsprojects/numbuf.vcxproj rename to src/numbuf/vsprojects/numbuf.vcxproj diff --git a/numbuf/vsprojects/numbuf.vcxproj.filters b/src/numbuf/vsprojects/numbuf.vcxproj.filters similarity index 100% rename from numbuf/vsprojects/numbuf.vcxproj.filters rename to src/numbuf/vsprojects/numbuf.vcxproj.filters diff --git a/src/photon/CMakeLists.txt b/src/photon/CMakeLists.txt index f7141baf8..ac4c0fd10 100644 --- a/src/photon/CMakeLists.txt +++ b/src/photon/CMakeLists.txt @@ -2,58 +2,8 @@ cmake_minimum_required(VERSION 2.8) project(photon) -message(STATUS "Trying custom approach for finding Python.") -# Start off by figuring out which Python executable to use. -find_program(CUSTOM_PYTHON_EXECUTABLE python) -message(STATUS "Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}") -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print('python' + sys.version[0:3])" - OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) -message(STATUS "PYTHON_LIBRARY_NAME: " ${PYTHON_LIBRARY_NAME}) -# Now find the Python include directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())" - OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE) -message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS}) -# Now find the Python libraries. We'll start by looking near the Python -# executable. If that fails, then we'll look near the Python include -# directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print(sys.exec_prefix)" - OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) -message(STATUS "PYTHON_PREFIX: " ${PYTHON_PREFIX}) -# The name ending in "m" is for miniconda. -FIND_LIBRARY(PYTHON_LIBRARIES - NAMES "${PYTHON_LIBRARY_NAME}" "${PYTHON_LIBRARY_NAME}m" - HINTS "${PYTHON_PREFIX}" - PATH_SUFFIXES "lib" "libs" - NO_DEFAULT_PATH) -message(STATUS "PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES}) -# If that failed, perhaps because the user is in a virtualenv, search around -# the Python include directories. -if(NOT PYTHON_LIBRARIES) - message(STATUS "Failed to find PYTHON_LIBRARIES near the Python executable, so now looking near the Python include directories.") - # The name ending in "m" is for miniconda. - FIND_LIBRARY(PYTHON_LIBRARIES - NAMES "${PYTHON_LIBRARY_NAME}" "${PYTHON_LIBRARY_NAME}m" - HINTS "${PYTHON_INCLUDE_DIRS}/../.." - PATH_SUFFIXES "lib" "libs" - NO_DEFAULT_PATH) - message(STATUS "PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES}) -endif() -# If we found the Python libraries and the include directories, then continue -# on. If not, then try find_package as a last resort, but it probably won't -# work. -if(PYTHON_LIBRARIES AND PYTHON_INCLUDE_DIRS) - message(STATUS "The custom approach for finding Python succeeded.") - SET(PYTHONLIBS_FOUND TRUE) -else() - message(WARNING "The custom approach for finding Python failed. Defaulting to find_package.") - find_package(PythonInterp REQUIRED) - find_package(PythonLibs ${PYTHON_VERSION_STRING} EXACT REQUIRED) - set(CUSTOM_PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}) -endif() - -message(STATUS "Using CUSTOM_PYTHON_EXECUTABLE: " ${CUSTOM_PYTHON_EXECUTABLE}) -message(STATUS "Using PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES}) -message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS}) +# Recursively include common +include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake) if(APPLE) SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") @@ -61,24 +11,15 @@ endif(APPLE) include_directories("${PYTHON_INCLUDE_DIRS}") -set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} --std=c99 -Werror") +# set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} --std=c99 -Werror") if(UNIX AND NOT APPLE) link_libraries(rt) endif() -set(PHOTON_CLIENT_LIB "${CMAKE_SOURCE_DIR}/build/photon_client.a" - CACHE STRING "Path to photon_client.a") - -set(COMMON_LIB "${CMAKE_SOURCE_DIR}/../common/build/libcommon.a" - CACHE STRING "Path to libcommon.a") - -include_directories("${CMAKE_SOURCE_DIR}/") -include_directories("${CMAKE_SOURCE_DIR}/../") -include_directories("${CMAKE_SOURCE_DIR}/../common/") -include_directories("${CMAKE_SOURCE_DIR}/../common/thirdparty/") -include_directories("${CMAKE_SOURCE_DIR}/../common/lib/python/") -include_directories("${CMAKE_SOURCE_DIR}/../plasma/") +include_directories("${CMAKE_CURRENT_LIST_DIR}/") +include_directories("${CMAKE_CURRENT_LIST_DIR}/../") +include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/") add_library(photon SHARED photon_extension.c @@ -90,6 +31,15 @@ if(APPLE) POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libphoton.so) endif(APPLE) -target_link_libraries(photon ${PHOTON_CLIENT_LIB} ${COMMON_LIB} ${PYTHON_LIBRARIES}) +add_library(photon_client STATIC photon_client.c) + +target_link_libraries(photon photon_client ${COMMON_LIB} ${PYTHON_LIBRARIES}) + +add_executable(photon_scheduler photon_scheduler.c photon_algorithm.c) +target_link_libraries(photon_scheduler photon_client common ${HIREDIS_LIB} plasma_lib) + +add_executable(photon_tests test/photon_tests.c photon_scheduler.c photon_algorithm.c ) +target_link_libraries(photon_tests photon_client common ${HIREDIS_LIB} plasma_lib) +target_compile_options(photon_tests PUBLIC "-DPHOTON_TEST") install(TARGETS photon DESTINATION ${CMAKE_SOURCE_DIR}/photon) diff --git a/src/photon/Makefile b/src/photon/Makefile deleted file mode 100644 index 162c00c66..000000000 --- a/src/photon/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -CC = gcc -CFLAGS = -g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -I.. -I../common -I../common/thirdparty -I../plasma/ -fPIC -TEST_CFLAGS = -DPHOTON_TEST=1 -I. -BUILD = build - -all: $(BUILD)/photon_scheduler $(BUILD)/photon_client.a - -$(BUILD)/photon_tests: test/photon_tests.c photon.h photon_scheduler.h photon_scheduler.c photon_algorithm.h photon_algorithm.c photon_client.h photon_client.c common - $(CC) $(CFLAGS) $(TEST_CFLAGS) -o $@ test/photon_tests.c photon_scheduler.c photon_algorithm.c photon_client.c ../common/build/libcommon.a ../common/thirdparty/hiredis/libhiredis.a -I../common/thirdparty/ -I../common/ ../plasma/build/libplasma_client.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a -I../plasma/ - -$(BUILD)/photon_client.a: photon_client.o - ar rcs $(BUILD)/photon_client.a photon_client.o - -$(BUILD)/photon_scheduler: photon.h photon_scheduler.c photon_algorithm.c common - $(CC) $(CFLAGS) -o $@ photon_scheduler.c photon_algorithm.c ../common/build/libcommon.a ../common/thirdparty/hiredis/libhiredis.a -I../common/thirdparty/ -I../common/ ../plasma/build/libplasma_client.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a - -common: FORCE - cd ../common; make; cd build; cmake ..; make - -clean: - cd ../common; make clean - rm -rf $(BUILD)/* - rm -f *.o - -# Set the request timeout low and logging level at FATAL for testing purposes. -test: CFLAGS += -DRAY_TIMEOUT=50 -DRAY_COMMON_LOG_LEVEL=4 -test: $(BUILD)/photon_tests FORCE - ../common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ../common/redis_module/ray_redis_module.so & - ../plasma/build/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 & - sleep 0.5s - ./build/photon_tests - ../common/thirdparty/redis/src/redis-cli shutdown - killall plasma_store - -valgrind: test - ../common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ../common/redis_module/ray_redis_module.so & - ../plasma/build/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 & - sleep 0.5s - valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 ./build/photon_tests - ../common/thirdparty/redis/src/redis-cli shutdown - killall plasma_store - -FORCE: diff --git a/src/photon/photon/__init__.py b/src/photon/photon/__init__.py deleted file mode 100644 index c81d5021f..000000000 --- a/src/photon/photon/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from .libphoton import * -from .photon_services import * diff --git a/src/photon/test/run_tests.sh b/src/photon/test/run_tests.sh new file mode 100644 index 000000000..fe88aeb63 --- /dev/null +++ b/src/photon/test/run_tests.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# This needs to be run in the build tree, which is normally ray/build + +# Cause the script to exit if a single command fails. +set -e + +./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so & +sleep 1s +./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 & +sleep 0.5s +./src/photon/photon_tests +./src/common/thirdparty/redis/src/redis-cli shutdown +killall plasma_store diff --git a/src/photon/test/run_valgrind.sh b/src/photon/test/run_valgrind.sh new file mode 100644 index 000000000..3ffeb66d5 --- /dev/null +++ b/src/photon/test/run_valgrind.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# This needs to be run in the build tree, which is normally ray/build + +# Cause the script to exit if a single command fails. +set -e + +./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so & +sleep 1s +./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 & +sleep 0.5s +valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 ./src/photon/photon_tests +./src/common/thirdparty/redis/src/redis-cli shutdown +killall plasma_store diff --git a/src/plasma/CMakeLists.txt b/src/plasma/CMakeLists.txt index ba8516ea9..5760ee382 100644 --- a/src/plasma/CMakeLists.txt +++ b/src/plasma/CMakeLists.txt @@ -3,60 +3,7 @@ cmake_minimum_required(VERSION 2.8) project(plasma) # Recursively include common -include(${CMAKE_CURRENT_SOURCE_DIR}/../common/CMakeLists.txt) - -message(STATUS "Trying custom approach for finding Python.") -# Start off by figuring out which Python executable to use. -find_program(CUSTOM_PYTHON_EXECUTABLE python) -message(STATUS "Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}") -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print('python' + sys.version[0:3])" - OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) -message(STATUS "PYTHON_LIBRARY_NAME: " ${PYTHON_LIBRARY_NAME}) -# Now find the Python include directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())" - OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE) -message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS}) -# Now find the Python libraries. We'll start by looking near the Python -# executable. If that fails, then we'll look near the Python include -# directories. -execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print(sys.exec_prefix)" - OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) -message(STATUS "PYTHON_PREFIX: " ${PYTHON_PREFIX}) -# The name ending in "m" is for miniconda. -FIND_LIBRARY(PYTHON_LIBRARIES - NAMES "${PYTHON_LIBRARY_NAME}" "${PYTHON_LIBRARY_NAME}m" - HINTS "${PYTHON_PREFIX}" - PATH_SUFFIXES "lib" "libs" - NO_DEFAULT_PATH) -message(STATUS "PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES}) -# If that failed, perhaps because the user is in a virtualenv, search around -# the Python include directories. -if(NOT PYTHON_LIBRARIES) - message(STATUS "Failed to find PYTHON_LIBRARIES near the Python executable, so now looking near the Python include directories.") - # The name ending in "m" is for miniconda. - FIND_LIBRARY(PYTHON_LIBRARIES - NAMES "${PYTHON_LIBRARY_NAME}" "${PYTHON_LIBRARY_NAME}m" - HINTS "${PYTHON_INCLUDE_DIRS}/../.." - PATH_SUFFIXES "lib" "libs" - NO_DEFAULT_PATH) - message(STATUS "PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES}) -endif() -# If we found the Python libraries and the include directories, then continue -# on. If not, then try find_package as a last resort, but it probably won't -# work. -if(PYTHON_LIBRARIES AND PYTHON_INCLUDE_DIRS) - message(STATUS "The custom approach for finding Python succeeded.") - SET(PYTHONLIBS_FOUND TRUE) -else() - message(WARNING "The custom approach for finding Python failed. Defaulting to find_package.") - find_package(PythonInterp REQUIRED) - find_package(PythonLibs ${PYTHON_VERSION_STRING} EXACT REQUIRED) - set(CUSTOM_PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}) -endif() - -message(STATUS "Using CUSTOM_PYTHON_EXECUTABLE: " ${CUSTOM_PYTHON_EXECUTABLE}) -message(STATUS "Using PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES}) -message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS}) +include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake) if(APPLE) SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") @@ -68,8 +15,8 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOU # Compile flatbuffers -set(PLASMA_FBS_SRC "${CMAKE_SOURCE_DIR}/format/plasma.fbs") -set(OUTPUT_DIR ${CMAKE_SOURCE_DIR}/format/) +set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs") +set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/) add_custom_target(gen_plasma_fbs ALL) @@ -84,16 +31,8 @@ if(UNIX AND NOT APPLE) link_libraries(rt) endif() -set(COMMON_LIB "${CMAKE_SOURCE_DIR}/../common/build/libcommon.a" - CACHE STRING "Path to libcommon.a") - -include_directories("${CMAKE_SOURCE_DIR}/") -include_directories("${CMAKE_SOURCE_DIR}/../") -include_directories("${CMAKE_SOURCE_DIR}/../common/") -include_directories("${CMAKE_SOURCE_DIR}/../common/thirdparty/") -include_directories("${CMAKE_SOURCE_DIR}/../common/lib/python/") - -include_directories("${FLATBUFFERS_INCLUDE_DIR}") +include_directories("${CMAKE_CURRENT_LIST_DIR}/") +include_directories("${CMAKE_CURRENT_LIST_DIR}/../") add_library(plasma SHARED plasma.c @@ -113,4 +52,39 @@ endif(APPLE) target_link_libraries(plasma ${COMMON_LIB} ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB}) -install(TARGETS plasma DESTINATION ${CMAKE_SOURCE_DIR}/plasma) +set_source_files_properties(thirdparty/dlmalloc.c PROPERTIES COMPILE_FLAGS -Wno-all) + +add_executable(plasma_store + plasma_store.c + plasma.c + plasma_protocol.c + eviction_policy.c + fling.c + malloc.c) + +add_dependencies(plasma_store hiredis gen_plasma_fbs) + +target_link_libraries(plasma_store common ${FLATBUFFERS_STATIC_LIB}) + +add_library(plasma_lib STATIC + plasma_client.c + plasma.c + plasma_protocol.c + fling.c + thirdparty/xxhash.c) + +target_link_libraries(plasma_lib common ${FLATBUFFERS_STATIC_LIB}) +add_dependencies(plasma_lib gen_plasma_fbs) + +add_executable(plasma_manager + plasma_manager.c) + +target_link_libraries(plasma_manager common plasma_lib ${FLATBUFFERS_STATIC_LIB}) + +add_library(plasma_client SHARED plasma_client.c) + +target_link_libraries(plasma_client common plasma_lib ${FLATBUFFERS_STATIC_LIB}) + +define_test(client_tests plasma_lib) +define_test(manager_tests plasma_lib plasma_manager.c) +define_test(serialization_tests plasma_lib) diff --git a/src/plasma/Makefile b/src/plasma/Makefile deleted file mode 100644 index e01742a9d..000000000 --- a/src/plasma/Makefile +++ /dev/null @@ -1,66 +0,0 @@ -CC = gcc -# The -rdynamic is here so we always get function names in backtraces. -CFLAGS = -g -Wall -rdynamic -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -I. -Ithirdparty -I../common -I../common/thirdparty -I../common/build/flatcc-prefix/src/flatcc/include -fPIC -TEST_CFLAGS = -DPLASMA_TEST=1 -I. -BUILD = build - -all: $(BUILD)/plasma_store $(BUILD)/plasma_manager $(BUILD)/plasma_client.so $(BUILD)/example $(BUILD)/libplasma_client.a - -debug: FORCE -debug: CFLAGS += -DRAY_COMMON_LOG_LEVEL=0 -debug: all - -clean: - cd ../common; make clean - rm -f *.o - rm -rf $(BUILD)/* - -$(BUILD)/manager_tests: test/manager_tests.c plasma.h plasma.c plasma_client.h plasma_client.c thirdparty/xxhash.c plasma_protocol.h plasma_protocol.c plasma_manager.h plasma_manager.c fling.h fling.c common - $(CC) $(CFLAGS) $(TEST_CFLAGS) -o $@ test/manager_tests.c plasma.c plasma_manager.c plasma_client.c thirdparty/xxhash.c plasma_protocol.c fling.c ../common/build/libcommon.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a ../common/thirdparty/hiredis/libhiredis.a - -$(BUILD)/client_tests: test/client_tests.c plasma.h plasma.c plasma_client.h plasma_client.c thirdparty/xxhash.c plasma_protocol.h plasma_protocol.c plasma_manager.h plasma_manager.c fling.h fling.c common - $(CC) $(CFLAGS) $(TEST_CFLAGS) -o $@ test/client_tests.c plasma.c plasma_manager.c plasma_client.c thirdparty/xxhash.c plasma_protocol.c fling.c ../common/build/libcommon.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a ../common/thirdparty/hiredis/libhiredis.a - -$(BUILD)/serialization_tests: test/serialization_tests.c plasma.h plasma.c plasma_protocol.h plasma_protocol.c common - $(CC) $(CFLAGS) $(TEST_CFLAGS) -o $@ test/serialization_tests.c plasma.c plasma_protocol.c ../common/build/libcommon.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a - -$(BUILD)/plasma_store: plasma_store.c plasma.h plasma.c plasma_protocol.c eviction_policy.c fling.h fling.c malloc.c malloc.h thirdparty/dlmalloc.c common - $(CC) $(CFLAGS) plasma_store.c plasma.c plasma_protocol.c eviction_policy.c fling.c malloc.c ../common/build/libcommon.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a -o $(BUILD)/plasma_store - -$(BUILD)/plasma_manager: plasma_manager.c plasma.h plasma.c plasma_protocol.c plasma_client.c thirdparty/xxhash.c fling.h fling.c common - $(CC) $(CFLAGS) plasma_manager.c plasma.c plasma_protocol.c plasma_client.c thirdparty/xxhash.c fling.c ../common/build/libcommon.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a ../common/thirdparty/hiredis/libhiredis.a -o $(BUILD)/plasma_manager - -$(BUILD)/plasma_client.so: plasma.h plasma.c plasma_protocol.c plasma_client.c thirdparty/xxhash.c fling.h fling.c common - $(CC) $(CFLAGS) plasma.c plasma_protocol.c plasma_client.c thirdparty/xxhash.c fling.c ../common/build/libcommon.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a -fPIC -shared -o $(BUILD)/plasma_client.so - -$(BUILD)/libplasma_client.a: plasma.o plasma_protocol.o plasma_client.o thirdparty/xxhash.o fling.o ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a - ar rcs $@ $^ - -$(BUILD)/example: plasma_client.c thirdparty/xxhash.c plasma.h plasma.c plasma_protocol.c example.c fling.h fling.c common - $(CC) $(CFLAGS) plasma_client.c thirdparty/xxhash.c plasma.c plasma_protocol.c example.c fling.c ../common/build/libcommon.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a -o $(BUILD)/example - -common: FORCE - cd ../common; make; cd build; cmake ..; make; cd ../../plasma/build; cmake ..; make - -# Set the request timeout low and logging level at FATAL for testing purposes. -test: CFLAGS += -DRAY_TIMEOUT=50 -DRAY_COMMON_LOG_LEVEL=4 -# First, build and run all the unit tests. -test: $(BUILD)/manager_tests $(BUILD)/client_tests $(BUILD)/serialization_tests FORCE - ./build/plasma_store -s /tmp/plasma_store_socket_1 -m 0 & - sleep 1 - ./build/manager_tests - killall plasma_store - ./build/serialization_tests - ./test/run_client_tests.sh - cd ../common; make redis -# Next, build all the executables for Python testing. -test: all - -valgrind: test - ./build/plasma_store -s /tmp/plasma_store_socket_1 -m 0 & - sleep 1 - valgrind --leak-check=full --error-exitcode=1 ./build/manager_tests - killall plasma_store - valgrind --leak-check=full --error-exitcode=1 ./build/serialization_tests - -FORCE: diff --git a/src/plasma/test/run_client_tests.sh b/src/plasma/test/run_client_tests.sh deleted file mode 100755 index d4fbef7dd..000000000 --- a/src/plasma/test/run_client_tests.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# Cause the script to exit if a single command fails. -set -e - -../common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ../common/redis_module/ray_redis_module.so & -sleep 1 -# flush the redis server -../common/thirdparty/redis/src/redis-cli flushall & -sleep 1 -./build/plasma_store -s /tmp/store1 -m 1000000000 & -./build/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 & -./build/plasma_store -s /tmp/store2 -m 1000000000 & -./build/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 & -sleep 1 -./build/client_tests -kill %4 -kill %3 -kill %6 -kill %5 -kill %1 diff --git a/src/plasma/test/run_tests.sh b/src/plasma/test/run_tests.sh new file mode 100644 index 000000000..451e8d05f --- /dev/null +++ b/src/plasma/test/run_tests.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Cause the script to exit if a single command fails. +set -e + +./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 & +sleep 1 +./src/plasma/manager_tests +killall plasma_store +./src/plasma/serialization_tests + +./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so & +redis_pid=$! +sleep 1 +# flush the redis server +../common/thirdparty/redis/src/redis-cli flushall & +sleep 1 +./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 & +plasma1_pid=$! +./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 & +plasma2_pid=$! +./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 & +plasma3_pid=$! +./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 & +plasma4_pid=$! +sleep 1 + +./src/plasma/client_tests + +kill $plasma4_pid +kill $plasma3_pid +kill $plasma2_pid +kill $plasma1_pid +kill $redis_pid diff --git a/src/plasma/test/run_valgrind.sh b/src/plasma/test/run_valgrind.sh new file mode 100644 index 000000000..74531e721 --- /dev/null +++ b/src/plasma/test/run_valgrind.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Cause the script to exit if a single command fails. +set -e + +./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 & +sleep 1 +valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests +killall plasma_store +valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests