wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-11 11:24:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Switch build system to use CMake completely. (#200)

Browse Source 
* switch to CMake completely

...

* cleanup

* Run C tests, update installation instructions.
This commit is contained in:
Philipp Moritz
2017-01-17 16:56:40 -08:00
committed by Robert Nishihara
parent ba8933e10f
commit a708e36225
106 changed files with 467 additions and 870 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1 -16
View File
@@ -1,22 +1,7 @@
# The build output should clearly not be checked in
/build
/src/plasma/build
/src/photon/build
/src/common/build
/python/core
/src/common/thirdparty/redis
/src/global_scheduler/build
/numbuf/build
/numbuf/thirdparty/arrow
/numbuf/dist
/webui/client/public
/webui/node_modules
# The files copied under lib/python should be ignored
/lib/python/common
/lib/python/global_scheduler
/lib/python/photon
/lib/python/plasma
/lib/python/webui
# Files generated by flatcc should be ignored
/src/plasma/format/*_builder.h
.gitmodules
-8
View File
@@ -1,8 +0,0 @@
[submodule "src/common/thirdparty/redis"]
path = src/common/thirdparty/redis-windows
url = https://github.com/MSOpenTech/redis.git
ignore = all
[submodule "src/common/thirdparty/python"]
path = src/common/thirdparty/python
url = https://github.com/austinsc/python.git
ignore = all
.travis.yml
+22 -30
View File
@@ -36,39 +36,39 @@ matrix:
- sudo apt-get install -qq valgrind
install:
- ./.travis/install-dependencies.sh
- cd src/common
- make valgrind
- cd ../..
- cd src/plasma
- make valgrind
- cd ../..
- cd src/photon
- make valgrind
- cd ../..
- ./.travis/install-ray.sh
- cd python/core
- bash ../../src/common/test/run_valgrind.sh
- bash ../../src/plasma/test/run_valgrind.sh
- bash ../../src/photon/test/run_valgrind.sh
- cd ../..
script:
- python src/plasma/test/test.py valgrind
- python src/photon/test/test.py valgrind
- python src/global_scheduler/test/test.py valgrind
- python ./python/plasma/test/test.py valgrind
- python ./python/photon/test/test.py valgrind
- python ./python/global_scheduler/test/test.py valgrind
install:
- ./.travis/install-dependencies.sh
- ./.travis/run-c-tests.sh
- ./.travis/install-ray.sh
- cd python/core
- bash ../../src/common/test/run_tests.sh
- bash ../../src/plasma/test/run_tests.sh
- bash ../../src/photon/test/run_tests.sh
- cd ../..
script:
- if [[ "$PYTHON" == "3.5" ]]; then export PATH="$HOME/miniconda/bin:$PATH"; fi
- python numbuf/python/test/runtest.py
- python src/numbuf/python/test/runtest.py
- python src/common/test/test.py
- python src/plasma/test/test.py
- python src/photon/test/test.py
- python src/global_scheduler/test/test.py
- python python/common/test/test.py
- python python/common/redis_module/runtest.py
- python python/plasma/test/test.py
- python python/photon/test/test.py
- python python/global_scheduler/test/test.py
- python test/runtest.py
- python test/array_test.py
@@ -76,11 +76,3 @@ script:
- python test/failure_test.py
- python test/microbenchmarks.py
- python test/stress_tests.py
after_script:
# Make sure that we can build numbuf as a standalone library.
- pip uninstall -y numbuf
- cd numbuf/build
- rm -rf *
- cmake -DHAS_PLASMA=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g" -DCMAKE_CXX_FLAGS="-g" ..
- make install
.travis/install-ray.sh
+2 -24
View File
@@ -9,36 +9,14 @@ echo "PYTHON is $PYTHON"
if [[ "$PYTHON" == "2.7" ]]; then
pushd "$ROOT_DIR/../numbuf"
sudo python setup.py install
popd
pushd "$ROOT_DIR/../src/common/lib/python"
pushd "$ROOT_DIR/../src/common"
make
popd
sudo python setup.py install
popd
pushd "$ROOT_DIR/../lib/python"
pushd "$ROOT_DIR/../python"
sudo python setup.py install
popd
elif [[ "$PYTHON" == "3.5" ]]; then
export PATH="$HOME/miniconda/bin:$PATH"
pushd "$ROOT_DIR/../numbuf"
python setup.py install --user
popd
pushd "$ROOT_DIR/../src/common/lib/python"
pushd "$ROOT_DIR/../src/common"
make
popd
python setup.py install --user
popd
pushd "$ROOT_DIR/../lib/python"
pushd "$ROOT_DIR/../python"
python setup.py install --user
popd
.travis/run-c-tests.sh
-22
View File
@@ -1,22 +0,0 @@
#!/usr/bin/env bash
# Cause the script to exit if a single command fails.
set -e
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
echo "PYTHON is $PYTHON"
if [[ "$PYTHON" == "3.5" ]]; then export PATH="$HOME/miniconda/bin:$PATH"; fi
pushd "$ROOT_DIR/../src/common"
make test
popd
pushd "$ROOT_DIR/../src/plasma"
make test
popd
pushd "$ROOT_DIR/../src/photon"
make test
popd
CMakeLists.txt
+9
View File
@@ -0,0 +1,9 @@
cmake_minimum_required(VERSION 2.8)
project(ray)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/photon/)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/global_scheduler/)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/numbuf/)
build.sh
+8 -56
View File
@@ -16,64 +16,16 @@ else
exit 1
fi
COMMON_DIR="$ROOT_DIR/src/common"
PLASMA_DIR="$ROOT_DIR/src/plasma"
PHOTON_DIR="$ROOT_DIR/src/photon"
GLOBAL_SCHEDULER_DIR="$ROOT_DIR/src/global_scheduler"
pushd "$ROOT_DIR/src/common/thirdparty/"
bash build-redis.sh
popd
PYTHON_DIR="$ROOT_DIR/lib/python"
PYTHON_COMMON_DIR="$PYTHON_DIR/common"
PYTHON_PLASMA_DIR="$PYTHON_DIR/plasma"
PYTHON_PHOTON_DIR="$PYTHON_DIR/photon"
PYTHON_GLOBAL_SCHEDULER_DIR="$PYTHON_DIR/global_scheduler"
# First clean up old build files.
pushd "$COMMON_DIR"
make clean
popd
pushd "$PLASMA_DIR"
make clean
popd
pushd "$PHOTON_DIR"
make clean
popd
pushd "$GLOBAL_SCHEDULER_DIR"
make clean
popd
bash "$ROOT_DIR/src/numbuf/thirdparty/download_thirdparty.sh"
bash "$ROOT_DIR/src/numbuf/thirdparty/build_thirdparty.sh"
# Now build everything.
pushd "$COMMON_DIR"
pushd "$ROOT_DIR/python/core"
cmake ../..
make clean
make
popd
cp "$COMMON_DIR/thirdparty/redis/src/redis-server" "$PYTHON_COMMON_DIR/thirdparty/redis/src/"
cp "$COMMON_DIR/redis_module/ray_redis_module.so" "$PYTHON_COMMON_DIR/redis_module/ray_redis_module.so"
pushd "$PLASMA_DIR"
make
pushd "$PLASMA_DIR/build"
cmake ..
make install
popd
popd
cp "$PLASMA_DIR/build/plasma_store" "$PYTHON_PLASMA_DIR/"
cp "$PLASMA_DIR/build/plasma_manager" "$PYTHON_PLASMA_DIR/"
cp "$PLASMA_DIR/plasma/plasma.py" "$PYTHON_PLASMA_DIR/"
cp "$PLASMA_DIR/plasma/utils.py" "$PYTHON_PLASMA_DIR/"
cp "$PLASMA_DIR/plasma/libplasma.so" "$PYTHON_PLASMA_DIR/"
pushd "$PHOTON_DIR"
make
pushd "$PHOTON_DIR/build"
cmake ..
make install
popd
popd
cp "$PHOTON_DIR/build/photon_scheduler" "$PYTHON_PHOTON_DIR/build/"
cp "$PHOTON_DIR/photon/libphoton.so" "$PYTHON_PHOTON_DIR/photon/"
cp "$PHOTON_DIR/photon/photon_services.py" "$PYTHON_PHOTON_DIR/photon/"
pushd "$GLOBAL_SCHEDULER_DIR"
make
popd
cp "$GLOBAL_SCHEDULER_DIR/build/global_scheduler" "$PYTHON_GLOBAL_SCHEDULER_DIR/build/"
cp "$GLOBAL_SCHEDULER_DIR/lib/python/global_scheduler_services.py" "$PYTHON_GLOBAL_SCHEDULER_DIR/lib/python/"
doc/install-on-macosx.md
+1 -8
View File
@@ -13,7 +13,6 @@ brew install cmake automake autoconf libtool boost wget
sudo easy_install pip # If you're using Anaconda, then this is unnecessary.
pip install numpy cloudpickle funcsigs colorama psutil redis --ignore-installed six
pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=numbuf&subdirectory=numbuf"
```
# Install Ray
@@ -26,14 +25,8 @@ cd ray/lib/python
python setup.py install --user
```
Alternatively, Ray can be installed with pip as follows. However, this is
slightly less likely to succeed.
```
pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=ray&subdirectory=lib/python"
```
## Test if the installation succeeded
To test if the installation was successful, try running some tests. This assumes
that you've cloned the git repository.
doc/install-on-ubuntu.md
-8
View File
@@ -13,7 +13,6 @@ sudo apt-get update
sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-pip libboost-all-dev unzip # If you're using Anaconda, then python-dev and python-pip are unnecessary.
pip install numpy cloudpickle funcsigs colorama psutil redis
pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=numbuf&subdirectory=numbuf"
```
# Install Ray
@@ -26,13 +25,6 @@ cd ray/lib/python
python setup.py install --user
```
Alternatively, Ray can be installed with pip as follows. However, this is
slightly less likely to succeed.
```
pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=ray&subdirectory=lib/python"
```
## Test if the installation succeeded
To test if the installation was successful, try running some tests. This assumes
docker/base-deps/Dockerfile
-1
View File
@@ -14,4 +14,3 @@ RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \
ENV PATH "/opt/conda/bin:$PATH"
RUN conda install libgcc
RUN pip install --upgrade pip
RUN pip install --upgrade --verbose "git+git://github.com/ray-project/ray.git#egg=numbuf&subdirectory=numbuf"
lib/python/global_scheduler/__init__.py
-5
View File
@@ -1,5 +0,0 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from .lib.python.global_scheduler_services import *
lib/python/photon/__init__.py
-1
View File
@@ -1 +0,0 @@
from .photon import *
numbuf/build.sh
-31
View File
@@ -1,31 +0,0 @@
#!/usr/bin/env bash
# Cause the script to exit if a single command fails.
set -e
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
# Determine how many parallel jobs to use for make based on the number of cores
unamestr="$(uname)"
if [[ "$unamestr" == "Linux" ]]; then
PARALLEL=$(nproc)
elif [[ "$unamestr" == "Darwin" ]]; then
PARALLEL=$(sysctl -n hw.ncpu)
else
echo "Unrecognized platform."
exit 1
fi
# Build plasma. We currently have to do this because we compile numbuf with the
# HAS_PLASMA flag.
PLASMA_DIR="$ROOT_DIR/../src/plasma"
pushd "$PLASMA_DIR"
make clean
make
popd
mkdir -p "$ROOT_DIR/build"
pushd "$ROOT_DIR/build"
cmake -DHAS_PLASMA=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g" -DCMAKE_CXX_FLAGS="-g" ..
make install -j$PARALLEL
popd
numbuf/setup.py
-37
View File
@@ -1,37 +0,0 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import subprocess
from setuptools import setup, find_packages
import setuptools.command.install as _install
from sys import platform
extension = ""
if platform == "linux" or platform == "linux2":
extension = ".so"
elif platform == "darwin":
extension = ".dylib"
# Because of relative paths, this must be run from inside numbuf/.
class install(_install.install):
def run(self):
subprocess.check_call(["./setup.sh"])
subprocess.check_call(["./build.sh"])
# Calling _install.install.run(self) does not fetch required packages and
# instead performs an old-style install. See command/install.py in
# setuptools. So, calling do_egg_install() manually here.
self.do_egg_install()
setup(name="numbuf",
version="0.0.1",
packages=find_packages(),
package_data={"numbuf": ["libnumbuf.so",
"libarrow" + extension,
"libarrow_io" + extension,
"libarrow_ipc" + extension]},
cmdclass={"install": install},
setup_requires=["numpy"],
include_package_data=True,
zip_safe=False)
numbuf/setup.sh
-24
View File
@@ -1,24 +0,0 @@
#!/usr/bin/env bash
# Cause the script to exit if a single command fails.
set -e
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
platform="unknown"
unamestr="$(uname)"
if [[ "$unamestr" == "Linux" ]]; then
echo "Platform is linux."
platform="linux"
elif [[ "$unamestr" == "Darwin" ]]; then
echo "Platform is macosx."
platform="macosx"
else
echo "Unrecognized platform."
exit 1
fi
pushd "$ROOT_DIR"
./thirdparty/download_thirdparty.sh
./thirdparty/build_thirdparty.sh
popd
lib/python/common/__init__.py → python/common/__init__.py
View File
lib/python/common/redis_module/.gitkeep → python/common/redis_module/.gitkeep
View File
src/common/redis_module/runtest.py → python/common/redis_module/runtest.py
+2 -2
View File
@@ -11,12 +11,12 @@ import unittest
import redis
# Check if the redis-server binary is present.
redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../thirdparty/redis/src/redis-server")
redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server")
if not os.path.exists(redis_path):
raise Exception("You do not have the redis-server binary. Run `make test` in the plasma directory to get it.")
# Absolute path of the ray redis module.
module_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "ray_redis_module.so")
module_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/redis_module/libray_redis_module.so")
print("path to the redis module is {}".format(module_path))
OBJECT_INFO_PREFIX = "OI:"
src/common/test/test.py → python/common/test/test.py
+16 -16
View File
@@ -7,18 +7,18 @@ import pickle
import sys
import unittest
import common
import photon
ID_SIZE = 20
def random_object_id():
return common.ObjectID(np.random.bytes(ID_SIZE))
return photon.ObjectID(np.random.bytes(ID_SIZE))
def random_function_id():
return common.ObjectID(np.random.bytes(ID_SIZE))
return photon.ObjectID(np.random.bytes(ID_SIZE))
def random_task_id():
return common.ObjectID(np.random.bytes(ID_SIZE))
return photon.ObjectID(np.random.bytes(ID_SIZE))
BASE_SIMPLE_OBJECTS = [
0, 1, 100000, 0.0, 0.5, 0.9, 100000.1, (), [], {},
@@ -62,9 +62,9 @@ class TestSerialization(unittest.TestCase):
def test_serialize_by_value(self):
for val in SIMPLE_OBJECTS:
self.assertTrue(common.check_simple_value(val))
self.assertTrue(photon.check_simple_value(val))
for val in COMPLEX_OBJECTS:
self.assertFalse(common.check_simple_value(val))
self.assertFalse(photon.check_simple_value(val))
class TestObjectID(unittest.TestCase):
@@ -89,17 +89,17 @@ class TestObjectID(unittest.TestCase):
self.assertRaises(Exception, lambda : pickling.dumps(h))
def test_equality_comparisons(self):
x1 = common.ObjectID(ID_SIZE * b"a")
x2 = common.ObjectID(ID_SIZE * b"a")
y1 = common.ObjectID(ID_SIZE * b"b")
y2 = common.ObjectID(ID_SIZE * b"b")
x1 = photon.ObjectID(ID_SIZE * b"a")
x2 = photon.ObjectID(ID_SIZE * b"a")
y1 = photon.ObjectID(ID_SIZE * b"b")
y2 = photon.ObjectID(ID_SIZE * b"b")
self.assertEqual(x1, x2)
self.assertEqual(y1, y2)
self.assertNotEqual(x1, y1)
random_strings = [np.random.bytes(ID_SIZE) for _ in range(256)]
object_ids1 = [common.ObjectID(random_strings[i]) for i in range(256)]
object_ids2 = [common.ObjectID(random_strings[i]) for i in range(256)]
object_ids1 = [photon.ObjectID(random_strings[i]) for i in range(256)]
object_ids2 = [photon.ObjectID(random_strings[i]) for i in range(256)]
self.assertEqual(len(set(object_ids1)), 256)
self.assertEqual(len(set(object_ids1 + object_ids2)), 256)
self.assertEqual(set(object_ids1), set(object_ids2))
@@ -118,7 +118,7 @@ class TestTask(unittest.TestCase):
self.assertEqual(num_return_vals, len(task.returns()))
self.assertEqual(len(args), len(retrieved_args))
for i in range(len(retrieved_args)):
if isinstance(retrieved_args[i], common.ObjectID):
if isinstance(retrieved_args[i], photon.ObjectID):
self.assertEqual(retrieved_args[i].id(), args[i].id())
else:
self.assertEqual(retrieved_args[i], args[i])
@@ -156,10 +156,10 @@ class TestTask(unittest.TestCase):
]
for args in args_list:
for num_return_vals in [0, 1, 2, 3, 5, 10, 100]:
task = common.Task(function_id, args, num_return_vals, parent_id, 0)
task = photon.Task(function_id, args, num_return_vals, parent_id, 0)
self.check_task(task, function_id, num_return_vals, args)
data = common.task_to_string(task)
task2 = common.task_from_string(data)
data = photon.task_to_string(task)
task2 = photon.task_from_string(data)
self.check_task(task2, function_id, num_return_vals, args)
if __name__ == "__main__":
lib/python/common/thirdparty/redis/src/.gitkeep → python/common/thirdparty/redis/src/.gitkeep
Vendored
View File
lib/python/global_scheduler/lib/__init__.py → python/core/__init__.py
View File
lib/python/global_scheduler/lib/python/__init__.py → python/core/src/__init__.py
View File
lib/python/ray/experimental/array/__init__.py → python/core/src/numbuf/__init__.py
View File
lib/python/ray/test/__init__.py → python/core/src/photon/__init__.py
View File
lib/python/ray/workers/__init__.py → python/core/src/plasma/__init__.py
View File
src/plasma/plasma/__init__.py → python/global_scheduler/__init__.py
+1 -1
View File
@@ -2,4 +2,4 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from plasma.plasma import *
from .global_scheduler_services import *
lib/python/global_scheduler/build/.gitkeep → python/global_scheduler/build/.gitkeep
View File
src/global_scheduler/lib/python/global_scheduler_services.py → python/global_scheduler/global_scheduler_services.py
+1 -1
View File
@@ -23,7 +23,7 @@ def start_global_scheduler(redis_address, use_valgrind=False, use_profiler=False
"""
if use_valgrind and use_profiler:
raise Exception("Cannot use valgrind and profiler at the same time.")
global_scheduler_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../build/global_scheduler")
global_scheduler_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../core/src/global_scheduler/global_scheduler")
command = [global_scheduler_executable, "-r", redis_address]
with open(os.devnull, "w") as FNULL:
stdout = FNULL if redirect_output else None
src/global_scheduler/test/test.py → python/global_scheduler/test/test.py
+2 -2
View File
@@ -46,8 +46,8 @@ class TestGlobalScheduler(unittest.TestCase):
def setUp(self):
# Start a Redis server.
redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../common/thirdparty/redis/src/redis-server")
redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../common/redis_module/ray_redis_module.so")
redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server")
redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../core/src/common/redis_module/libray_redis_module.so")
assert os.path.isfile(redis_path)
assert os.path.isfile(redis_module)
node_ip_address = "127.0.0.1"
python/numbuf/__init__.py
+29
View File
@@ -0,0 +1,29 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# See https://github.com/ray-project/ray/issues/131.
helpful_message = """
If you are using Anaconda, try fixing this problem by running:
conda install libgcc
"""
try:
from core.src.numbuf.libnumbuf import *
except ImportError as e:
if hasattr(e, "msg") and isinstance(e.msg, str) and "GLIBCXX" in e.msg:
# This code path should be taken with Python 3.
e.msg += helpful_message
elif hasattr(e, "message") and isinstance(e.message, str) and "GLIBCXX" in e.message:
# This code path should be taken with Python 2.
if hasattr(e, "args") and isinstance(e.args, tuple) and len(e.args) == 1 and isinstance(e.args[0], str):
e.args = (e.args[0] + helpful_message,)
else:
if not hasattr(e, "args"):
e.args = ()
elif not isinstance(e.args, tuple):
e.args = (e.args,)
e.args += (helpful_message,)
raise
lib/python/photon/photon/__init__.py → python/photon/__init__.py
+1 -1
View File
@@ -2,5 +2,5 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from .libphoton import *
from core.src.photon.libphoton import *
from .photon_services import *
lib/python/photon/build/.gitkeep → python/photon/build/.gitkeep
View File
src/photon/photon/photon_services.py → python/photon/photon_services.py
+1 -1
View File
@@ -40,7 +40,7 @@ def start_local_scheduler(plasma_store_name, plasma_manager_name=None, plasma_ad
raise Exception("If one of the plasma_manager_name and the redis_address is provided, then both must be provided.")
if use_valgrind and use_profiler:
raise Exception("Cannot use valgrind and profiler at the same time.")
local_scheduler_executable = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../build/photon_scheduler")
local_scheduler_executable = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/photon/photon_scheduler")
local_scheduler_name = "/tmp/scheduler{}".format(random_name())
command = [local_scheduler_executable, "-s", local_scheduler_name, "-p", plasma_store_name, "-h", node_ip_address]
if plasma_manager_name is not None:
src/photon/test/test.py → python/photon/test/test.py
View File
lib/python/plasma/.gitkeep → python/plasma/.gitkeep
View File
lib/python/plasma/__init__.py → python/plasma/__init__.py
View File
src/plasma/plasma/plasma.py → python/plasma/plasma.py
+5 -5
View File
@@ -8,9 +8,9 @@ import subprocess
import sys
import time
from . import libplasma
from .libplasma import plasma_object_exists_error
from .libplasma import plasma_out_of_memory_error
import core.src.plasma.libplasma as libplasma
from core.src.plasma.libplasma import plasma_object_exists_error
from core.src.plasma.libplasma import plasma_out_of_memory_error
PLASMA_ID_SIZE = 20
PLASMA_WAIT_TIMEOUT = 2 ** 30
@@ -280,7 +280,7 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, use_valg
"""
if use_valgrind and use_profiler:
raise Exception("Cannot use valgrind and profiler at the same time.")
plasma_store_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "plasma_store")
plasma_store_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../core/src/plasma/plasma_store")
plasma_store_name = "/tmp/plasma_store{}".format(random_name())
command = [plasma_store_executable, "-s", plasma_store_name, "-m", str(plasma_store_memory)]
with open(os.devnull, "w") as FNULL:
@@ -316,7 +316,7 @@ def start_plasma_manager(store_name, redis_address, node_ip_address="127.0.0.1",
Raises:
Exception: An exception is raised if the manager could not be started.
"""
plasma_manager_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "plasma_manager")
plasma_manager_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../core/src/plasma/plasma_manager")
plasma_manager_name = "/tmp/plasma_manager{}".format(random_name())
port = None
process = None
src/plasma/test/test.py → python/plasma/test/test.py
+2 -2
View File
@@ -463,8 +463,8 @@ class TestPlasmaManager(unittest.TestCase):
store_name1, self.p2 = plasma.start_plasma_store(use_valgrind=USE_VALGRIND)
store_name2, self.p3 = plasma.start_plasma_store(use_valgrind=USE_VALGRIND)
# Start a Redis server.
redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../common/thirdparty/redis/src/redis-server")
redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../common/redis_module/ray_redis_module.so")
redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server")
redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../core/src/common/redis_module/libray_redis_module.so")
assert os.path.isfile(redis_path)
assert os.path.isfile(redis_module)
redis_port = 6379
src/plasma/plasma/utils.py → python/plasma/utils.py
View File
lib/python/ray/__init__.py → python/ray/__init__.py
View File
lib/python/ray/experimental/__init__.py → python/ray/experimental/__init__.py
View File
lib/python/webui/.gitkeep → python/ray/experimental/array/__init__.py
View File
lib/python/ray/experimental/array/distributed/__init__.py → python/ray/experimental/array/distributed/__init__.py
View File
lib/python/ray/experimental/array/distributed/core.py → python/ray/experimental/array/distributed/core.py
View File
lib/python/ray/experimental/array/distributed/linalg.py → python/ray/experimental/array/distributed/linalg.py
View File
lib/python/ray/experimental/array/distributed/random.py → python/ray/experimental/array/distributed/random.py
View File
lib/python/ray/experimental/array/remote/__init__.py → python/ray/experimental/array/remote/__init__.py
View File
lib/python/ray/experimental/array/remote/core.py → python/ray/experimental/array/remote/core.py
View File
lib/python/ray/experimental/array/remote/linalg.py → python/ray/experimental/array/remote/linalg.py
View File
lib/python/ray/experimental/array/remote/random.py → python/ray/experimental/array/remote/random.py
View File
lib/python/ray/experimental/tfutils.py → python/ray/experimental/tfutils.py
View File
lib/python/ray/experimental/utils.py → python/ray/experimental/utils.py
View File
lib/python/ray/pickling.py → python/ray/pickling.py
View File
lib/python/ray/serialization.py → python/ray/serialization.py
View File
lib/python/ray/services.py → python/ray/services.py
+2 -2
View File
@@ -154,8 +154,8 @@ def start_redis(node_ip_address, num_retries=20, cleanup=True, redirect_output=F
Raises:
Exception: An exception is raised if Redis could not be started.
"""
redis_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../common/thirdparty/redis/src/redis-server")
redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../common/redis_module/ray_redis_module.so")
redis_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/common/thirdparty/redis/src/redis-server")
redis_module = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../core/src/common/redis_module/libray_redis_module.so")
assert os.path.isfile(redis_filepath)
assert os.path.isfile(redis_module)
counter = 0
src/plasma/build/.gitkeep → python/ray/test/__init__.py
View File
lib/python/ray/test/test_functions.py → python/ray/test/test_functions.py
View File
lib/python/ray/worker.py → python/ray/worker.py
View File
python/ray/workers/__init__.py
View File
lib/python/ray/workers/default_worker.py → python/ray/workers/default_worker.py
View File
lib/python/setup.py → python/setup.py
+11 -9
View File
@@ -10,7 +10,7 @@ import setuptools.command.install as _install
class install(_install.install):
def run(self):
subprocess.check_call(["../../build.sh"])
subprocess.check_call(["../build.sh"])
# Calling _install.install.run(self) does not fetch required packages and
# instead performs an old-style install. See command/install.py in
# setuptools. So, calling do_egg_install() manually here.
@@ -19,14 +19,16 @@ class install(_install.install):
setup(name="ray",
version="0.0.1",
packages=find_packages(),
package_data={"common": ["thirdparty/redis/src/redis-server",
"redis_module/ray_redis_module.so"],
"plasma": ["plasma_store",
"plasma_manager",
"libplasma.so"],
"photon": ["build/photon_scheduler",
"photon/libphoton.so"],
"global_scheduler": ["build/global_scheduler"]},
package_data={"core": ["src/common/thirdparty/redis/src/redis-server",
"src/common/redis_module/libray_redis_module.so",
"src/plasma/plasma_store",
"src/plasma/plasma_manager",
"src/plasma/libplasma.so",
"src/photon/photon_scheduler",
"src/photon/libphoton.so",
"src/numbuf/libarrow.so",
"src/numbuf/libnumbuf.so",
"src/global_scheduler/global_scheduler"]},
cmdclass={"install": install},
install_requires=["numpy",
"funcsigs",
python/webui/.gitkeep
View File
src/common/CMakeLists.txt
+54 -9
View File
@@ -2,15 +2,60 @@ cmake_minimum_required(VERSION 2.8)
project(common)
include(ExternalProject)
include(${CMAKE_CURRENT_LIST_DIR}/cmake/Common.cmake)
set(FLATCC_PREFIX "${CMAKE_CURRENT_LIST_DIR}/build/flatcc-prefix/src/flatcc")
add_subdirectory(redis_module)
ExternalProject_Add(flatcc
URL "https://github.com/dvidelabs/flatcc/archive/v0.4.0.tar.gz"
INSTALL_COMMAND ""
CMAKE_ARGS "-DCMAKE_C_FLAGS=-fPIC")
include_directories(thirdparty/ae)
set(FLATBUFFERS_INCLUDE_DIR "${FLATCC_PREFIX}/include")
set(FLATBUFFERS_STATIC_LIB "${FLATCC_PREFIX}/lib/libflatcc.a")
set(FLATBUFFERS_COMPILER "${FLATCC_PREFIX}/bin/flatcc")
add_custom_target(
hiredis
COMMAND make
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis)
add_library(common STATIC
event_loop.c
common.c
task.c
io.c
net.c
logging.c
state/redis.c
state/table.c
state/object_table.c
state/task_table.c
state/db_client_table.c
state/local_scheduler_table.c
thirdparty/ae/ae.c
thirdparty/sha256.c)
target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhiredis.a")
function(define_test test_name library)
add_executable(${test_name} test/${test_name}.c ${ARGN})
add_dependencies(${test_name} hiredis flatcc)
target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ${library})
target_compile_options(${test_name} PUBLIC "-DPLASMA_TEST -DPHOTON_TEST -DCOMMON_TEST -DRAY_COMMON_LOG_LEVEL=4 -DRAY_TIMEOUT=50")
endfunction()
define_test(common_tests "")
define_test(db_tests "")
define_test(io_tests "")
define_test(task_tests "")
define_test(redis_tests "")
define_test(task_table_tests "")
define_test(object_table_tests "")
add_custom_target(
redis
COMMAND make
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/thirdparty/redis)
add_custom_target(copy_redis ALL)
foreach(file "redis-cli" "redis-server")
add_custom_command(TARGET copy_redis POST_BUILD
COMMAND ${CMAKE_COMMAND} -E
copy ${CMAKE_CURRENT_LIST_DIR}/thirdparty/redis/src/${file}
${CMAKE_BINARY_DIR}/src/common/thirdparty/redis/src/${file})
endforeach()
add_dependencies(copy_redis redis)
src/common/Makefile
-71
View File
@@ -1,71 +0,0 @@
CC = gcc
CFLAGS = -g -Wall -Wextra -Werror=implicit-function-declaration -Wno-typedef-redefinition -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -fPIC -I. -Ithirdparty -Ithirdparty/ae
BUILD = build
all: hiredis redis redismodule $(BUILD)/libcommon.a
$(BUILD)/libcommon.a: event_loop.o common.o task.o io.o net.o logging.o state/redis.o state/table.o state/object_table.o state/task_table.o state/db_client_table.o state/local_scheduler_table.o thirdparty/ae/ae.o thirdparty/sha256.o
ar rcs $@ $^
$(BUILD)/common_tests: test/common_tests.c $(BUILD)/libcommon.a
$(CC) -o $@ test/common_tests.c $(BUILD)/libcommon.a $(CFLAGS)
$(BUILD)/db_tests: hiredis test/db_tests.c $(BUILD)/libcommon.a
$(CC) -o $@ test/db_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS)
$(BUILD)/object_table_tests: hiredis test/object_table_tests.c $(BUILD)/libcommon.a
$(CC) -o $@ test/object_table_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS)
$(BUILD)/task_table_tests: hiredis test/task_table_tests.c $(BUILD)/libcommon.a
$(CC) -o $@ test/task_table_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS)
$(BUILD)/io_tests: test/io_tests.c $(BUILD)/libcommon.a
$(CC) -o $@ $^ $(CFLAGS)
$(BUILD)/task_tests: test/task_tests.c $(BUILD)/libcommon.a
$(CC) -o $@ $^ thirdparty/hiredis/libhiredis.a $(CFLAGS)
$(BUILD)/redis_tests: hiredis test/redis_tests.c $(BUILD)/libcommon.a logging.h
$(CC) -o $@ test/redis_tests.c logging.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS)
clean:
rm -f *.o state/*.o test/*.o thirdparty/ae/*.o
rm -rf $(BUILD)/*
cd redis_module; make clean
redis:
cd thirdparty ; bash ./build-redis.sh
hiredis:
cd thirdparty/hiredis ; make
redismodule:
cd redis_module && make && cd ..
test: CFLAGS += -DRAY_COMMON_LOG_LEVEL=4
test: hiredis redis redismodule $(BUILD)/common_tests $(BUILD)/task_table_tests $(BUILD)/object_table_tests $(BUILD)/db_tests $(BUILD)/io_tests $(BUILD)/task_tests $(BUILD)/redis_tests FORCE
./thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./redis_module/ray_redis_module.so &
sleep 1s
./build/common_tests
./build/db_tests
./build/io_tests
./build/task_tests
./build/redis_tests
./build/task_table_tests
./build/object_table_tests
./thirdparty/redis/src/redis-cli shutdown
python ./redis_module/runtest.py
valgrind: test
./thirdparty/redis/src/redis-server --loglevel warning --loadmodule redis_module/ray_redis_module.so &
sleep 1s
valgrind --leak-check=full --error-exitcode=1 ./build/common_tests
valgrind --leak-check=full --error-exitcode=1 ./build/db_tests
valgrind --leak-check=full --error-exitcode=1 ./build/io_tests
valgrind --leak-check=full --error-exitcode=1 ./build/task_tests
valgrind --leak-check=full --error-exitcode=1 ./build/redis_tests
valgrind --leak-check=full --error-exitcode=1 ./build/task_table_tests
valgrind --leak-check=full --error-exitcode=1 ./build/object_table_tests
./thirdparty/redis/src/redis-cli shutdown
FORCE:
numbuf/CMakeLists.txt → src/common/cmake/Common.cmake
+26 -72
View File
@@ -1,17 +1,28 @@
cmake_minimum_required(VERSION 2.8)
# Code for compiling flatbuffers
project(numbuf)
include(ExternalProject)
list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
set(FLATCC_PREFIX "${CMAKE_BINARY_DIR}/src/common/flatcc-prefix/src/flatcc")
option(HAS_PLASMA
"Are we linking with the plasma object store? Recommended if numbuf is used as part of ray."
OFF)
if(HAS_PLASMA)
add_definitions(-DHAS_PLASMA)
if (NOT TARGET flatcc)
ExternalProject_Add(flatcc
URL "https://github.com/dvidelabs/flatcc/archive/v0.4.0.tar.gz"
INSTALL_COMMAND ""
CMAKE_ARGS "-DCMAKE_C_FLAGS=-fPIC")
endif()
set(FLATBUFFERS_INCLUDE_DIR "${FLATCC_PREFIX}/include")
set(FLATBUFFERS_STATIC_LIB "${FLATCC_PREFIX}/lib/libflatcc.a")
set(FLATBUFFERS_COMPILER "${FLATCC_PREFIX}/bin/flatcc")
include_directories("${FLATBUFFERS_INCLUDE_DIR}")
# Custom CFLAGS
set(CMAKE_C_FLAGS "-g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -fPIC -std=c99")
# Code for finding Python
message(STATUS "Trying custom approach for finding Python.")
# Start off by figuring out which Python executable to use.
find_program(CUSTOM_PYTHON_EXECUTABLE python)
@@ -65,68 +76,11 @@ message(STATUS "Using CUSTOM_PYTHON_EXECUTABLE: " ${CUSTOM_PYTHON_EXECUTABLE})
message(STATUS "Using PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES})
message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
find_package(NumPy REQUIRED)
# Common libraries
if(APPLE)
SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
endif(APPLE)
set(COMMON_LIB "${CMAKE_BINARY_DIR}/src/common/libcommon.a"
CACHE STRING "Path to libcommon.a")
include_directories("${PYTHON_INCLUDE_DIRS}")
include_directories("${NUMPY_INCLUDE_DIR}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
if(UNIX AND NOT APPLE)
link_libraries(rt)
endif()
set(ARROW_DIR "${CMAKE_SOURCE_DIR}/thirdparty/arrow/"
CACHE STRING "Path of the arrow source directory")
set(ARROW_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a"
CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)")
set(ARROW_IO_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_io.a"
CACHE STRING "Path to libarrow_io.a (needs to be changed if arrow is build in debug mode)")
set(ARROW_IPC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow_ipc.a"
CACHE STRING "Path to libarrow_ipc.a (needs to be changed if arrow is build in debug mode)")
include_directories("${ARROW_DIR}/cpp/src/")
include_directories("cpp/src/")
include_directories("python/src/")
if(HAS_PLASMA)
include_directories("../src/plasma")
include_directories("../src/common")
include_directories("../src/common/thirdparty")
include_directories("../src/common/build/flatcc-prefix/src/flatcc/include")
endif()
add_definitions(-fPIC)
add_library(numbuf SHARED
cpp/src/numbuf/tensor.cc
cpp/src/numbuf/dict.cc
cpp/src/numbuf/sequence.cc
python/src/pynumbuf/numbuf.cc
python/src/pynumbuf/adapters/numpy.cc
python/src/pynumbuf/adapters/python.cc)
get_filename_component(PYTHON_SHARED_LIBRARY ${PYTHON_LIBRARIES} NAME)
if(APPLE)
add_custom_command(TARGET numbuf
POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libnumbuf.so)
endif(APPLE)
if(APPLE)
target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES})
else()
target_link_libraries(numbuf -Wl,--whole-archive ${ARROW_LIB} -Wl,--no-whole-archive ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES})
endif()
if(HAS_PLASMA)
target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES} "${CMAKE_SOURCE_DIR}/../src/plasma/build/libplasma_client.a" "${CMAKE_SOURCE_DIR}/../src/common/build/libcommon.a" "${CMAKE_SOURCE_DIR}/../src/common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a")
else()
target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES})
endif()
install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/)
include_directories("${CMAKE_CURRENT_LIST_DIR}/..")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../thirdparty/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python")
src/common/lib/python/common_module.c
-82
View File
@@ -1,82 +0,0 @@
#include <Python.h>
#include "common_extension.h"
static PyMethodDef common_methods[] = {
{"check_simple_value", check_simple_value, METH_VARARGS,
"Should the object be passed by value?"},
{"compute_put_id", compute_put_id, METH_VARARGS,
"Return the object ID for a put call within a task."},
{"task_from_string", PyTask_from_string, METH_VARARGS,
"Creates a Python PyTask object from a string representation of "
"task_spec."},
{"task_to_string", PyTask_to_string, METH_VARARGS,
"Translates a PyTask python object to a byte string."},
{NULL} /* Sentinel */
};
#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
"common", /* m_name */
"A module for common types. This is used for testing.", /* m_doc */
0, /* m_size */
common_methods, /* m_methods */
NULL, /* m_reload */
NULL, /* m_traverse */
NULL, /* m_clear */
NULL, /* m_free */
};
#endif
#if PY_MAJOR_VERSION >= 3
#define INITERROR return NULL
#else
#define INITERROR return
#endif
#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
#define PyMODINIT_FUNC void
#endif
#if PY_MAJOR_VERSION >= 3
#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
#else
#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
#endif
MOD_INIT(common) {
PyObject *m;
if (PyType_Ready(&PyTaskType) < 0) {
INITERROR;
}
if (PyType_Ready(&PyObjectIDType) < 0) {
INITERROR;
}
#if PY_MAJOR_VERSION >= 3
m = PyModule_Create(&moduledef);
#else
m = Py_InitModule3("common", common_methods,
"A module for common types. This is used for testing.");
#endif
init_pickle_module();
Py_INCREF(&PyTaskType);
PyModule_AddObject(m, "Task", (PyObject *) &PyTaskType);
Py_INCREF(&PyObjectIDType);
PyModule_AddObject(m, "ObjectID", (PyObject *) &PyObjectIDType);
char common_error[] = "common.error";
CommonError = PyErr_NewException(common_error, NULL, NULL);
Py_INCREF(CommonError);
PyModule_AddObject(m, "common_error", CommonError);
#if PY_MAJOR_VERSION >= 3
return m;
#endif
}
src/common/lib/python/setup.py
-12
View File
@@ -1,12 +0,0 @@
from setuptools import setup, find_packages, Extension
common_module = Extension("common",
sources=["common_module.c", "common_extension.c"],
include_dirs=["../../", "../../thirdparty"],
extra_objects=["../../build/libcommon.a"],
extra_compile_args=["--std=c99", "-Werror"])
setup(name="Common",
version="0.0.1",
description="Common library for Ray",
ext_modules=[common_module])
src/common/redis_module/CMakeLists.txt
+17
View File
@@ -0,0 +1,17 @@
cmake_minimum_required(VERSION 2.8)
project(ray_redis_module)
if(APPLE)
set(REDIS_MODULE_CFLAGS -W -Wall -dynamic -fno-common -g -ggdb -std=c99 -O2)
set(REDIS_MODULE_LDFLAGS "-undefined dynamic_lookup")
set(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
else()
set(REDIS_MODULE_CFLAGS -W -Wall -fno-common -g -ggdb -std=c99 -O2)
set(REDIS_MODULE_LDFLAGS -shared)
endif()
add_library(ray_redis_module SHARED ray_redis_module.c)
target_compile_options(ray_redis_module PUBLIC ${REDIS_MODULE_CFLAGS} -fPIC)
target_link_libraries(ray_redis_module ${REDIS_MODULE_LDFLAGS})
src/common/redis_module/Makefile
-28
View File
@@ -1,28 +0,0 @@
# find the OS
uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
# Compile flags for linux / osx
ifeq ($(uname_S),Linux)
SHOBJ_CFLAGS ?= -W -Wall -fno-common -g -ggdb -std=c99 -O2
SHOBJ_LDFLAGS ?= -shared
else
SHOBJ_CFLAGS ?= -W -Wall -dynamic -fno-common -g -ggdb -std=c99 -O2
SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup
endif
SHOBJ_CFLAGS += -I../thirdparty
.SUFFIXES: .c .so .xo .o
all: ray_redis_module.so
.c.xo:
$(CC) -I. -I.. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
ray_redis_module.xo: redismodule.h
ray_redis_module.so: ray_redis_module.xo
$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
clean:
rm -rf *.xo *.so
src/common/test/run_tests.sh
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
# This needs to be run in the build tree, which is normally ray/build
# Cause the script to exit if a single command fails.
set -e
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
sleep 1s
./src/common/common_tests
./src/common/db_tests
./src/common/io_tests
./src/common/task_tests
./src/common/redis_tests
./src/common/task_table_tests
./src/common/object_table_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
src/common/test/run_valgrind.sh
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
# This needs to be run in the build tree, which is normally ray/build
# Cause the script to exit if a single command fails.
set -e
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
sleep 1s
valgrind --leak-check=full --error-exitcode=1 ./src/common/common_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/db_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/io_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/task_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/redis_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/task_table_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/object_table_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
src/global_scheduler/CMakeLists.txt
+8
View File
@@ -0,0 +1,8 @@
cmake_minimum_required(VERSION 2.8)
project(global_scheduler)
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
add_executable(global_scheduler global_scheduler.c global_scheduler_algorithm.c)
target_link_libraries(global_scheduler common ${HIREDIS_LIB})
src/global_scheduler/Makefile
-12
View File
@@ -1,12 +0,0 @@
CC = gcc
CFLAGS = -g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -I. -I../common -I../common/thirdparty -I../common/state
BUILD = build
all: $(BUILD)/global_scheduler
$(BUILD)/global_scheduler: global_scheduler.c global_scheduler_algorithm.c
$(CC) $(CFLAGS) -o $@ global_scheduler.c global_scheduler_algorithm.c ../common/build/libcommon.a ../common/thirdparty/hiredis/libhiredis.a
clean:
rm -f *.o
rm -rf $(BUILD)/*
src/global_scheduler/global_scheduler_algorithm.c
+1 -1
View File
@@ -2,7 +2,7 @@
#include "object_info.h"
#include "task.h"
#include "task_table.h"
#include "state/task_table.h"
#include "global_scheduler_algorithm.h"
numbuf/.clang-format → src/numbuf/.clang-format
View File
src/numbuf/CMakeLists.txt
+81
View File
@@ -0,0 +1,81 @@
cmake_minimum_required(VERSION 2.8)
project(numbuf)
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
option(HAS_PLASMA
"Are we linking with the plasma object store? Recommended if numbuf is used as part of ray."
ON)
if(HAS_PLASMA)
add_definitions(-DHAS_PLASMA)
endif()
find_package(NumPy REQUIRED)
if(APPLE)
set(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
endif(APPLE)
include_directories("${PYTHON_INCLUDE_DIRS}")
include_directories("${NUMPY_INCLUDE_DIR}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
if(UNIX AND NOT APPLE)
link_libraries(rt)
endif()
set(ARROW_DIR "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/"
CACHE STRING "Path of the arrow source directory")
set(ARROW_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a"
CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)")
set(ARROW_IO_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow_io.a"
CACHE STRING "Path to libarrow_io.a (needs to be changed if arrow is build in debug mode)")
set(ARROW_IPC_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow_ipc.a"
CACHE STRING "Path to libarrow_ipc.a (needs to be changed if arrow is build in debug mode)")
include_directories("${ARROW_DIR}/cpp/src/")
include_directories("cpp/src/")
include_directories("python/src/")
if(HAS_PLASMA)
include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/build/flatcc-prefix/src/flatcc/include")
endif()
add_definitions(-fPIC)
add_library(numbuf SHARED
cpp/src/numbuf/tensor.cc
cpp/src/numbuf/dict.cc
cpp/src/numbuf/sequence.cc
python/src/pynumbuf/numbuf.cc
python/src/pynumbuf/adapters/numpy.cc
python/src/pynumbuf/adapters/python.cc)
get_filename_component(PYTHON_SHARED_LIBRARY ${PYTHON_LIBRARIES} NAME)
if(APPLE)
add_custom_command(TARGET numbuf
POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libnumbuf.so)
endif(APPLE)
if(APPLE)
target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES})
else()
target_link_libraries(numbuf -Wl,--whole-archive ${ARROW_LIB} -Wl,--no-whole-archive ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES})
endif()
if(HAS_PLASMA)
target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES} plasma_lib common ${FLATBUFFERS_STATIC_LIB})
else()
target_link_libraries(numbuf ${ARROW_LIB} ${ARROW_IO_LIB} ${ARROW_IPC_LIB} ${PYTHON_LIBRARIES})
endif()
install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/)
numbuf/cmake/Modules/FindNumPy.cmake → src/numbuf/cmake/Modules/FindNumPy.cmake
View File
numbuf/cpp/src/numbuf/dict.cc → src/numbuf/cpp/src/numbuf/dict.cc
View File
numbuf/cpp/src/numbuf/dict.h → src/numbuf/cpp/src/numbuf/dict.h
View File
numbuf/cpp/src/numbuf/sequence.cc → src/numbuf/cpp/src/numbuf/sequence.cc
View File
numbuf/cpp/src/numbuf/sequence.h → src/numbuf/cpp/src/numbuf/sequence.h
View File
numbuf/cpp/src/numbuf/tensor.cc → src/numbuf/cpp/src/numbuf/tensor.cc
View File
numbuf/cpp/src/numbuf/tensor.h → src/numbuf/cpp/src/numbuf/tensor.h
View File
numbuf/numbuf/__init__.py → src/numbuf/numbuf/__init__.py
+1 -1
View File
@@ -11,7 +11,7 @@ If you are using Anaconda, try fixing this problem by running:
"""
try:
from numbuf.libnumbuf import *
from .libnumbuf import *
except ImportError as e:
if hasattr(e, "msg") and isinstance(e.msg, str) and "GLIBCXX" in e.msg:
# This code path should be taken with Python 3.
numbuf/python/src/pynumbuf/adapters/numpy.cc → src/numbuf/python/src/pynumbuf/adapters/numpy.cc
View File
numbuf/python/src/pynumbuf/adapters/numpy.h → src/numbuf/python/src/pynumbuf/adapters/numpy.h
View File
numbuf/python/src/pynumbuf/adapters/python.cc → src/numbuf/python/src/pynumbuf/adapters/python.cc
View File
numbuf/python/src/pynumbuf/adapters/python.h → src/numbuf/python/src/pynumbuf/adapters/python.h
View File
numbuf/python/src/pynumbuf/adapters/scalars.h → src/numbuf/python/src/pynumbuf/adapters/scalars.h
View File
numbuf/python/src/pynumbuf/memory.h → src/numbuf/python/src/pynumbuf/memory.h
View File
numbuf/python/src/pynumbuf/numbuf.cc → src/numbuf/python/src/pynumbuf/numbuf.cc
View File
numbuf/python/test/runtest.py → src/numbuf/python/test/runtest.py
View File
numbuf/thirdparty/build_thirdparty.sh → src/numbuf/thirdparty/build_thirdparty.sh
Vendored
View File
numbuf/thirdparty/download_thirdparty.sh → src/numbuf/thirdparty/download_thirdparty.sh
Vendored
View File
numbuf/vsprojects/numbuf.vcxproj → src/numbuf/vsprojects/numbuf.vcxproj
View File
numbuf/vsprojects/numbuf.vcxproj.filters → src/numbuf/vsprojects/numbuf.vcxproj.filters
View File
src/photon/CMakeLists.txt
+16 -66
View File
@@ -2,58 +2,8 @@ cmake_minimum_required(VERSION 2.8)
project(photon)
message(STATUS "Trying custom approach for finding Python.")
# Start off by figuring out which Python executable to use.
find_program(CUSTOM_PYTHON_EXECUTABLE python)
message(STATUS "Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}")
execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print('python' + sys.version[0:3])"
OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
message(STATUS "PYTHON_LIBRARY_NAME: " ${PYTHON_LIBRARY_NAME})
# Now find the Python include directories.
execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())"
OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE)
message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
# Now find the Python libraries. We'll start by looking near the Python
# executable. If that fails, then we'll look near the Python include
# directories.
execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c "import sys; print(sys.exec_prefix)"
OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE)
message(STATUS "PYTHON_PREFIX: " ${PYTHON_PREFIX})
# The name ending in "m" is for miniconda.
FIND_LIBRARY(PYTHON_LIBRARIES
NAMES "${PYTHON_LIBRARY_NAME}" "${PYTHON_LIBRARY_NAME}m"
HINTS "${PYTHON_PREFIX}"
PATH_SUFFIXES "lib" "libs"
NO_DEFAULT_PATH)
message(STATUS "PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES})
# If that failed, perhaps because the user is in a virtualenv, search around
# the Python include directories.
if(NOT PYTHON_LIBRARIES)
message(STATUS "Failed to find PYTHON_LIBRARIES near the Python executable, so now looking near the Python include directories.")
# The name ending in "m" is for miniconda.
FIND_LIBRARY(PYTHON_LIBRARIES
NAMES "${PYTHON_LIBRARY_NAME}" "${PYTHON_LIBRARY_NAME}m"
HINTS "${PYTHON_INCLUDE_DIRS}/../.."
PATH_SUFFIXES "lib" "libs"
NO_DEFAULT_PATH)
message(STATUS "PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES})
endif()
# If we found the Python libraries and the include directories, then continue
# on. If not, then try find_package as a last resort, but it probably won't
# work.
if(PYTHON_LIBRARIES AND PYTHON_INCLUDE_DIRS)
message(STATUS "The custom approach for finding Python succeeded.")
SET(PYTHONLIBS_FOUND TRUE)
else()
message(WARNING "The custom approach for finding Python failed. Defaulting to find_package.")
find_package(PythonInterp REQUIRED)
find_package(PythonLibs ${PYTHON_VERSION_STRING} EXACT REQUIRED)
set(CUSTOM_PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE})
endif()
message(STATUS "Using CUSTOM_PYTHON_EXECUTABLE: " ${CUSTOM_PYTHON_EXECUTABLE})
message(STATUS "Using PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES})
message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
# Recursively include common
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
if(APPLE)
SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
@@ -61,24 +11,15 @@ endif(APPLE)
include_directories("${PYTHON_INCLUDE_DIRS}")
set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} --std=c99 -Werror")
# set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} --std=c99 -Werror")
if(UNIX AND NOT APPLE)
link_libraries(rt)
endif()
set(PHOTON_CLIENT_LIB "${CMAKE_SOURCE_DIR}/build/photon_client.a"
CACHE STRING "Path to photon_client.a")
set(COMMON_LIB "${CMAKE_SOURCE_DIR}/../common/build/libcommon.a"
CACHE STRING "Path to libcommon.a")
include_directories("${CMAKE_SOURCE_DIR}/")
include_directories("${CMAKE_SOURCE_DIR}/../")
include_directories("${CMAKE_SOURCE_DIR}/../common/")
include_directories("${CMAKE_SOURCE_DIR}/../common/thirdparty/")
include_directories("${CMAKE_SOURCE_DIR}/../common/lib/python/")
include_directories("${CMAKE_SOURCE_DIR}/../plasma/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/")
add_library(photon SHARED
photon_extension.c
@@ -90,6 +31,15 @@ if(APPLE)
POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libphoton.so)
endif(APPLE)
target_link_libraries(photon ${PHOTON_CLIENT_LIB} ${COMMON_LIB} ${PYTHON_LIBRARIES})
add_library(photon_client STATIC photon_client.c)
target_link_libraries(photon photon_client ${COMMON_LIB} ${PYTHON_LIBRARIES})
add_executable(photon_scheduler photon_scheduler.c photon_algorithm.c)
target_link_libraries(photon_scheduler photon_client common ${HIREDIS_LIB} plasma_lib)
add_executable(photon_tests test/photon_tests.c photon_scheduler.c photon_algorithm.c )
target_link_libraries(photon_tests photon_client common ${HIREDIS_LIB} plasma_lib)
target_compile_options(photon_tests PUBLIC "-DPHOTON_TEST")
install(TARGETS photon DESTINATION ${CMAKE_SOURCE_DIR}/photon)
src/photon/Makefile
-43
View File
@@ -1,43 +0,0 @@
CC = gcc
CFLAGS = -g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -I.. -I../common -I../common/thirdparty -I../plasma/ -fPIC
TEST_CFLAGS = -DPHOTON_TEST=1 -I.
BUILD = build
all: $(BUILD)/photon_scheduler $(BUILD)/photon_client.a
$(BUILD)/photon_tests: test/photon_tests.c photon.h photon_scheduler.h photon_scheduler.c photon_algorithm.h photon_algorithm.c photon_client.h photon_client.c common
$(CC) $(CFLAGS) $(TEST_CFLAGS) -o $@ test/photon_tests.c photon_scheduler.c photon_algorithm.c photon_client.c ../common/build/libcommon.a ../common/thirdparty/hiredis/libhiredis.a -I../common/thirdparty/ -I../common/ ../plasma/build/libplasma_client.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a -I../plasma/
$(BUILD)/photon_client.a: photon_client.o
ar rcs $(BUILD)/photon_client.a photon_client.o
$(BUILD)/photon_scheduler: photon.h photon_scheduler.c photon_algorithm.c common
$(CC) $(CFLAGS) -o $@ photon_scheduler.c photon_algorithm.c ../common/build/libcommon.a ../common/thirdparty/hiredis/libhiredis.a -I../common/thirdparty/ -I../common/ ../plasma/build/libplasma_client.a ../common/build/flatcc-prefix/src/flatcc/lib/libflatcc.a
common: FORCE
cd ../common; make; cd build; cmake ..; make
clean:
cd ../common; make clean
rm -rf $(BUILD)/*
rm -f *.o
# Set the request timeout low and logging level at FATAL for testing purposes.
test: CFLAGS += -DRAY_TIMEOUT=50 -DRAY_COMMON_LOG_LEVEL=4
test: $(BUILD)/photon_tests FORCE
../common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ../common/redis_module/ray_redis_module.so &
../plasma/build/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 &
sleep 0.5s
./build/photon_tests
../common/thirdparty/redis/src/redis-cli shutdown
killall plasma_store
valgrind: test
../common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ../common/redis_module/ray_redis_module.so &
../plasma/build/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 &
sleep 0.5s
valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 ./build/photon_tests
../common/thirdparty/redis/src/redis-cli shutdown
killall plasma_store
FORCE:
src/photon/photon/__init__.py
-6
View File
@@ -1,6 +0,0 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from .libphoton import *
from .photon_services import *
src/photon/test/run_tests.sh
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
# This needs to be run in the build tree, which is normally ray/build
# Cause the script to exit if a single command fails.
set -e
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
sleep 1s
./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 &
sleep 0.5s
./src/photon/photon_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
killall plasma_store

Some files were not shown because too many files have changed in this diff Show More