wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-09 11:32:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

[JavaWorker] Enable java worker support (#2094)

Browse Source 
* Enable java worker support
--------------------------
This commit includes a tailored version of the Java worker implementation from Ant Financial.
The changes for build system, python module, src module and arrow are in other commits, this commit consists of the following modules:
 - java/api: Ray API definition
 - java/common: utilities
 - java/hook: binary rewrite of the Java byte-code for remote execution
 - java/runtime-common: common implementation of the runtime in worker
 - java/runtime-dev: a pure-java mock implementation of the runtime for fast development
 - java/runtime-native: a native implementation of the runtime
 - java/test: various tests

Contributors for this work:
 Guyang Song, Peng Cao, Senlin Zhu,Xiaoying Chu, Yiming Yu, Yujie Liu, Zhenyu Guo

* change the format of java help document from markdown to RST

* update the vesion of Arrow for java worker

* adapt the new version of plasma java client from arrow which use byte[] instead of custom type

* add java worker test to ci

* add the example module for better usage guide
This commit is contained in:
Yujie Liu
2018-05-26 14:38:50 -07:00
committed by Philipp Moritz
parent 74cca3b284
commit a8d3c057c1
193 changed files with 22675 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
thirdparty/scripts/build_arrow.sh
Vendored
+14 -2
View File
@@ -15,6 +15,12 @@ else
fi
echo "Using Python executable $PYTHON_EXECUTABLE."
LANGUAGE="python"
if [[ -n "$2" ]]; then
LANGUAGE=$2
fi
echo "Build language is $LANGUAGE."
unamestr="$(uname)"
if [[ "$unamestr" == "Linux" ]]; then
@@ -44,10 +50,10 @@ if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]]; then
pushd $TP_DIR/build/arrow
git fetch origin master
# The PR for this commit is https://github.com/apache/arrow/pull/2073. We
# The PR for this commit is https://github.com/apache/arrow/pull/2065. We
# include the link here to make it easier to find the right commit because
# Arrow often rewrites git history and invalidates certain commits.
git checkout 1d9d8939efe54a640b46fcbe8e14e5b347c7a422
git checkout ce23c06469de9cf0c3e38e35cdb8d135f341b964
cd cpp
if [ ! -d "build" ]; then
@@ -55,6 +61,11 @@ if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]]; then
fi
cd build
BUILD_ARROW_PLASMA_JAVA_CLIENT=off
if [[ "$LANGUAGE" == "java" ]]; then
BUILD_ARROW_PLASMA_JAVA_CLIENT=on
fi
ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install
BOOST_ROOT=$TP_DIR/pkg/boost \
FLATBUFFERS_HOME=$FLATBUFFERS_HOME \
@@ -74,6 +85,7 @@ if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]]; then
-DARROW_WITH_LZ4=off \
-DARROW_WITH_ZLIB=off \
-DARROW_WITH_ZSTD=off \
-DARROW_PLASMA_JAVA_CLIENT=$BUILD_ARROW_PLASMA_JAVA_CLIENT \
..
make VERBOSE=1 -j$PARALLEL
make install
thirdparty/scripts/setup.sh
Vendored
+7 -1
View File
@@ -18,6 +18,12 @@ else
fi
echo "Using Python executable $PYTHON_EXECUTABLE."
LANGUAGE="python"
if [[ -n "$2" ]]; then
LANGUAGE=$2
fi
echo "Build language is $LANGUAGE."
unamestr="$(uname)"
##############################################
@@ -46,7 +52,7 @@ fi
##############################################
# arrow
##############################################
bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE
bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE $LANGUAGE
##############################################
# parquet (skipped as it is inlined in build_arrow.sh)