From e1a74eadbe0245149ecf118385e27b702b3d4653 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Fri, 8 Jul 2016 20:03:21 -0700 Subject: [PATCH] remove installation of dependencies from setup script (#239) --- .travis.yml | 1 + README.md | 53 +++++++++++++++++- doc/install-on-macosx.md | 45 ++++++++++++++++ doc/install-on-ubuntu.md | 43 +++++++++++++++ ...oad-and-setup.md => install-on-windows.md} | 33 +----------- doc/introduction.md | 54 ------------------- install-dependencies.sh | 41 ++++++++++++++ requirements.txt | 9 ---- scripts/cluster.py | 1 + setup.sh | 25 --------- 10 files changed, 185 insertions(+), 120 deletions(-) create mode 100644 doc/install-on-macosx.md create mode 100644 doc/install-on-ubuntu.md rename doc/{download-and-setup.md => install-on-windows.md} (50%) delete mode 100644 doc/introduction.md create mode 100755 install-dependencies.sh delete mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index 0c77afa93..4a3edc383 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ before_install: - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then sudo add-apt-repository --yes ppa:kalakris/cmake ; fi install: + - ./install-dependencies.sh - ./setup.sh - ./build.sh diff --git a/README.md b/README.md index 7302c53ae..ad76c51b3 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,55 @@ Ray is an experimental distributed execution framework with a Python-like programming model. It is under development and not ready for general use. -Read this [introduction to Ray](doc/introduction.md). +The goal of Ray is to make it easy to write machine learning applications that +run on a cluster while providing the development and debugging experience of +working on a single machine. + +Before jumping into the details, here's a simple Python example for doing a +Monte Carlo estimation of pi (using multiple cores or potentially multiple +machines). +```python +import ray +import functions # See definition below + +results = [] +for _ in range(10): + results.append(functions.estimate_pi(100)) +estimate = np.mean([ray.get(ref) for ref in results]) +print "Pi is approximately {}.".format(estimate) +``` + +This assumes that we've defined the file `functions.py` as below. +```python +import ray +import numpy as np + +@ray.remote([int], [float]) +def estimate_pi(n): + x = np.random.uniform(size=n) + y = np.random.uniform(size=n) + return 4 * np.mean(x ** 2 + y ** 2 < 1) +``` + +Within the for loop, each call to `functions.estimate_pi(100)` sends a message +to the scheduler asking it to schedule the task of running +`functions.estimate_pi` with the argument `100`. This call returns right away +without waiting for the actual estimation of pi to take place. Instead of +returning a float, it returns an **object reference**, which represents the +eventual output of the computation. + +The call to `ray.get(ref)` takes an object reference and returns the actual +estimate of pi (waiting until the computation has finished if necessary). + +## Next Steps + +- Installation on [Ubuntu](doc/install-on-ubuntu.md), [Mac OS X](doc/install-on-macosx.md), [Windows](doc/install-on-windows.md) +- [Basic Usage](doc/basic-usage.md) +- [Tutorial](doc/tutorial.md) +- [About the System](doc/about-the-system.md) +- [Using Ray on a Cluster](doc/using-ray-on-a-cluster.md) + +## Example Applications + +- [Hyperparameter Optimization](examples/hyperopt/README.md) +- [Batch L-BFGS](examples/lbfgs/README.md) diff --git a/doc/install-on-macosx.md b/doc/install-on-macosx.md new file mode 100644 index 000000000..7d8209e2f --- /dev/null +++ b/doc/install-on-macosx.md @@ -0,0 +1,45 @@ +## Installation on Mac OS X + +Ray must currently be built from source. + +### Clone the Ray repository + +``` +git clone https://github.com/amplab/ray.git +``` + +### Dependencies + +First install the dependencies using brew. + +``` +brew update +brew install git cmake automake autoconf libtool boost libjpeg graphviz +sudo easy_install pip +sudo pip install ipython --user +sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0-alpha-2 boto3 botocore Pillow colorama graphviz --ignore-installed six +``` + +### Build + +Then run the setup scripts. + +``` +cd ray +./setup.sh # This builds all necessary third party libraries (e.g., gRPC and Apache Arrow). +./build.sh # This builds Ray. +source setup-env.sh # This adds Ray to your Python path. +``` + +For convenience, you may also want to add the line `source +"$RAY_ROOT/setup-env.sh"` to your `~/.bashrc` file manually, where `$RAY_ROOT` +is the Ray directory (e.g., `/home/ubuntu/ray`). + +### Test if the installation succeeded + +To test if the installation was successful, try running some tests. + +``` +python test/runtest.py # This tests basic functionality. +python test/array_test.py # This tests some array libraries. +``` diff --git a/doc/install-on-ubuntu.md b/doc/install-on-ubuntu.md new file mode 100644 index 000000000..1fb93ea7b --- /dev/null +++ b/doc/install-on-ubuntu.md @@ -0,0 +1,43 @@ +## Installation on Ubuntu + +Ray must currently be built from source. + +### Clone the Ray repository + +``` +git clone https://github.com/amplab/ray.git +``` + +### Dependencies + +First install the dependencies. + +``` +sudo apt-get update +sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip libjpeg8-dev graphviz +sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0-alpha-2 boto3 botocore Pillow colorama graphviz +``` + +### Build + +Then run the setup scripts. + +``` +cd ray +./setup.sh # This builds all necessary third party libraries (e.g., gRPC and Apache Arrow). +./build.sh # This builds Ray. +source setup-env.sh # This adds Ray to your Python path. +``` + +For convenience, you may also want to add the line `source +"$RAY_ROOT/setup-env.sh"` to your `~/.bashrc` file manually, where `$RAY_ROOT` +is the Ray directory (e.g., `/home/ubuntu/ray`). + +### Test if the installation succeeded + +To test if the installation was successful, try running some tests. + +``` +python test/runtest.py # This tests basic functionality. +python test/array_test.py # This tests some array libraries. +``` diff --git a/doc/download-and-setup.md b/doc/install-on-windows.md similarity index 50% rename from doc/download-and-setup.md rename to doc/install-on-windows.md index 2090abadb..58e82452f 100644 --- a/doc/download-and-setup.md +++ b/doc/install-on-windows.md @@ -1,33 +1,4 @@ -## Download and Setup - -Ray must currently be built from source. - -### Clone the Ray repository - -``` -git clone https://github.com/amplab/ray.git -``` -These instructions will install the latest master branch for Ray. - -### Installation for Ubuntu and Mac OS X - -For convenience, we provide a setup script that pulls the necessary -dependencies. - -``` -cd ray -./setup.sh # This builds all necessary third party libraries (e.g., gRPC and Apache Arrow). It will require a sudo password. -./build.sh # This builds Ray. -source setup-env.sh # This adds Ray to your Python path. -``` - -For convenience, you may also want to add the line `source -"$RAY_ROOT/setup-env.sh"` to your `~/.bashrc` file manually, where `$RAY_ROOT` -is the Ray directory (e.g., `/home/ubuntu/ray`). - -To test if the installation was successful, try running some tests. - -### Installation for Windows +## Installation on Windows Ray currently does not run on Windows. However, it can be compiled with the following instructions. @@ -45,7 +16,7 @@ re-running the batch file. ### Test if the installation succeeded -Try running some tests. +To test if the installation was successful, try running some tests. ``` python test/runtest.py # This tests basic functionality. diff --git a/doc/introduction.md b/doc/introduction.md deleted file mode 100644 index 19fbbebd8..000000000 --- a/doc/introduction.md +++ /dev/null @@ -1,54 +0,0 @@ -## Introduction - -The goal of Ray is to make it easy to write machine learning applications that -run on a cluster while providing the development and debugging experience of -working on a single machine. - -Before jumping into the details, here's a simple Python example for doing a -Monte Carlo estimation of pi (using multiple cores or potentially multiple -machines). -```python -import ray -import functions # See definition below - -results = [] -for _ in range(10): - results.append(functions.estimate_pi(100)) -estimate = np.mean([ray.get(ref) for ref in results]) -print "Pi is approximately {}.".format(estimate) -``` - -This assumes that we've defined the file `functions.py` as below. -```python -import ray -import numpy as np - -@ray.remote([int], [float]) -def estimate_pi(n): - x = np.random.uniform(size=n) - y = np.random.uniform(size=n) - return 4 * np.mean(x ** 2 + y ** 2 < 1) -``` - -Within the for loop, each call to `functions.estimate_pi(100)` sends a message -to the scheduler asking it to schedule the task of running -`functions.estimate_pi` with the argument `100`. This call returns right away -without waiting for the actual estimation of pi to take place. Instead of -returning a float, it returns an **object reference**, which represents the -eventual output of the computation. - -The call to `ray.get(ref)` takes an object reference and returns the actual -estimate of pi (waiting until the computation has finished if necessary). - -## Next Steps - -- [Download and Setup](download-and-setup.md) -- [Basic Usage](basic-usage.md) -- [Tutorial](tutorial.md) -- [About the System](about-the-system.md) -- [Using Ray on a Cluster](using-ray-on-a-cluster.md) - -## Example Applications - -- [Hyperparameter Optimization](../examples/hyperopt/README.md) -- [Batch L-BFGS](../examples/lbfgs/README.md) diff --git a/install-dependencies.sh b/install-dependencies.sh new file mode 100755 index 000000000..40d8afd96 --- /dev/null +++ b/install-dependencies.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +platform="unknown" +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + echo "Platform is linux." + platform="linux" +elif [[ "$unamestr" == "Darwin" ]]; then + echo "Platform is macosx." + platform="macosx" +else + echo "Unrecognized platform." + exit 1 +fi + +if [[ $platform == "macosx" ]]; then + # check that brew is installed + which -s brew + if [[ $? != 0 ]]; then + echo "Could not find brew, please install brew (see http://brew.sh/)." + exit 1 + else + echo "Updating brew." + brew update + fi +fi + +if [[ $platform == "linux" ]]; then + # These commands must be kept in sync with the installation instructions. + sudo apt-get update + sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip libjpeg8-dev graphviz + sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0-alpha-2 boto3 botocore Pillow colorama graphviz +elif [[ $platform == "macosx" ]]; then + # These commands must be kept in sync with the installation instructions. + brew install git cmake automake autoconf libtool boost libjpeg graphviz + sudo easy_install pip + sudo pip install ipython --user + sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0-alpha-2 boto3 botocore Pillow colorama graphviz --ignore-installed six +fi diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 95564e4d4..000000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -typing -funcsigs -subprocess32 -protobuf==3.0.0-alpha-2 -boto3 -botocore -Pillow -colorama -graphviz diff --git a/scripts/cluster.py b/scripts/cluster.py index c92bb86f2..f4cdae33d 100644 --- a/scripts/cluster.py +++ b/scripts/cluster.py @@ -51,6 +51,7 @@ def _install_ray(node_ip_addresses, username, key_file, installation_directory): cd "{}" && git clone "https://github.com/amplab/ray"; cd ray; + ./install-dependencies.sh; ./setup.sh; ./build.sh """.format(installation_directory, installation_directory) diff --git a/setup.sh b/setup.sh index 5a5beb0a1..71482975e 100755 --- a/setup.sh +++ b/setup.sh @@ -15,31 +15,6 @@ else exit 1 fi -if [[ $platform == "macosx" ]]; then - # check that brew is installed - which -s brew - if [[ $? != 0 ]]; then - echo "Could not find brew, please install brew (see http://brew.sh/)." - exit 1 - else - echo "Updating brew." - brew update - fi -fi - - -if [[ $platform == "linux" ]]; then - sudo apt-get update - sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip libjpeg8-dev graphviz - sudo pip install ipython - sudo pip install -r requirements.txt -elif [[ $platform == "macosx" ]]; then - brew install git cmake automake autoconf libtool boost libjpeg graphviz - sudo easy_install pip - sudo pip install numpy - sudo pip install ipython --user - sudo pip install -r requirements.txt --ignore-installed six -fi pushd "$ROOT_DIR/thirdparty" ./download_thirdparty.sh ./build_thirdparty.sh