mirror of
https://github.com/wassname/ray.git
synced 2026-09-09 11:32:43 +08:00
[projects] Project examples and documentation (#5407)
This commit is contained in:
Philipp Moritz
@@ -14,5 +14,6 @@ redis
|
||||
setproctitle
|
||||
sphinx
|
||||
sphinx-click
|
||||
sphinx-jsonschema
|
||||
sphinx_rtd_theme
|
||||
pandas
|
||||
|
||||
@@ -70,6 +70,7 @@ extensions = [
|
||||
'sphinx.ext.viewcode',
|
||||
'sphinx.ext.napoleon',
|
||||
'sphinx_click.ext',
|
||||
'sphinx-jsonschema',
|
||||
]
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
|
||||
@@ -218,6 +218,7 @@ The following are good places to discuss Ray.
|
||||
|
||||
distributed_training.rst
|
||||
pandas_on_ray.rst
|
||||
projects.rst
|
||||
signals.rst
|
||||
async_api.rst
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
Ray Projects (Experimental)
|
||||
===========================
|
||||
|
||||
Ray projects make it easy to package a Ray application so it can be
|
||||
rerun later in the same environment. They allow for the sharing and
|
||||
reliable reuse of existing code.
|
||||
|
||||
Quick start (CLI)
|
||||
-----------------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# Creates a project in the current directory. It will create a
|
||||
# project.yaml defining the code and environment and a cluster.yaml
|
||||
# describing the cluster configuration. Both will be created in the
|
||||
# .rayproject subdirectory of the current directory.
|
||||
$ ray project create <project-name>
|
||||
|
||||
# Create a new session from the given project.
|
||||
# Launch a cluster and run the appropriate command.
|
||||
$ ray session start
|
||||
|
||||
# Open a console for the given session.
|
||||
$ ray session attach
|
||||
|
||||
# Stop the given session and all of its worker nodes. The nodes/clusters
|
||||
# are not actually terminated.
|
||||
$ ray session stop
|
||||
|
||||
Examples
|
||||
--------
|
||||
- `Open Tacotron <https://github.com/ray-project/ray/blob/master/python/ray/projects/examples/open-tacotron/.rayproject/project.yaml>`__:
|
||||
A TensorFlow implementation of Google's Tacotron speech synthesis with pre-trained model (unofficial)
|
||||
- `PyTorch Transformers <https://github.com/ray-project/ray/blob/master/python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml>`__:
|
||||
A library of state-of-the-art pretrained models for Natural Language Processing (NLP)
|
||||
|
||||
Project file format (project.yaml)
|
||||
----------------------------------
|
||||
|
||||
A project file contains everything required to run a project.
|
||||
This includes a cluster configuration, the environment and dependencies
|
||||
for the application, and the specific inputs used to run the project.
|
||||
|
||||
Here is an example for a minimal project format:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
name: test-project
|
||||
description: "This is a simple test project"
|
||||
repo: https://github.com/ray-project/ray
|
||||
|
||||
# Cluster to be instantiated by default when starting the project.
|
||||
cluster: .rayproject/cluster.yaml
|
||||
|
||||
# Commands/information to build the environment, once the cluster is
|
||||
# instantiated. This can include the versions of python libraries etc.
|
||||
# It can be specified as a Python requirements.txt, a conda environment,
|
||||
# a Dockerfile, or a shell script to run to set up the libraries.
|
||||
environment:
|
||||
requirements: requirements.txt
|
||||
|
||||
# List of commands that can be executed once the cluster is instantiated
|
||||
# and the environment is set up.
|
||||
# A command can also specify a cluster that overwrites the default cluster.
|
||||
commands:
|
||||
- name: test
|
||||
command: python test.py
|
||||
|
||||
Project files have to adhere to the following schema:
|
||||
|
||||
.. jsonschema:: ../../python/ray/projects/schema.json
|
||||
|
||||
Cluster file format (cluster.yaml)
|
||||
----------------------------------
|
||||
|
||||
This is the same as for the autoscaler, see
|
||||
`Cluster Launch page <autoscaling.html>`_.
|
||||
@@ -0,0 +1,18 @@
|
||||
# This file is generated by `ray project create`
|
||||
|
||||
# A unique identifier for the head node and workers of this cluster.
|
||||
cluster_name: open-tacotron
|
||||
|
||||
# The maximum number of workers nodes to launch in addition to the head
|
||||
# node. This takes precedence over min_workers. min_workers defaults to 0.
|
||||
max_workers: 1
|
||||
|
||||
# Cloud-provider specific configuration.
|
||||
provider:
|
||||
type: aws
|
||||
region: us-west-2
|
||||
availability_zone: us-west-2a
|
||||
|
||||
# How Ray will authenticate with newly launched nodes.
|
||||
auth:
|
||||
ssh_user: ubuntu
|
||||
@@ -0,0 +1,17 @@
|
||||
# This file is generated by `ray project create`
|
||||
|
||||
name: open-tacotron
|
||||
description: "A TensorFlow implementation of Google's Tacotron speech synthesis with pre-trained model (unofficial)"
|
||||
repo: https://github.com/keithito/tacotron
|
||||
|
||||
cluster: .rayproject/cluster.yaml
|
||||
|
||||
environment:
|
||||
requirements: requirements.txt
|
||||
|
||||
shell:
|
||||
- curl http://data.keithito.com/data/speech/tacotron-20180906.tar.gz | tar xzC /tmp
|
||||
|
||||
commands:
|
||||
- name: serve
|
||||
command: python demo_server.py --checkpoint /tmp/tacotron-20180906/model.ckpt
|
||||
@@ -0,0 +1,11 @@
|
||||
# Adapted from https://github.com/keithito/tacotron/blob/master/requirements.txt
|
||||
# Note: this doesn't include tensorflow or tensorflow-gpu because the package you need to install
|
||||
# depends on your platform. It is assumed you have already installed tensorflow.
|
||||
falcon==1.2.0
|
||||
inflect==0.2.5
|
||||
librosa==0.5.1
|
||||
matplotlib==2.0.2
|
||||
numpy==1.14.3
|
||||
scipy==0.19.0
|
||||
tqdm==4.11.2
|
||||
Unidecode==0.4.20
|
||||
@@ -0,0 +1,18 @@
|
||||
# This file is generated by `ray project create`
|
||||
|
||||
# An unique identifier for the head node and workers of this cluster.
|
||||
cluster_name: pytorch-transformers
|
||||
|
||||
# The maximum number of workers nodes to launch in addition to the head
|
||||
# node. This takes precedence over min_workers. min_workers default to 0.
|
||||
max_workers: 1
|
||||
|
||||
# Cloud-provider specific configuration.
|
||||
provider:
|
||||
type: aws
|
||||
region: us-west-2
|
||||
availability_zone: us-west-2a
|
||||
|
||||
# How Ray will authenticate with newly launched nodes.
|
||||
auth:
|
||||
ssh_user: ubuntu
|
||||
@@ -0,0 +1,30 @@
|
||||
# This file is generated by `ray project create`
|
||||
|
||||
name: pytorch-transformers
|
||||
description: "A library of state-of-the-art pretrained models for Natural Language Processing (NLP)"
|
||||
repo: https://github.com/huggingface/pytorch-transformers
|
||||
|
||||
cluster: .rayproject/cluster.yaml
|
||||
|
||||
environment:
|
||||
requirements: requirements.txt
|
||||
|
||||
commands:
|
||||
- name: train_sst_2
|
||||
command: |
|
||||
wget https://raw.githubusercontent.com/nyu-mll/GLUE-baselines/master/download_glue_data.py && \
|
||||
python download_glue_data.py -d /tmp -t SST && \
|
||||
python ./examples/run_glue.py \
|
||||
--model_type bert \
|
||||
--model_name_or_path bert-base-uncased \
|
||||
--task_name SST-2 \
|
||||
--do_train \
|
||||
--do_eval \
|
||||
--do_lower_case \
|
||||
--data_dir /tmp/SST-2 \
|
||||
--max_seq_length 128 \
|
||||
--per_gpu_eval_batch_size=8 \
|
||||
--per_gpu_train_batch_size=8 \
|
||||
--learning_rate 2e-5 \
|
||||
--num_train_epochs 3.0 \
|
||||
--output_dir /tmp/output/
|
||||
@@ -0,0 +1,17 @@
|
||||
# Adapted from https://github.com/huggingface/pytorch-transformers/blob/master/requirements.txt
|
||||
# PyTorch
|
||||
torch>=1.0.0
|
||||
# progress bars in model download and training scripts
|
||||
tqdm
|
||||
# Accessing files from S3 directly.
|
||||
boto3
|
||||
# Used for downloading models over HTTP
|
||||
requests
|
||||
# For OpenAI GPT
|
||||
regex
|
||||
# For XLNet
|
||||
sentencepiece
|
||||
# TensorBoard visualization
|
||||
tensorboardX
|
||||
# Pytorch transformers
|
||||
pytorch_transformers
|
||||
Reference in New Issue
Block a user