From c852213b8349b6b9e9e7353573e2259a1b9ef925 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Tue, 20 Aug 2019 20:49:15 -0700 Subject: [PATCH] [projects] Project examples and documentation (#5407) --- doc/requirements-doc.txt | 1 + doc/source/conf.py | 1 + doc/source/index.rst | 1 + doc/source/projects.rst | 77 +++++++++++++++++++ .../open-tacotron/.rayproject/cluster.yaml | 18 +++++ .../open-tacotron/.rayproject/project.yaml | 17 ++++ .../examples/open-tacotron/requirements.txt | 11 +++ .../.rayproject/cluster.yaml | 18 +++++ .../.rayproject/project.yaml | 30 ++++++++ .../pytorch-transformers/requirements.txt | 17 ++++ 10 files changed, 191 insertions(+) create mode 100644 doc/source/projects.rst create mode 100644 python/ray/projects/examples/open-tacotron/.rayproject/cluster.yaml create mode 100644 python/ray/projects/examples/open-tacotron/.rayproject/project.yaml create mode 100644 python/ray/projects/examples/open-tacotron/requirements.txt create mode 100644 python/ray/projects/examples/pytorch-transformers/.rayproject/cluster.yaml create mode 100644 python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml create mode 100644 python/ray/projects/examples/pytorch-transformers/requirements.txt diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt index 506a3540e..ecde5d566 100644 --- a/doc/requirements-doc.txt +++ b/doc/requirements-doc.txt @@ -14,5 +14,6 @@ redis setproctitle sphinx sphinx-click +sphinx-jsonschema sphinx_rtd_theme pandas diff --git a/doc/source/conf.py b/doc/source/conf.py index 994f30bc2..ee686073f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -70,6 +70,7 @@ extensions = [ 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', 'sphinx_click.ext', + 'sphinx-jsonschema', ] # Add any paths that contain templates here, relative to this directory. diff --git a/doc/source/index.rst b/doc/source/index.rst index 2fabc6acd..75ecc9474 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -218,6 +218,7 @@ The following are good places to discuss Ray. distributed_training.rst pandas_on_ray.rst + projects.rst signals.rst async_api.rst diff --git a/doc/source/projects.rst b/doc/source/projects.rst new file mode 100644 index 000000000..9f16de580 --- /dev/null +++ b/doc/source/projects.rst @@ -0,0 +1,77 @@ +Ray Projects (Experimental) +=========================== + +Ray projects make it easy to package a Ray application so it can be +rerun later in the same environment. They allow for the sharing and +reliable reuse of existing code. + +Quick start (CLI) +----------------- + +.. code-block:: bash + + # Creates a project in the current directory. It will create a + # project.yaml defining the code and environment and a cluster.yaml + # describing the cluster configuration. Both will be created in the + # .rayproject subdirectory of the current directory. + $ ray project create + + # Create a new session from the given project. + # Launch a cluster and run the appropriate command. + $ ray session start + + # Open a console for the given session. + $ ray session attach + + # Stop the given session and all of its worker nodes. The nodes/clusters + # are not actually terminated. + $ ray session stop + +Examples +-------- +- `Open Tacotron `__: + A TensorFlow implementation of Google's Tacotron speech synthesis with pre-trained model (unofficial) +- `PyTorch Transformers `__: + A library of state-of-the-art pretrained models for Natural Language Processing (NLP) + +Project file format (project.yaml) +---------------------------------- + +A project file contains everything required to run a project. +This includes a cluster configuration, the environment and dependencies +for the application, and the specific inputs used to run the project. + +Here is an example for a minimal project format: + +.. code-block:: yaml + + name: test-project + description: "This is a simple test project" + repo: https://github.com/ray-project/ray + + # Cluster to be instantiated by default when starting the project. + cluster: .rayproject/cluster.yaml + + # Commands/information to build the environment, once the cluster is + # instantiated. This can include the versions of python libraries etc. + # It can be specified as a Python requirements.txt, a conda environment, + # a Dockerfile, or a shell script to run to set up the libraries. + environment: + requirements: requirements.txt + + # List of commands that can be executed once the cluster is instantiated + # and the environment is set up. + # A command can also specify a cluster that overwrites the default cluster. + commands: + - name: test + command: python test.py + +Project files have to adhere to the following schema: + +.. jsonschema:: ../../python/ray/projects/schema.json + +Cluster file format (cluster.yaml) +---------------------------------- + +This is the same as for the autoscaler, see +`Cluster Launch page `_. diff --git a/python/ray/projects/examples/open-tacotron/.rayproject/cluster.yaml b/python/ray/projects/examples/open-tacotron/.rayproject/cluster.yaml new file mode 100644 index 000000000..2dbfb452a --- /dev/null +++ b/python/ray/projects/examples/open-tacotron/.rayproject/cluster.yaml @@ -0,0 +1,18 @@ +# This file is generated by `ray project create` + +# A unique identifier for the head node and workers of this cluster. +cluster_name: open-tacotron + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers defaults to 0. +max_workers: 1 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu diff --git a/python/ray/projects/examples/open-tacotron/.rayproject/project.yaml b/python/ray/projects/examples/open-tacotron/.rayproject/project.yaml new file mode 100644 index 000000000..6485de712 --- /dev/null +++ b/python/ray/projects/examples/open-tacotron/.rayproject/project.yaml @@ -0,0 +1,17 @@ +# This file is generated by `ray project create` + +name: open-tacotron +description: "A TensorFlow implementation of Google's Tacotron speech synthesis with pre-trained model (unofficial)" +repo: https://github.com/keithito/tacotron + +cluster: .rayproject/cluster.yaml + +environment: + requirements: requirements.txt + + shell: + - curl http://data.keithito.com/data/speech/tacotron-20180906.tar.gz | tar xzC /tmp + +commands: + - name: serve + command: python demo_server.py --checkpoint /tmp/tacotron-20180906/model.ckpt diff --git a/python/ray/projects/examples/open-tacotron/requirements.txt b/python/ray/projects/examples/open-tacotron/requirements.txt new file mode 100644 index 000000000..100979f08 --- /dev/null +++ b/python/ray/projects/examples/open-tacotron/requirements.txt @@ -0,0 +1,11 @@ +# Adapted from https://github.com/keithito/tacotron/blob/master/requirements.txt +# Note: this doesn't include tensorflow or tensorflow-gpu because the package you need to install +# depends on your platform. It is assumed you have already installed tensorflow. +falcon==1.2.0 +inflect==0.2.5 +librosa==0.5.1 +matplotlib==2.0.2 +numpy==1.14.3 +scipy==0.19.0 +tqdm==4.11.2 +Unidecode==0.4.20 diff --git a/python/ray/projects/examples/pytorch-transformers/.rayproject/cluster.yaml b/python/ray/projects/examples/pytorch-transformers/.rayproject/cluster.yaml new file mode 100644 index 000000000..b2143d8c4 --- /dev/null +++ b/python/ray/projects/examples/pytorch-transformers/.rayproject/cluster.yaml @@ -0,0 +1,18 @@ +# This file is generated by `ray project create` + +# An unique identifier for the head node and workers of this cluster. +cluster_name: pytorch-transformers + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers default to 0. +max_workers: 1 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu diff --git a/python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml b/python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml new file mode 100644 index 000000000..0d548819c --- /dev/null +++ b/python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml @@ -0,0 +1,30 @@ +# This file is generated by `ray project create` + +name: pytorch-transformers +description: "A library of state-of-the-art pretrained models for Natural Language Processing (NLP)" +repo: https://github.com/huggingface/pytorch-transformers + +cluster: .rayproject/cluster.yaml + +environment: + requirements: requirements.txt + +commands: + - name: train_sst_2 + command: | + wget https://raw.githubusercontent.com/nyu-mll/GLUE-baselines/master/download_glue_data.py && \ + python download_glue_data.py -d /tmp -t SST && \ + python ./examples/run_glue.py \ + --model_type bert \ + --model_name_or_path bert-base-uncased \ + --task_name SST-2 \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir /tmp/SST-2 \ + --max_seq_length 128 \ + --per_gpu_eval_batch_size=8 \ + --per_gpu_train_batch_size=8 \ + --learning_rate 2e-5 \ + --num_train_epochs 3.0 \ + --output_dir /tmp/output/ diff --git a/python/ray/projects/examples/pytorch-transformers/requirements.txt b/python/ray/projects/examples/pytorch-transformers/requirements.txt new file mode 100644 index 000000000..0ebd68f98 --- /dev/null +++ b/python/ray/projects/examples/pytorch-transformers/requirements.txt @@ -0,0 +1,17 @@ +# Adapted from https://github.com/huggingface/pytorch-transformers/blob/master/requirements.txt +# PyTorch +torch>=1.0.0 +# progress bars in model download and training scripts +tqdm +# Accessing files from S3 directly. +boto3 +# Used for downloading models over HTTP +requests +# For OpenAI GPT +regex +# For XLNet +sentencepiece +# TensorBoard visualization +tensorboardX +# Pytorch transformers +pytorch_transformers