diff --git a/doc/source/projects.rst b/doc/source/projects.rst index 1cd9a0435..8e13043fe 100644 --- a/doc/source/projects.rst +++ b/doc/source/projects.rst @@ -16,19 +16,23 @@ Quick start (CLI) # .rayproject subdirectory of the current directory. $ ray project create - # Create a new session from the given project. - # Launch a cluster and run the appropriate command. - $ ray session start [arguments] + # Create a new session from the given project. Launch a cluster and run + # the command, which must be specified in the project.yaml file. If no + # command is specified, the "default" command in .rayproject/project.yaml + # will be used. Alternatively, use --shell to run a raw shell command. + $ ray session start [arguments] [--shell] # Open a console for the given session. $ ray session attach - # Stop the given session and all of its worker nodes. The nodes/clusters - # are not actually terminated. + # Stop the given session and terminate all of its worker nodes. $ ray session stop Examples -------- +See `the readme `__ +for instructions on how to run these examples: + - `Open Tacotron `__: A TensorFlow implementation of Google's Tacotron speech synthesis with pre-trained model (unofficial) - `PyTorch Transformers `__: diff --git a/python/ray/projects/examples/README.md b/python/ray/projects/examples/README.md new file mode 100644 index 000000000..cee7ece16 --- /dev/null +++ b/python/ray/projects/examples/README.md @@ -0,0 +1,41 @@ +Ray Projects +============ + +To run these example projects, we first have to make sure the full +repository is checked out into the project directory. + +Open Tacotron +------------- + +```shell +cd open-tacotron +# Check out the original repository +git init +git remote add origin https://github.com/keithito/tacotron.git +git fetch +git checkout -t origin/master + +# Serve the model +ray session start serve + +# Terminate the session +ray session stop +``` + +PyTorch Transformers +-------------------- + +```shell +cd python-transformers +# Check out the original repository +git init +git remote add origin https://github.com/huggingface/pytorch-transformers.git +git fetch +git checkout -t origin/master + +# Now we can start the training +ray session start train --dataset SST-2 + +# Terminate the session +ray session stop +``` diff --git a/python/ray/projects/examples/open-tacotron/.rayproject/project.yaml b/python/ray/projects/examples/open-tacotron/.rayproject/project.yaml index 6485de712..4aa268197 100644 --- a/python/ray/projects/examples/open-tacotron/.rayproject/project.yaml +++ b/python/ray/projects/examples/open-tacotron/.rayproject/project.yaml @@ -7,7 +7,7 @@ repo: https://github.com/keithito/tacotron cluster: .rayproject/cluster.yaml environment: - requirements: requirements.txt + requirements: .rayproject/requirements.txt shell: - curl http://data.keithito.com/data/speech/tacotron-20180906.tar.gz | tar xzC /tmp diff --git a/python/ray/projects/examples/open-tacotron/requirements.txt b/python/ray/projects/examples/open-tacotron/.rayproject/requirements.txt similarity index 100% rename from python/ray/projects/examples/open-tacotron/requirements.txt rename to python/ray/projects/examples/open-tacotron/.rayproject/requirements.txt diff --git a/python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml b/python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml index e62b98157..bfac5525b 100644 --- a/python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml +++ b/python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml @@ -7,7 +7,7 @@ repo: https://github.com/huggingface/pytorch-transformers cluster: .rayproject/cluster.yaml environment: - requirements: requirements.txt + requirements: .rayproject/requirements.txt commands: - name: train diff --git a/python/ray/projects/examples/pytorch-transformers/requirements.txt b/python/ray/projects/examples/pytorch-transformers/.rayproject/requirements.txt similarity index 100% rename from python/ray/projects/examples/pytorch-transformers/requirements.txt rename to python/ray/projects/examples/pytorch-transformers/.rayproject/requirements.txt diff --git a/python/ray/projects/scripts.py b/python/ray/projects/scripts.py index 72a4572dd..efd4bfed4 100644 --- a/python/ray/projects/scripts.py +++ b/python/ray/projects/scripts.py @@ -3,13 +3,14 @@ from __future__ import division from __future__ import print_function import argparse -import logging -import os -import sys -from shutil import copyfile -import time import click import jsonschema +import logging +import os +from shutil import copyfile +import subprocess +import sys +import time import ray from ray.autoscaler.commands import ( @@ -20,7 +21,7 @@ from ray.autoscaler.commands import ( teardown_cluster, ) -logging.basicConfig(format=ray.ray_constants.LOGGER_FORMAT) +logging.basicConfig(format=ray.ray_constants.LOGGER_FORMAT, level=logging.INFO) logger = logging.getLogger(__file__) # File layout for generated project files @@ -100,6 +101,14 @@ def create(project_name, cluster_yaml, requirements): requirements = REQUIREMENTS_TXT + repo = None + try: + repo = subprocess.check_output( + "git remote get-url origin".split(" ")).strip() + logger.info("Setting repo URL to %s", repo) + except subprocess.CalledProcessError: + pass + with open(PROJECT_TEMPLATE) as f: project_template = f.read() # NOTE(simon): @@ -109,7 +118,12 @@ def create(project_name, cluster_yaml, requirements): cluster_yaml) project_template = project_template.replace(r"{{requirements}}", requirements) - + if repo is None: + project_template = project_template.replace( + r"{{repo_string}}", "# repo: {}".format("...")) + else: + project_template = project_template.replace( + r"{{repo_string}}", "repo: {}".format(repo)) with open(PROJECT_YAML, "w") as f: f.write(project_template) @@ -159,10 +173,18 @@ def stop(): help="Start a session based on current project config") @click.argument("command", required=False) @click.argument("args", nargs=-1, type=click.UNPROCESSED) -def start(command, args): +@click.option( + "--shell", + help=( + "If set, run the command as a raw shell command instead of looking up " + "the command in the project config"), + is_flag=True) +def start(command, args, shell): project_definition = load_project_or_throw() - if command: + if shell: + command_to_run = command + elif command: command_to_run = _get_command_to_run(command, project_definition, args) else: command_to_run = _get_command_to_run("default", project_definition, @@ -192,22 +214,19 @@ def start(command, args): override_cluster_name=None, ) - logger.info("[2/4] Syncing the repo") - if "repo" in project_definition: - # HACK: Skip git clone if exists so the this command can be idempotent - # More advanced repo update behavior can be found at - # https://github.com/jupyterhub/nbgitpuller/blob/master/nbgitpuller/pull.py - session_exec_cluster( - cluster_yaml, - "git clone {repo} {directory} || true".format( - repo=project_definition["repo"], - directory=project_definition["name"]), - ) - else: - session_exec_cluster( - cluster_yaml, - "mkdir {directory} || true".format( - directory=project_definition["name"])) + logger.info("[2/4] Syncing the project") + project_root = ray.projects.find_root(os.getcwd()) + # This is so that rsync syncs directly to the target directory, instead of + # nesting inside the target directory. + if not project_root.endswith("/"): + project_root += "/" + rsync( + cluster_yaml, + source=project_root, + target="~/{}/".format(working_directory), + override_cluster_name=None, + down=False, + ) logger.info("[3/4] Setting up environment") _setup_environment( diff --git a/python/ray/projects/templates/project_template.yaml b/python/ray/projects/templates/project_template.yaml index 9921a0734..087061d48 100644 --- a/python/ray/projects/templates/project_template.yaml +++ b/python/ray/projects/templates/project_template.yaml @@ -3,7 +3,8 @@ name: {{name}} # description: A short description of the project. -# repo: The URL of the repo this project is part of. +# The URL of the repo this project is part of. +{{repo_string}} cluster: {{cluster}} @@ -16,5 +17,5 @@ environment: - echo "Setting up the environment" commands: - - name: first-command + - name: default command: echo "Starting ray job" diff --git a/python/ray/tests/test_projects.py b/python/ray/tests/test_projects.py index cd285b0f0..bce416c7b 100644 --- a/python/ray/tests/test_projects.py +++ b/python/ray/tests/test_projects.py @@ -118,7 +118,9 @@ def test_session_start_default_project(): # Part 2/3: Rsync Calls rsync_call = mock_calls["rsync"] - assert rsync_call.call_count == 1 + # 1 for rsyncing the project directory, 1 for rsyncing the + # requirements.txt. + assert rsync_call.call_count == 2 _, kwargs = rsync_call.call_args assert kwargs["source"] == loaded_project["environment"]["requirements"] @@ -141,16 +143,6 @@ def test_session_start_default_project(): cmd for cmd in commands_executed if "pip install -r" not in cmd ] - # if we don't have a repo, we will be creating a directory - if "repo" not in loaded_project: - mkdir_command = "mkdir {project_name}".format( - project_name=loaded_project["name"]) - assert any(mkdir_command in cmd for cmd in commands_executed) - # pop the `pip install` off commands executed - commands_executed = [ - cmd for cmd in commands_executed if mkdir_command not in cmd - ] - assert expected_commands == commands_executed @@ -163,25 +155,6 @@ def test_session_start_docker_fail(): "not implemented") in result.output -def test_session_git_repo_cloned(): - result, mock_calls, test_dir = run_test_project( - "session-tests/git-repo-pass", start, []) - - loaded_project = ray.projects.load_project(test_dir) - assert result.exit_code == 0 - - exec_cluster_call = mock_calls["exec_cluster"] - commands_executed = [] - for _, kwargs in exec_cluster_call.call_args_list: - command_executed = kwargs["cmd"] - # Filter out the cd call that was appended to each command - cd_project_dir_call = "cd {}; ".format(loaded_project["name"]) - command_executed = command_executed.replace(cd_project_dir_call, "") - commands_executed.append(command_executed) - - assert any("git clone" in cmd for cmd in commands_executed) - - def test_session_invalid_config_errored(): result, _, _ = run_test_project("session-tests/invalid-config-fail", start, [])