[Projects] Add small tutorial for projects (#6641)

doc/source/projects.rst

@@ -38,6 +38,102 @@ for instructions on how to run these examples:
 - `PyTorch Transformers <https://github.com/ray-project/ray/blob/master/python/ray/projects/examples/pytorch-transformers/ray-project/project.yaml>`__:
   A library of state-of-the-art pretrained models for Natural Language Processing (NLP)
 
+Tutorial
+--------
+
+We will walk through how to use projects by executing the `streaming MapReduce example <auto_examples/plot_streaming.html>`_.
+Commands always apply to the project in the current directory.
+Let us switch into the project directory with
+
+.. code-block:: bash
+
+    cd ray/doc/examples/streaming
+
+
+A session represents a running instance of a project. Let's start one with
+
+.. code-block:: bash
+
+    ray session start
+
+
+The ``ray session start`` command
+will bring up a new cluster and initialize the environment of the cluster
+according to the `environment` section of the `project.yaml`, installing all
+dependencies of the project.
+
+Now we can execute a command in the session. To see a list of all available
+commands of the project, run
+
+.. code-block:: bash
+
+    ray session commands
+
+
+which produces the following output:
+
+.. code-block::
+
+  Active project: ray-example-streaming
+
+  Command "run":
+    usage: run [--num-mappers NUM_MAPPERS] [--num-reducers NUM_REDUCERS]
+
+    Start the streaming example.
+
+    optional arguments:
+      --num-mappers NUM_MAPPERS
+                          Number of mapper actors used
+      --num-reducers NUM_REDUCERS
+                          Number of reducer actors used
+
+
+As you see, in this project there is only a single ``run`` command which has arguments
+``--num-mappers`` and ``--num-reducers``. We can execute the streaming
+wordcount with the default parameters by running
+
+.. code-block:: bash
+
+    ray session execute run
+
+
+You can interrupt the command with ``<Control>-c`` and attach to the running session by executing
+
+.. code-block:: bash
+
+    ray session attach --tmux
+
+
+Inside the session you can for example edit the streaming applications with
+
+.. code-block:: bash
+
+    cd ray-example-streaming
+    emacs streaming.py
+
+
+Try for example to add the following lines after the ``for count in counts:`` loop:
+
+.. code-block:: python
+
+    if "million" in wordcounts:
+      print("Found the word!")
+
+
+and re-run the application from outside the session with
+
+.. code-block:: bash
+
+    ray session execute run
+
+
+The session can be terminated from outside the session with
+
+.. code-block:: bash
+
+    ray session stop
+
+
 Project file format (project.yaml)
 ----------------------------------
 

python/ray/projects/scripts.py

@@ -1,3 +1,4 @@
+import argparse
 import click
 import copy
 import jsonschema
@@ -386,6 +387,31 @@ def session_start(command, args, shell, name):
             runner.execute_command(run["command"], config)
 
 
+@session_cli.command(
+    name="commands",
+    help="Print available commands for sessions of this project.")
+def session_commands():
+    project_definition = load_project_or_throw()
+    print("Active project: " + project_definition.config["name"])
+    print()
+
+    commands = project_definition.config["commands"]
+
+    for command in commands:
+        print("Command \"{}\":".format(command["name"]))
+        parser = argparse.ArgumentParser(
+            command["name"], description=command.get("help"), add_help=False)
+        params = command.get("params", [])
+        for param in params:
+            name = param.pop("name")
+            if "type" in param:
+                param.pop("type")
+            parser.add_argument("--" + name, **param)
+        help_string = parser.format_help()
+        # Indent the help message by two spaces and print it.
+        print("\n".join(["  " + line for line in help_string.split("\n")]))
+
+
 @session_cli.command(
     name="execute",
     context_settings=dict(ignore_unknown_options=True, ),

python/ray/tests/test_projects.py

@@ -9,7 +9,8 @@ from unittest.mock import patch, DEFAULT
 
 from contextlib import contextmanager
 
-from ray.projects.scripts import session_start, session_execute
+from ray.projects.scripts import (session_start, session_commands,
+                                  session_execute)
 import ray
 
 TEST_DIR = os.path.join(
@@ -231,6 +232,17 @@ def test_session_create_multiple():
     assert result.exit_code == 1
 
 
+def test_session_commands():
+    result, mock_calls, test_dir = run_test_project(
+        "session-tests/commands-test", session_commands, [])
+
+    assert "This is the first parameter" in result.output
+    assert "This is the second parameter" in result.output
+
+    assert 'Command "first"' in result.output
+    assert 'Command "second"' in result.output
+
+
 if __name__ == "__main__":
     # Make subprocess happy in bazel.
     os.environ["LC_ALL"] = "en_US.UTF-8"