[Project] Implementing Project CLI (#5397)

python/ray/projects/projects.py

@@ -1,3 +1,7 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
 import json
 import jsonschema
 import os
@@ -110,7 +114,7 @@ def load_project(current_dir):
         raise ValueError("Project file {} not found".format(project_file))
 
     with open(project_file) as f:
-        project_definition = yaml.load(f)
+        project_definition = yaml.safe_load(f)
 
     check_project_definition(project_root, project_definition)
 

python/ray/projects/schema.json

@@ -60,5 +60,8 @@
       }
     }
   },
-  "required": ["name", "cluster"]
+  "required": [
+    "name",
+    "cluster"
+  ]
 }

python/ray/projects/scripts.py

@@ -0,0 +1,112 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import logging
+import os
+import sys
+from shutil import copyfile
+
+import click
+import jsonschema
+
+import ray
+
+logging.basicConfig(format=ray.ray_constants.LOGGER_FORMAT)
+logger = logging.getLogger(__file__)
+
+# File layout for generated project files
+# user-dir/
+#   .rayproject/
+#     project.yaml
+#     cluster.yaml
+#     requirements.txt
+PROJECT_DIR = ".rayproject"
+PROJECT_YAML = os.path.join(PROJECT_DIR, "project.yaml")
+CLUSTER_YAML = os.path.join(PROJECT_DIR, "cluster.yaml")
+REQUIREMENTS_TXT = os.path.join(PROJECT_DIR, "requirements.txt")
+
+# File layout for templates file
+# RAY/.../projects/
+#   templates/
+#     cluster_template.yaml
+#     project_template.yaml
+#     requirements.txt
+_THIS_FILE_DIR = os.path.split(os.path.abspath(__file__))[0]
+_TEMPLATE_DIR = os.path.join(_THIS_FILE_DIR, "templates")
+PROJECT_TEMPLATE = os.path.join(_TEMPLATE_DIR, "project_template.yaml")
+CLUSTER_TEMPLATE = os.path.join(_TEMPLATE_DIR, "cluster_template.yaml")
+REQUIREMENTS_TXT_TEMPLATE = os.path.join(_TEMPLATE_DIR, "requirements.txt")
+
+
+@click.group(
+    "project", help="[Experimental] Commands working with ray project")
+def project_cli():
+    pass
+
+
+@project_cli.command(help="Validate current project spec")
+@click.option(
+    "--verbose", help="If set, print the validated file", is_flag=True)
+def validate(verbose):
+    try:
+        project = ray.projects.load_project(os.getcwd())
+        print("🍰 Project files validated!", file=sys.stderr)
+        if verbose:
+            print(project)
+    except (jsonschema.exceptions.ValidationError, ValueError) as e:
+        print("💔 Validation failed for the following reason", file=sys.stderr)
+        raise click.ClickException(e)
+
+
+@project_cli.command(help="Create a new project within current directory")
+@click.argument("project_name")
+@click.option(
+    "--cluster-yaml",
+    help="Path to autoscaler yaml. Created by default",
+    default=None)
+@click.option(
+    "--requirements",
+    help="Path to requirements.txt. Created by default",
+    default=None)
+def create(project_name, cluster_yaml, requirements):
+    if os.path.exists(PROJECT_DIR):
+        raise click.ClickException(
+            "Project directory {} already exists.".format(PROJECT_DIR))
+    os.makedirs(PROJECT_DIR)
+
+    if cluster_yaml is None:
+        logger.warn("Using default autoscaler yaml")
+
+        with open(CLUSTER_TEMPLATE) as f:
+            template = f.read().replace(r"{{name}}", project_name)
+        with open(CLUSTER_YAML, "w") as f:
+            f.write(template)
+
+        cluster_yaml = CLUSTER_YAML
+
+    if requirements is None:
+        logger.warn("Using default requirements.txt")
+        # no templating required, just copy the file
+        copyfile(REQUIREMENTS_TXT_TEMPLATE, REQUIREMENTS_TXT)
+
+        requirements = REQUIREMENTS_TXT
+
+    with open(PROJECT_TEMPLATE) as f:
+        project_template = f.read()
+        # NOTE(simon):
+        # We could use jinja2, which will make the templating part easier.
+        project_template = project_template.replace(r"{{name}}", project_name)
+        project_template = project_template.replace(r"{{cluster}}",
+                                                    cluster_yaml)
+        project_template = project_template.replace(r"{{requirements}}",
+                                                    requirements)
+
+    with open(PROJECT_YAML, "w") as f:
+        f.write(project_template)
+
+
+@click.group(
+    "session", help="[Experimental] Commands working with ray session")
+def session_cli():
+    pass

python/ray/projects/templates/cluster_template.yaml

@@ -0,0 +1,18 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: {{name}}
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 1
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu

python/ray/projects/templates/project_template.yaml

@@ -0,0 +1,20 @@
+# This file is generated by `ray project create`.
+
+name: {{name}}
+
+# description: A short description of the project.
+# repo: The URL of the repo this project is part of.
+
+cluster: {{cluster}}
+
+environment:
+  # dockerfile: The dockerfile to be built and ran the commands with.
+  # dockerimage: The docker image to be used to run the project in, e.g. ubuntu:18.04.
+  requirements: {{requirements}}
+
+  shell: # Shell commands to be ran for environment setup.
+    - echo "Setting up the environment"
+
+commands:
+  - name: first-command
+    command: echo "Starting ray job"

python/ray/projects/templates/requirements.txt

@@ -0,0 +1 @@
+ray[debug]

python/ray/scripts/scripts.py

@@ -16,6 +16,7 @@ from ray.autoscaler.commands import (
     rsync, teardown_cluster, get_head_node_ip, kill_node, get_worker_node_ips)
 import ray.ray_constants as ray_constants
 import ray.utils
+from ray.projects.scripts import project_cli, session_cli
 
 logger = logging.getLogger(__name__)
 
@@ -706,17 +707,6 @@ def get_worker_ips(cluster_config_file, cluster_name):
     click.echo("\n".join(worker_ips))
 
 
-@cli.command()
-@click.argument("command", required=True, type=str)
-@click.option(
-    "--dry",
-    is_flag=True,
-    default=False,
-    help="Print actions instead of running them.")
-def session(command, dry):
-    ray.projects.load_project(os.getcwd())
-
-
 @cli.command()
 def stack():
     COMMAND = """
@@ -802,9 +792,10 @@ cli.add_command(teardown, name="down")
 cli.add_command(kill_random_node)
 cli.add_command(get_head_ip, name="get_head_ip")
 cli.add_command(get_worker_ips)
-cli.add_command(session)
 cli.add_command(stack)
 cli.add_command(timeline)
+cli.add_command(project_cli)
+cli.add_command(session_cli)
 
 
 def main():

python/ray/tests/test_projects.py

@@ -16,7 +16,7 @@ TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 def load_project_description(project_file):
     path = os.path.join(TEST_DIR, "project_files", project_file)
     with open(path) as f:
-        return yaml.load(f)
+        return yaml.safe_load(f)
 
 
 def test_validation_success():
@@ -58,4 +58,10 @@ def test_project_root():
 
 def test_project_validation():
     path = os.path.join(TEST_DIR, "project_files", "project1")
-    subprocess.check_call(["ray", "session", "create", "--dry"], cwd=path)
+    subprocess.check_call(["ray", "project", "validate"], cwd=path)
+
+
+def test_project_no_validation():
+    path = os.path.join(TEST_DIR, "project_files")
+    with pytest.raises(subprocess.CalledProcessError):
+        subprocess.check_call(["ray", "project", "validate"], cwd=path)

python/setup.py

@@ -22,11 +22,14 @@ import setuptools.command.build_ext as _build_ext
 ray_files = [
     "ray/core/src/ray/thirdparty/redis/src/redis-server",
     "ray/core/src/ray/gcs/redis_module/libray_redis_module.so",
-    "ray/core/src/plasma/plasma_store_server", "ray/_raylet.so",
-    "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet",
-    "ray/dashboard/dashboard.py", "ray/dashboard/index.html",
-    "ray/dashboard/res/main.css", "ray/dashboard/res/main.js",
-    "ray/projects/schema.json"
+    "ray/core/src/plasma/plasma_store_server",
+    "ray/_raylet.so",
+    "ray/core/src/ray/raylet/raylet_monitor",
+    "ray/core/src/ray/raylet/raylet",
+    "ray/dashboard/dashboard.py",
+    "ray/dashboard/index.html",
+    "ray/dashboard/res/main.css",
+    "ray/dashboard/res/main.js",
 ]
 
 # These are the directories where automatically generated Python protobuf
@@ -43,6 +46,12 @@ ray_autoscaler_files = [
     "ray/autoscaler/local/example-full.yaml",
 ]
 
+ray_project_files = [
+    "ray/projects/schema.json", "ray/projects/template/cluster_template.yaml",
+    "ray/projects/template/project_template.yaml",
+    "ray/projects/template/requirements.txt"
+]
+
 if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on":
     ray_files += [
         "ray/core/src/credis/build/src/libmember.so",
@@ -51,6 +60,7 @@ if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on":
     ]
 
 optional_ray_files += ray_autoscaler_files
+optional_ray_files += ray_project_files
 
 extras = {
     "rllib": [