diff --git a/python/ray/projects/projects.py b/python/ray/projects/projects.py index bf2a11582..1ff0cb9c3 100644 --- a/python/ray/projects/projects.py +++ b/python/ray/projects/projects.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import json import jsonschema import os @@ -110,7 +114,7 @@ def load_project(current_dir): raise ValueError("Project file {} not found".format(project_file)) with open(project_file) as f: - project_definition = yaml.load(f) + project_definition = yaml.safe_load(f) check_project_definition(project_root, project_definition) diff --git a/python/ray/projects/schema.json b/python/ray/projects/schema.json index fd21c8b1e..0f05179e6 100644 --- a/python/ray/projects/schema.json +++ b/python/ray/projects/schema.json @@ -60,5 +60,8 @@ } } }, - "required": ["name", "cluster"] + "required": [ + "name", + "cluster" + ] } diff --git a/python/ray/projects/scripts.py b/python/ray/projects/scripts.py new file mode 100644 index 000000000..ffc381bdc --- /dev/null +++ b/python/ray/projects/scripts.py @@ -0,0 +1,112 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import logging +import os +import sys +from shutil import copyfile + +import click +import jsonschema + +import ray + +logging.basicConfig(format=ray.ray_constants.LOGGER_FORMAT) +logger = logging.getLogger(__file__) + +# File layout for generated project files +# user-dir/ +# .rayproject/ +# project.yaml +# cluster.yaml +# requirements.txt +PROJECT_DIR = ".rayproject" +PROJECT_YAML = os.path.join(PROJECT_DIR, "project.yaml") +CLUSTER_YAML = os.path.join(PROJECT_DIR, "cluster.yaml") +REQUIREMENTS_TXT = os.path.join(PROJECT_DIR, "requirements.txt") + +# File layout for templates file +# RAY/.../projects/ +# templates/ +# cluster_template.yaml +# project_template.yaml +# requirements.txt +_THIS_FILE_DIR = os.path.split(os.path.abspath(__file__))[0] +_TEMPLATE_DIR = os.path.join(_THIS_FILE_DIR, "templates") +PROJECT_TEMPLATE = os.path.join(_TEMPLATE_DIR, "project_template.yaml") +CLUSTER_TEMPLATE = os.path.join(_TEMPLATE_DIR, "cluster_template.yaml") +REQUIREMENTS_TXT_TEMPLATE = os.path.join(_TEMPLATE_DIR, "requirements.txt") + + +@click.group( + "project", help="[Experimental] Commands working with ray project") +def project_cli(): + pass + + +@project_cli.command(help="Validate current project spec") +@click.option( + "--verbose", help="If set, print the validated file", is_flag=True) +def validate(verbose): + try: + project = ray.projects.load_project(os.getcwd()) + print("🍰 Project files validated!", file=sys.stderr) + if verbose: + print(project) + except (jsonschema.exceptions.ValidationError, ValueError) as e: + print("💔 Validation failed for the following reason", file=sys.stderr) + raise click.ClickException(e) + + +@project_cli.command(help="Create a new project within current directory") +@click.argument("project_name") +@click.option( + "--cluster-yaml", + help="Path to autoscaler yaml. Created by default", + default=None) +@click.option( + "--requirements", + help="Path to requirements.txt. Created by default", + default=None) +def create(project_name, cluster_yaml, requirements): + if os.path.exists(PROJECT_DIR): + raise click.ClickException( + "Project directory {} already exists.".format(PROJECT_DIR)) + os.makedirs(PROJECT_DIR) + + if cluster_yaml is None: + logger.warn("Using default autoscaler yaml") + + with open(CLUSTER_TEMPLATE) as f: + template = f.read().replace(r"{{name}}", project_name) + with open(CLUSTER_YAML, "w") as f: + f.write(template) + + cluster_yaml = CLUSTER_YAML + + if requirements is None: + logger.warn("Using default requirements.txt") + # no templating required, just copy the file + copyfile(REQUIREMENTS_TXT_TEMPLATE, REQUIREMENTS_TXT) + + requirements = REQUIREMENTS_TXT + + with open(PROJECT_TEMPLATE) as f: + project_template = f.read() + # NOTE(simon): + # We could use jinja2, which will make the templating part easier. + project_template = project_template.replace(r"{{name}}", project_name) + project_template = project_template.replace(r"{{cluster}}", + cluster_yaml) + project_template = project_template.replace(r"{{requirements}}", + requirements) + + with open(PROJECT_YAML, "w") as f: + f.write(project_template) + + +@click.group( + "session", help="[Experimental] Commands working with ray session") +def session_cli(): + pass diff --git a/python/ray/projects/templates/cluster_template.yaml b/python/ray/projects/templates/cluster_template.yaml new file mode 100644 index 000000000..962dd1768 --- /dev/null +++ b/python/ray/projects/templates/cluster_template.yaml @@ -0,0 +1,18 @@ +# This file is generated by `ray project create`. + +# A unique identifier for the head node and workers of this cluster. +cluster_name: {{name}} + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers defaults to 0. +max_workers: 1 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu diff --git a/python/ray/projects/templates/project_template.yaml b/python/ray/projects/templates/project_template.yaml new file mode 100644 index 000000000..9921a0734 --- /dev/null +++ b/python/ray/projects/templates/project_template.yaml @@ -0,0 +1,20 @@ +# This file is generated by `ray project create`. + +name: {{name}} + +# description: A short description of the project. +# repo: The URL of the repo this project is part of. + +cluster: {{cluster}} + +environment: + # dockerfile: The dockerfile to be built and ran the commands with. + # dockerimage: The docker image to be used to run the project in, e.g. ubuntu:18.04. + requirements: {{requirements}} + + shell: # Shell commands to be ran for environment setup. + - echo "Setting up the environment" + +commands: + - name: first-command + command: echo "Starting ray job" diff --git a/python/ray/projects/templates/requirements.txt b/python/ray/projects/templates/requirements.txt new file mode 100644 index 000000000..0f026d879 --- /dev/null +++ b/python/ray/projects/templates/requirements.txt @@ -0,0 +1 @@ +ray[debug] \ No newline at end of file diff --git a/python/ray/scripts/scripts.py b/python/ray/scripts/scripts.py index d99e0d326..4b31edf98 100644 --- a/python/ray/scripts/scripts.py +++ b/python/ray/scripts/scripts.py @@ -16,6 +16,7 @@ from ray.autoscaler.commands import ( rsync, teardown_cluster, get_head_node_ip, kill_node, get_worker_node_ips) import ray.ray_constants as ray_constants import ray.utils +from ray.projects.scripts import project_cli, session_cli logger = logging.getLogger(__name__) @@ -706,17 +707,6 @@ def get_worker_ips(cluster_config_file, cluster_name): click.echo("\n".join(worker_ips)) -@cli.command() -@click.argument("command", required=True, type=str) -@click.option( - "--dry", - is_flag=True, - default=False, - help="Print actions instead of running them.") -def session(command, dry): - ray.projects.load_project(os.getcwd()) - - @cli.command() def stack(): COMMAND = """ @@ -802,9 +792,10 @@ cli.add_command(teardown, name="down") cli.add_command(kill_random_node) cli.add_command(get_head_ip, name="get_head_ip") cli.add_command(get_worker_ips) -cli.add_command(session) cli.add_command(stack) cli.add_command(timeline) +cli.add_command(project_cli) +cli.add_command(session_cli) def main(): diff --git a/python/ray/tests/test_projects.py b/python/ray/tests/test_projects.py index dcee1400c..dba09baaf 100644 --- a/python/ray/tests/test_projects.py +++ b/python/ray/tests/test_projects.py @@ -16,7 +16,7 @@ TEST_DIR = os.path.dirname(os.path.abspath(__file__)) def load_project_description(project_file): path = os.path.join(TEST_DIR, "project_files", project_file) with open(path) as f: - return yaml.load(f) + return yaml.safe_load(f) def test_validation_success(): @@ -58,4 +58,10 @@ def test_project_root(): def test_project_validation(): path = os.path.join(TEST_DIR, "project_files", "project1") - subprocess.check_call(["ray", "session", "create", "--dry"], cwd=path) + subprocess.check_call(["ray", "project", "validate"], cwd=path) + + +def test_project_no_validation(): + path = os.path.join(TEST_DIR, "project_files") + with pytest.raises(subprocess.CalledProcessError): + subprocess.check_call(["ray", "project", "validate"], cwd=path) diff --git a/python/setup.py b/python/setup.py index a51575d89..abca6870f 100644 --- a/python/setup.py +++ b/python/setup.py @@ -22,11 +22,14 @@ import setuptools.command.build_ext as _build_ext ray_files = [ "ray/core/src/ray/thirdparty/redis/src/redis-server", "ray/core/src/ray/gcs/redis_module/libray_redis_module.so", - "ray/core/src/plasma/plasma_store_server", "ray/_raylet.so", - "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet", - "ray/dashboard/dashboard.py", "ray/dashboard/index.html", - "ray/dashboard/res/main.css", "ray/dashboard/res/main.js", - "ray/projects/schema.json" + "ray/core/src/plasma/plasma_store_server", + "ray/_raylet.so", + "ray/core/src/ray/raylet/raylet_monitor", + "ray/core/src/ray/raylet/raylet", + "ray/dashboard/dashboard.py", + "ray/dashboard/index.html", + "ray/dashboard/res/main.css", + "ray/dashboard/res/main.js", ] # These are the directories where automatically generated Python protobuf @@ -43,6 +46,12 @@ ray_autoscaler_files = [ "ray/autoscaler/local/example-full.yaml", ] +ray_project_files = [ + "ray/projects/schema.json", "ray/projects/template/cluster_template.yaml", + "ray/projects/template/project_template.yaml", + "ray/projects/template/requirements.txt" +] + if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on": ray_files += [ "ray/core/src/credis/build/src/libmember.so", @@ -51,6 +60,7 @@ if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on": ] optional_ray_files += ray_autoscaler_files +optional_ray_files += ray_project_files extras = { "rllib": [