diff --git a/doc/examples/cython/.rayproject/cluster.yaml b/doc/examples/cython/.rayproject/cluster.yaml new file mode 100644 index 000000000..5f08c13b0 --- /dev/null +++ b/doc/examples/cython/.rayproject/cluster.yaml @@ -0,0 +1,18 @@ +# This file is generated by `ray project create`. + +# A unique identifier for the head node and workers of this cluster. +cluster_name: ray-example-cython + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers defaults to 0. +max_workers: 1 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu diff --git a/doc/examples/cython/.rayproject/project.yaml b/doc/examples/cython/.rayproject/project.yaml new file mode 100644 index 000000000..70c3abf26 --- /dev/null +++ b/doc/examples/cython/.rayproject/project.yaml @@ -0,0 +1,46 @@ +# This file is generated by `ray project create`. + +name: ray-example-cython + +description: "Example of how to use Cython with ray" +tags: ["ray-example", "cython"] +documentation: https://ray.readthedocs.io/en/latest/advanced.html#cython-code-in-ray + +cluster: .rayproject/cluster.yaml + +environment: + requirements: .rayproject/requirements.txt + + shell: # Shell commands to be ran for environment setup. + - pip install -e . + +commands: + - name: example1 + command: python cython_main.py example1 + help: "Run a simple Cython function" + - name: example2 + command: python cython_main.py example2 + help: "Run a simple recursive Cython function" + - name: example3 + command: python cython_main.py example3 + help: "Run a simple typed Cython function" + - name: example4 + command: python cython_main.py example4 + help: "Run a Cython defined with cpdef" + - name: example5 + command: python cython_main.py example5 + help: "Run a Cython defined with cdef" + - name: example6 + command: python cython_main.py example6 + help: "Run a simple Cython class" + - name: example7 + command: python cython_main.py example7 + help: "Run Cython with function from BrainIAK" + - name: example8 + command: python cython_main.py example8 + help: "Run Cython with BLAS" + +output_files: [ + # Save the logs from the latest run in snapshots. + "/tmp/ray/session_latest/logs" +] diff --git a/doc/examples/cython/.rayproject/requirements.txt b/doc/examples/cython/.rayproject/requirements.txt new file mode 100644 index 000000000..43eabb232 --- /dev/null +++ b/doc/examples/cython/.rayproject/requirements.txt @@ -0,0 +1,2 @@ +ray[debug] +scipy diff --git a/doc/examples/lbfgs/.rayproject/cluster.yaml b/doc/examples/lbfgs/.rayproject/cluster.yaml new file mode 100644 index 000000000..dbf739d4c --- /dev/null +++ b/doc/examples/lbfgs/.rayproject/cluster.yaml @@ -0,0 +1,18 @@ +# This file is generated by `ray project create`. + +# A unique identifier for the head node and workers of this cluster. +cluster_name: ray-example-lbfgs + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers defaults to 0. +max_workers: 1 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu diff --git a/doc/examples/lbfgs/.rayproject/project.yaml b/doc/examples/lbfgs/.rayproject/project.yaml new file mode 100644 index 000000000..3fa7e0117 --- /dev/null +++ b/doc/examples/lbfgs/.rayproject/project.yaml @@ -0,0 +1,22 @@ +# This file is generated by `ray project create`. + +name: ray-example-lbfgs + +description: "Parallelizing the L-BFGS algorithm in ray" +tags: ["ray-example", "optimization", "lbfgs"] +documentation: https://ray.readthedocs.io/en/latest/auto_examples/plot_lbfgs.html + +cluster: .rayproject/cluster.yaml + +environment: + requirements: .rayproject/requirements.txt + +commands: + - name: run + command: python driver.py + help: "Run the L-BFGS example" + +output_files: [ + # Save the logs from the latest run in snapshots. + "/tmp/ray/session_latest/logs" +] diff --git a/doc/examples/lbfgs/.rayproject/requirements.txt b/doc/examples/lbfgs/.rayproject/requirements.txt new file mode 100644 index 000000000..dc3a91b64 --- /dev/null +++ b/doc/examples/lbfgs/.rayproject/requirements.txt @@ -0,0 +1 @@ +ray[debug,rllib] diff --git a/doc/examples/newsreader/.rayproject/cluster.yaml b/doc/examples/newsreader/.rayproject/cluster.yaml new file mode 100644 index 000000000..1ab3bf8ad --- /dev/null +++ b/doc/examples/newsreader/.rayproject/cluster.yaml @@ -0,0 +1,31 @@ +# This file is generated by `ray project create`. + +# A unique identifier for the head node and workers of this cluster. +cluster_name: ray-example-newsreader + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers defaults to 0. +max_workers: 1 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +head_node: + InstanceType: m5.large + ImageId: ami-06f2f779464715dc5 + +setup_commands: + # Install Anaconda. + - wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true + - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true + - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc + - sudo apt -y update + - sudo apt -y install npm + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.8.0.dev5-cp36-cp36m-manylinux1_x86_64.whl + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu diff --git a/doc/examples/newsreader/.rayproject/project.yaml b/doc/examples/newsreader/.rayproject/project.yaml new file mode 100644 index 000000000..55127ed17 --- /dev/null +++ b/doc/examples/newsreader/.rayproject/project.yaml @@ -0,0 +1,31 @@ +# This file is generated by `ray project create`. + +name: ray-example-newsreader + +description: "A simple news reader example that uses ray actors to serve requests" +tags: ["ray-example", "flask", "rss", "newsreader"] +documentation: https://ray.readthedocs.io/en/latest/auto_examples/plot_newsreader.html + +cluster: .rayproject/cluster.yaml + +environment: + requirements: .rayproject/requirements.txt + +commands: + - name: run-backend + command: python server.py + config: + port_forward: 5000 + - name: run-frontend + command: | + git clone https://github.com/ray-project/qreader + cd qreader + npm install + npm run dev + config: + port_forward: 8080 + +output_files: [ + # Save the logs from the latest run in snapshots. + "/tmp/ray/session_latest/logs" +] diff --git a/doc/examples/newsreader/.rayproject/requirements.txt b/doc/examples/newsreader/.rayproject/requirements.txt new file mode 100644 index 000000000..9040cd6cc --- /dev/null +++ b/doc/examples/newsreader/.rayproject/requirements.txt @@ -0,0 +1,3 @@ +ray[debug] +atoma +flask diff --git a/doc/examples/parameter_server/.rayproject/cluster.yaml b/doc/examples/parameter_server/.rayproject/cluster.yaml new file mode 100644 index 000000000..4a6c9f418 --- /dev/null +++ b/doc/examples/parameter_server/.rayproject/cluster.yaml @@ -0,0 +1,18 @@ +# This file is generated by `ray project create`. + +# A unique identifier for the head node and workers of this cluster. +cluster_name: ray-example-parameter-server + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers defaults to 0. +max_workers: 1 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu diff --git a/doc/examples/parameter_server/.rayproject/project.yaml b/doc/examples/parameter_server/.rayproject/project.yaml new file mode 100644 index 000000000..79fecabbc --- /dev/null +++ b/doc/examples/parameter_server/.rayproject/project.yaml @@ -0,0 +1,40 @@ +# This file is generated by `ray project create`. + +name: ray-example-parameter-server + +description: "A simple parameter server example implemented with ray actors" +tags: ["ray-example", "parameter-server", "machine-learning"] +documentation: https://ray.readthedocs.io/en/latest/auto_examples/plot_parameter_server.html + +cluster: .rayproject/cluster.yaml + +environment: + requirements: .rayproject/requirements.txt + +commands: + - name: run-sync + command: python sync_parameter_server.py --num-workers {{num_workers}} + help: "Start the synchronous parameter server." + params: + - name: num-workers + help: "Number of workers" + default: 4 + type: int + config: + tmux: true + + - name: run-async + command: python async_parameter_server.py --num-workers {{num_workers}} + help: "Start the asynchronous parameter server." + params: + - name: num-workers + help: "Number of workers" + default: 4 + type: int + config: + tmux: true + +output_files: [ + # Save the logs from the latest run in snapshots. + "/tmp/ray/session_latest/logs" +] diff --git a/doc/examples/parameter_server/.rayproject/requirements.txt b/doc/examples/parameter_server/.rayproject/requirements.txt new file mode 100644 index 000000000..72d631c31 --- /dev/null +++ b/doc/examples/parameter_server/.rayproject/requirements.txt @@ -0,0 +1,4 @@ +ray[debug,rllib] +torch +torchvision +filelock diff --git a/doc/examples/resnet/.rayproject/cluster.yaml b/doc/examples/resnet/.rayproject/cluster.yaml new file mode 100644 index 000000000..6c19b691f --- /dev/null +++ b/doc/examples/resnet/.rayproject/cluster.yaml @@ -0,0 +1,22 @@ +# This file is generated by `ray project create`. + +# A unique identifier for the head node and workers of this cluster. +cluster_name: ray-example-resnet + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers defaults to 0. +max_workers: 1 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +head_node: + InstanceType: m5.2xlarge + ImageId: ami-0b294f219d14e6a82 # Deep Learning AMI (Ubuntu) Version 21.0 + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu diff --git a/doc/examples/resnet/.rayproject/project.yaml b/doc/examples/resnet/.rayproject/project.yaml new file mode 100644 index 000000000..3f6dedb59 --- /dev/null +++ b/doc/examples/resnet/.rayproject/project.yaml @@ -0,0 +1,58 @@ +# This file is generated by `ray project create`. + +name: ray-example-resnet + +description: "Using ray to train resnet on multiple gpus" +tags: ["ray-example", "machine-learning", "tensorflow", "resnet"] +documentation: https://ray.readthedocs.io/en/latest/auto_examples/plot_resnet.html + +cluster: .rayproject/cluster.yaml + +environment: + requirements: .rayproject/requirements.txt + +commands: + - name: train + command: | + if [ "{{dataset}}" == "cifar10" ]; then + # Get the CIFAR-10 dataset. + curl -o cifar-10-binary.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz; + tar -xvf cifar-10-binary.tar.gz; + else + # Get the CIFAR-100 dataset. + curl -o cifar-100-binary.tar.gz https://www.cs.toronto.edu/~kriz/cifar-100-binary.tar.gz; + tar -xvf cifar-100-binary.tar.gz; + fi + python resnet_main.py --dataset {{dataset}} --train_data_path {{train_data_path}} --eval_data_path {{eval_data_path}} --eval_dir {{eval_data_path}} --eval_batch_count {{eval_batch_count}} --num_gpus {{num_gpus}} + params: + - name: dataset + help: "The dataset to train on." + default: "cifar10" + choices: ["cifar10", "cifar100"] + - name: train-data-path + help: "Data path for the training data." + default: "'cifar-10-batches-bin/data_batch*'" + type: str + - name: eval-data-path + help: "Data path for the testing data." + default: "cifar-10-batches-bin/test_batch.bin" + type: str + - name: eval-dir + help: "Data path for the tensorboard logs." + default: "/tmp/resnet-model/eval" + type: str + - name: eval-batch-count + help: "Number of batches to evaluate over." + default: 50 + type: int + - name: num-gpus + help: "Number of GPUs to use for training." + default: 0 + type: int + config: + tmux: true + +output_files: [ + # Save the logs from the latest run in snapshots. + "/tmp/ray/session_latest/logs" +] diff --git a/doc/examples/resnet/.rayproject/requirements.txt b/doc/examples/resnet/.rayproject/requirements.txt new file mode 100644 index 000000000..96b2688ae --- /dev/null +++ b/doc/examples/resnet/.rayproject/requirements.txt @@ -0,0 +1 @@ +ray[rllib,debug] diff --git a/doc/examples/resnet/cifar_input.py b/doc/examples/resnet/cifar_input.py index 283a66a27..d52a53115 100644 --- a/doc/examples/resnet/cifar_input.py +++ b/doc/examples/resnet/cifar_input.py @@ -47,7 +47,7 @@ def build_data(data_path, size, dataset): # Read examples from files in the filename queue. data_files = tf.gfile.Glob(data_path) - data = tf.contrib.data.FixedLengthRecordDataset( + data = tf.data.FixedLengthRecordDataset( data_files, record_bytes=record_bytes) data = data.map(load_transform) data = data.batch(size) @@ -76,7 +76,7 @@ def build_input(data, batch_size, dataset, train): num_classes = 10 if dataset == "cifar10" else 100 images, labels = data num_samples = images.shape[0] - images.shape[0] % batch_size - dataset = tf.contrib.data.Dataset.from_tensor_slices( + dataset = tf.data.Dataset.from_tensor_slices( (images[:num_samples], labels[:num_samples])) def map_train(image, label): diff --git a/doc/examples/streaming/.rayproject/cluster.yaml b/doc/examples/streaming/.rayproject/cluster.yaml new file mode 100644 index 000000000..62cc45e23 --- /dev/null +++ b/doc/examples/streaming/.rayproject/cluster.yaml @@ -0,0 +1,18 @@ +# This file is generated by `ray project create`. + +# A unique identifier for the head node and workers of this cluster. +cluster_name: ray-example-streaming + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers defaults to 0. +max_workers: 1 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu diff --git a/doc/examples/streaming/.rayproject/project.yaml b/doc/examples/streaming/.rayproject/project.yaml new file mode 100644 index 000000000..8bd2099f2 --- /dev/null +++ b/doc/examples/streaming/.rayproject/project.yaml @@ -0,0 +1,32 @@ +# This file is generated by `ray project create`. + +name: ray-example-streaming + +description: "A simple ray example for a streaming wordcount" +tags: ["ray-example", "streaming", "wordcount", "data-processing"] + +cluster: .rayproject/cluster.yaml + +environment: + requirements: .rayproject/requirements.txt + +commands: + - name: run + command: python streaming.py --num-mappers {{num_mappers}} --num-reducers {{num_reducers}} + help: "Start the streaming example." + params: + - name: num-mappers + help: "Number of mapper actors used" + default: 3 + type: int + - name: num-reducers + help: "Number of reducer actors used" + default: 4 + type: int + config: + tmux: true + +output_files: [ + # Save the logs from the latest run in snapshots. + "/tmp/ray/session_latest/logs" +] diff --git a/doc/examples/streaming/.rayproject/requirements.txt b/doc/examples/streaming/.rayproject/requirements.txt new file mode 100644 index 000000000..99bbe32ce --- /dev/null +++ b/doc/examples/streaming/.rayproject/requirements.txt @@ -0,0 +1,2 @@ +ray[debug] +wikipedia diff --git a/doc/examples/streaming/streaming.py b/doc/examples/streaming/streaming.py index b4153af3d..e4434bbe1 100644 --- a/doc/examples/streaming/streaming.py +++ b/doc/examples/streaming/streaming.py @@ -8,9 +8,9 @@ import wikipedia parser = argparse.ArgumentParser() parser.add_argument("--num-mappers", - help="number of mapper actors used", default=3) + help="number of mapper actors used", default=3, type=int) parser.add_argument("--num-reducers", - help="number of reducer actors used", default=4) + help="number of reducer actors used", default=4, type=int) @ray.remote diff --git a/python/ray/projects/projects.py b/python/ray/projects/projects.py index 674c219c4..2443e2808 100644 --- a/python/ray/projects/projects.py +++ b/python/ray/projects/projects.py @@ -96,6 +96,15 @@ class ProjectDefinition: if wildcards and "choices" in param: choices[name] = copy.deepcopy(param["choices"]) param["choices"] = param["choices"] + ["*"] + if "type" in param: + types = {"int": int, "str": str, "float": float} + if param["type"] in types: + param["type"] = types[param["type"]] + else: + raise ValueError( + "Parameter {} has type {} which is not supported. " + "Type must be one of {}".format( + name, param["type"], list(types.keys()))) parser.add_argument("--" + name, **param) parsed_args = parser.parse_args(list(args)).__dict__ diff --git a/python/ray/projects/schema.json b/python/ray/projects/schema.json index 555d290f5..bb0fb62fd 100644 --- a/python/ray/projects/schema.json +++ b/python/ray/projects/schema.json @@ -13,6 +13,17 @@ "description": "The URL of the repo this project is part of", "type": "string" }, + "tags": { + "description": "Relevant tags for this project", + "type": "array", + "items": { + "type": "string" + } + }, + "documentation": { + "description": "Link to the documentation of this project", + "type": "string" + }, "cluster": { "description": "Path to a .yaml cluster configuration file (relative to the project root)", "type": "string" diff --git a/python/ray/projects/scripts.py b/python/ray/projects/scripts.py index a28417d6c..94e2ae736 100644 --- a/python/ray/projects/scripts.py +++ b/python/ray/projects/scripts.py @@ -238,7 +238,7 @@ class SessionRunner(object): stop=False, start=False, override_cluster_name=self.session_name, - port_forward=None, + port_forward=config.get("port_forward", None), )