Make it possible to run ray examples as projects (#5816)

2026-07-13 01:51:51 +08:00 · 2019-10-03 14:52:37 -07:00
parent 972dddd776
commit 0dee225ce1
23 changed files with 392 additions and 5 deletions
@@ -0,0 +1,18 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: ray-example-cython
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 1
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
@@ -0,0 +1,46 @@
+# This file is generated by `ray project create`.
+
+name: ray-example-cython
+
+description: "Example of how to use Cython with ray"
+tags: ["ray-example", "cython"]
+documentation: https://ray.readthedocs.io/en/latest/advanced.html#cython-code-in-ray
+
+cluster: .rayproject/cluster.yaml
+
+environment:
+  requirements: .rayproject/requirements.txt
+
+  shell: # Shell commands to be ran for environment setup.
+    - pip install -e .
+
+commands:
+  - name: example1
+    command: python cython_main.py example1
+    help: "Run a simple Cython function"
+  - name: example2
+    command: python cython_main.py example2
+    help: "Run a simple recursive Cython function"
+  - name: example3
+    command: python cython_main.py example3
+    help: "Run a simple typed Cython function"
+  - name: example4
+    command: python cython_main.py example4
+    help: "Run a Cython defined with cpdef"
+  - name: example5
+    command: python cython_main.py example5
+    help: "Run a Cython defined with cdef"
+  - name: example6
+    command: python cython_main.py example6
+    help: "Run a simple Cython class"
+  - name: example7
+    command: python cython_main.py example7
+    help: "Run Cython with function from BrainIAK"
+  - name: example8
+    command: python cython_main.py example8
+    help: "Run Cython with BLAS"
+
+output_files: [
+  # Save the logs from the latest run in snapshots.
+  "/tmp/ray/session_latest/logs"
+]
@@ -0,0 +1,2 @@
+ray[debug]
+scipy
@@ -0,0 +1,18 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: ray-example-lbfgs
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 1
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
@@ -0,0 +1,22 @@
+# This file is generated by `ray project create`.
+
+name: ray-example-lbfgs
+
+description: "Parallelizing the L-BFGS algorithm in ray"
+tags: ["ray-example", "optimization", "lbfgs"]
+documentation: https://ray.readthedocs.io/en/latest/auto_examples/plot_lbfgs.html
+
+cluster: .rayproject/cluster.yaml
+
+environment:
+  requirements: .rayproject/requirements.txt
+
+commands:
+  - name: run
+    command: python driver.py
+    help: "Run the L-BFGS example"
+
+output_files: [
+  # Save the logs from the latest run in snapshots.
+  "/tmp/ray/session_latest/logs"
+]
@@ -0,0 +1 @@
+ray[debug,rllib]
@@ -0,0 +1,31 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: ray-example-newsreader
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 1
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+
+head_node:
+    InstanceType: m5.large
+    ImageId: ami-06f2f779464715dc5
+
+setup_commands:
+  # Install Anaconda.
+  - wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true
+  - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true
+  - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
+  - sudo apt -y update
+  - sudo apt -y install npm
+  - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.8.0.dev5-cp36-cp36m-manylinux1_x86_64.whl
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
@@ -0,0 +1,31 @@
+# This file is generated by `ray project create`.
+
+name: ray-example-newsreader
+
+description: "A simple news reader example that uses ray actors to serve requests"
+tags: ["ray-example", "flask", "rss", "newsreader"]
+documentation: https://ray.readthedocs.io/en/latest/auto_examples/plot_newsreader.html
+
+cluster: .rayproject/cluster.yaml
+
+environment:
+  requirements: .rayproject/requirements.txt
+
+commands:
+  - name: run-backend
+    command: python server.py
+    config:
+      port_forward: 5000
+  - name: run-frontend
+    command: |
+      git clone https://github.com/ray-project/qreader
+      cd qreader
+      npm install
+      npm run dev
+    config:
+      port_forward: 8080
+
+output_files: [
+  # Save the logs from the latest run in snapshots.
+  "/tmp/ray/session_latest/logs"
+]
@@ -0,0 +1,3 @@
+ray[debug]
+atoma
+flask
@@ -0,0 +1,18 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: ray-example-parameter-server
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 1
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
@@ -0,0 +1,40 @@
+# This file is generated by `ray project create`.
+
+name: ray-example-parameter-server
+
+description: "A simple parameter server example implemented with ray actors"
+tags: ["ray-example", "parameter-server", "machine-learning"]
+documentation: https://ray.readthedocs.io/en/latest/auto_examples/plot_parameter_server.html
+
+cluster: .rayproject/cluster.yaml
+
+environment:
+  requirements: .rayproject/requirements.txt
+
+commands:
+  - name: run-sync
+    command: python sync_parameter_server.py --num-workers {{num_workers}}
+    help: "Start the synchronous parameter server."
+    params:
+      - name: num-workers
+        help: "Number of workers"
+        default: 4
+        type: int
+    config:
+      tmux: true
+
+  - name: run-async
+    command: python async_parameter_server.py --num-workers {{num_workers}}
+    help: "Start the asynchronous parameter server."
+    params:
+      - name: num-workers
+        help: "Number of workers"
+        default: 4
+        type: int
+    config:
+      tmux: true
+
+output_files: [
+  # Save the logs from the latest run in snapshots.
+  "/tmp/ray/session_latest/logs"
+]
@@ -0,0 +1,4 @@
+ray[debug,rllib]
+torch
+torchvision
+filelock
@@ -0,0 +1,22 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: ray-example-resnet
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 1
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-0b294f219d14e6a82 # Deep Learning AMI (Ubuntu) Version 21.0
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
@@ -0,0 +1,58 @@
+# This file is generated by `ray project create`.
+
+name: ray-example-resnet
+
+description: "Using ray to train resnet on multiple gpus"
+tags: ["ray-example", "machine-learning", "tensorflow", "resnet"]
+documentation: https://ray.readthedocs.io/en/latest/auto_examples/plot_resnet.html
+
+cluster: .rayproject/cluster.yaml
+
+environment:
+  requirements: .rayproject/requirements.txt
+
+commands:
+  - name: train
+    command: |
+      if [ "{{dataset}}" == "cifar10" ]; then
+        # Get the CIFAR-10 dataset.
+        curl -o cifar-10-binary.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz;
+        tar -xvf cifar-10-binary.tar.gz;
+      else
+        # Get the CIFAR-100 dataset.
+        curl -o cifar-100-binary.tar.gz https://www.cs.toronto.edu/~kriz/cifar-100-binary.tar.gz;
+        tar -xvf cifar-100-binary.tar.gz;
+      fi
+      python resnet_main.py --dataset {{dataset}} --train_data_path {{train_data_path}} --eval_data_path {{eval_data_path}} --eval_dir {{eval_data_path}} --eval_batch_count {{eval_batch_count}} --num_gpus {{num_gpus}}
+    params:
+      - name: dataset
+        help: "The dataset to train on."
+        default: "cifar10"
+        choices: ["cifar10", "cifar100"]
+      - name: train-data-path
+        help: "Data path for the training data."
+        default: "'cifar-10-batches-bin/data_batch*'"
+        type: str
+      - name: eval-data-path
+        help: "Data path for the testing data."
+        default: "cifar-10-batches-bin/test_batch.bin"
+        type: str
+      - name: eval-dir
+        help: "Data path for the tensorboard logs."
+        default: "/tmp/resnet-model/eval"
+        type: str
+      - name: eval-batch-count
+        help: "Number of batches to evaluate over."
+        default: 50
+        type: int
+      - name: num-gpus
+        help: "Number of GPUs to use for training."
+        default: 0
+        type: int
+    config:
+      tmux: true
+
+output_files: [
+  # Save the logs from the latest run in snapshots.
+  "/tmp/ray/session_latest/logs"
+]
@@ -0,0 +1 @@
+ray[rllib,debug]
@@ -47,7 +47,7 @@ def build_data(data_path, size, dataset):

    # Read examples from files in the filename queue.
    data_files = tf.gfile.Glob(data_path)
-    data = tf.contrib.data.FixedLengthRecordDataset(
+    data = tf.data.FixedLengthRecordDataset(
        data_files, record_bytes=record_bytes)
    data = data.map(load_transform)
    data = data.batch(size)
@@ -76,7 +76,7 @@ def build_input(data, batch_size, dataset, train):
    num_classes = 10 if dataset == "cifar10" else 100
    images, labels = data
    num_samples = images.shape[0] - images.shape[0] % batch_size
-    dataset = tf.contrib.data.Dataset.from_tensor_slices(
+    dataset = tf.data.Dataset.from_tensor_slices(
        (images[:num_samples], labels[:num_samples]))

    def map_train(image, label):
@@ -0,0 +1,18 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: ray-example-streaming
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 1
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
@@ -0,0 +1,32 @@
+# This file is generated by `ray project create`.
+
+name: ray-example-streaming
+
+description: "A simple ray example for a streaming wordcount"
+tags: ["ray-example", "streaming", "wordcount", "data-processing"]
+
+cluster: .rayproject/cluster.yaml
+
+environment:
+  requirements: .rayproject/requirements.txt
+
+commands:
+  - name: run
+    command: python streaming.py --num-mappers {{num_mappers}} --num-reducers {{num_reducers}}
+    help: "Start the streaming example."
+    params:
+      - name: num-mappers
+        help: "Number of mapper actors used"
+        default: 3
+        type: int
+      - name: num-reducers
+        help: "Number of reducer actors used"
+        default: 4
+        type: int
+    config:
+      tmux: true
+
+output_files: [
+  # Save the logs from the latest run in snapshots.
+  "/tmp/ray/session_latest/logs"
+]
@@ -0,0 +1,2 @@
+ray[debug]
+wikipedia
@@ -8,9 +8,9 @@ import wikipedia

 parser = argparse.ArgumentParser()
 parser.add_argument("--num-mappers",
-                    help="number of mapper actors used", default=3)
+                    help="number of mapper actors used", default=3, type=int)
 parser.add_argument("--num-reducers",
-                    help="number of reducer actors used", default=4)
+                    help="number of reducer actors used", default=4, type=int)


@ray.remote
@@ -96,6 +96,15 @@ class ProjectDefinition:
            if wildcards and "choices" in param:
                choices[name] = copy.deepcopy(param["choices"])
                param["choices"] = param["choices"] + ["*"]
+            if "type" in param:
+                types = {"int": int, "str": str, "float": float}
+                if param["type"] in types:
+                    param["type"] = types[param["type"]]
+                else:
+                    raise ValueError(
+                        "Parameter {} has type {} which is not supported. "
+                        "Type must be one of {}".format(
+                            name, param["type"], list(types.keys())))
            parser.add_argument("--" + name, **param)

        parsed_args = parser.parse_args(list(args)).__dict__
@@ -13,6 +13,17 @@
      "description": "The URL of the repo this project is part of",
      "type": "string"
    },
+    "tags": {
+      "description": "Relevant tags for this project",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "documentation": {
+      "description": "Link to the documentation of this project",
+      "type": "string"
+    },
    "cluster": {
      "description": "Path to a .yaml cluster configuration file (relative to the project root)",
      "type": "string"
@@ -238,7 +238,7 @@ class SessionRunner(object):
            stop=False,
            start=False,
            override_cluster_name=self.session_name,
-            port_forward=None,
+            port_forward=config.get("port_forward", None),
        )