Release testing things

doc/dev/release_logs/1.0.0/microbenchmark.txt

@@ -0,0 +1,22 @@
+single client get calls per second 30584.85 +- 661.06
+single client put calls per second 23923.57 +- 360.12
+multi client put calls per second 124576.32 +- 1847.41
+single client get calls (Plasma Store) per second 7940.16 +- 456.8
+single client put calls (Plasma Store) per second 4956.74 +- 31.4
+multi client put calls (Plasma Store) per second 7705.71 +- 120.7
+single client put gigabytes per second 13.74 +- 11.88
+multi client put gigabytes per second 36.83 +- 1.89
+single client tasks sync per second 977.6 +- 9.39
+single client tasks async per second 14228.3 +- 195.64
+multi client tasks async per second 36049.11 +- 790.33
+1:1 actor calls sync per second 1454.92 +- 20.51
+1:1 actor calls async per second 6612.21 +- 195.01
+1:1 actor calls concurrent per second 6034.76 +- 141.42
+1:n actor calls async per second 14032.19 +- 434.44
+n:n actor calls async per second 36895.26 +- 642.94
+n:n actor calls with arg async per second 9548.31 +- 218.19
+1:1 async-actor calls sync per second 1031.86 +- 17.59
+1:1 async-actor calls async per second 3675.11 +- 37.81
+1:1 async-actor calls with args async per second 2268.17 +- 20.47
+1:n async-actor calls async per second 11776.12 +- 641.59
+n:n async-actor calls async per second 24358.79 +- 362.87

doc/dev/release_logs/1.0.1/microbenchmark.txt

@@ -0,0 +1,22 @@
+single client get calls per second 30921.88 +- 291.82
+single client put calls per second 26506.75 +- 278.09
+multi client put calls per second 129520.73 +- 2304.11
+single client get calls (Plasma Store) per second 7555.25 +- 109.32
+single client put calls (Plasma Store) per second 4936.57 +- 31.22
+multi client put calls (Plasma Store) per second 7459.17 +- 271.1
+single client put gigabytes per second 11.87 +- 12.03
+multi client put gigabytes per second 36.23 +- 2.17
+single client tasks sync per second 1044.57 +- 16.2
+single client tasks async per second 14319.28 +- 105.0
+multi client tasks async per second 35140.53 +- 498.82
+1:1 actor calls sync per second 1545.69 +- 25.38
+1:1 actor calls async per second 6656.71 +- 268.57
+1:1 actor calls concurrent per second 5851.38 +- 75.39
+1:n actor calls async per second 11856.28 +- 322.15
+n:n actor calls async per second 37089.65 +- 847.78
+n:n actor calls with arg async per second 9919.05 +- 85.45
+1:1 async-actor calls sync per second 1027.93 +- 31.65
+1:1 async-actor calls async per second 3861.91 +- 96.54
+1:1 async-actor calls with args async per second 2351.35 +- 48.13
+1:n async-actor calls async per second 12652.4 +- 637.73
+n:n async-actor calls async per second 24527.67 +- 743.48

release/long_running_tests/actor_deaths.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-actor-deaths-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/long_running_tests/apex.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-apex-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/long_running_tests/cluster.yaml

@@ -1,4 +1,4 @@
-cluster_name: default
+cluster_name: long-running-release-tests-alex
 min_workers: 0
 max_workers: 0
 target_utilization_fraction: 0.8
@@ -7,14 +7,14 @@ idle_timeout_minutes: 5
 # Cloud-provider specific configuration.
 provider:
     type: aws
-    region: us-west-2
-    availability_zone: us-west-2a
+    region: us-west-1
+    availability_zone: us-west-1a
 auth:
     ssh_user: ubuntu
 
 head_node:
     InstanceType: m5.2xlarge
-    ImageId: ami-0888a3b5189309429  # DLAMI 7/1/19
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
     BlockDeviceMappings:
         - DeviceName: /dev/sda1
           Ebs:
@@ -22,7 +22,7 @@ head_node:
 
 worker_nodes:
     InstanceType: m5.large
-    ImageId: ami-0888a3b5189309429  # DLAMI 7/1/19
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
     BlockDeviceMappings:
         - DeviceName: /dev/sda1
           Ebs:

release/long_running_tests/impala.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-apex-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/long_running_tests/many_actor_tasks.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-actor-tasks-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/long_running_tests/many_drivers.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-drivers-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/long_running_tests/many_tasks_serialized_ids.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-serialized-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/long_running_tests/node_failures.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-node-failures-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/long_running_tests/pbt.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-pbt-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/long_running_tests/run.sh

@@ -53,7 +53,7 @@ wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ra
 echo set-window-option -g mouse on > ~/.tmux.conf
 echo 'termcapinfo xterm* ti@:te@' > ~/.screenrc
 # Serve load testing tool
-rm -r wrk || true && git clone https://github.com/wg/wrk.git wrk && cd wrk && make -j && sudo cp wrk /usr/local/bin
+rm -rf wrk || true && git clone https://github.com/wg/wrk.git wrk && cd wrk && make -j && sudo cp wrk /usr/local/bin
 pip install -U pip
 unset RAY_ADDRESS
 source activate tensorflow_p36 
@@ -62,5 +62,7 @@ pip install --upgrade pip
 pip install -U tensorflow==1.14
 pip install -q -U "$wheel" Click
 pip install -q "ray[all]" "gym[atari]"
-python "workloads/$workload.py"
+ray stop
+sleep 5
+python "$HOME/workloads/$workload.py"
 

release/long_running_tests/serve.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-serve-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/long_running_tests/serve_failure.yaml

@@ -0,0 +1,52 @@
+cluster_name: long-running-serve-failure-alex
+min_workers: 0
+max_workers: 0
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-1
+    availability_zone: us-west-1a
+auth:
+    ssh_user: ubuntu
+
+head_node:
+    InstanceType: m5.2xlarge
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+    InstanceType: m5.large
+    ImageId: ami-05fbe06ba1b862ef6  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+    # Run workers on spot by default. Comment this out to use on-demand.
+    InstanceMarketOptions:
+        MarketType: spot
+
+# List of shell commands to run to set up nodes.
+setup_commands: []
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands: []
+
+file_mounts:
+    "$HOME/run.sh": "run.sh"
+    "$HOME/workloads": "workloads"

release/microbenchmark/cluster.yaml

@@ -1,4 +1,4 @@
-cluster_name: ray-release-microbenchmark
+cluster_name: ray-release-microbenchmark-alex
 min_workers: 0
 max_workers: 0
 target_utilization_fraction: 0.8
@@ -7,14 +7,14 @@ idle_timeout_minutes: 5
 # Cloud-provider specific configuration.
 provider:
     type: aws
-    region: us-west-2
-    availability_zone: us-west-2a
+    region: us-west-1
+    availability_zone: us-west-1a
 auth:
     ssh_user: ubuntu
 
 head_node:
     InstanceType: m4.16xlarge
-    ImageId: ami-06d51e91cea0dac8d  # Ubuntu 18.04
+    ImageId: ami-03fac5402e10ea93b  # Ubuntu 18.04
     BlockDeviceMappings:
         - DeviceName: /dev/sda1
           Ebs:
@@ -22,7 +22,7 @@ head_node:
 
 worker_nodes:
     InstanceType: m5.large
-    ImageId: ami-06d51e91cea0dac8d  # Ubuntu 18.04
+    ImageId: ami-03fac5402e10ea93b  # Ubuntu 18.04
     BlockDeviceMappings:
         - DeviceName: /dev/sda1
           Ebs:

release/microbenchmark/run.sh

@@ -47,14 +47,16 @@ echo "commit: $commit"
 echo "branch: $ray_branch"
 echo "workload: $workload"
 
-wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp38-cp38-manylinux1_x86_64.whl"
+wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl"
 
 
 echo set-window-option -g mouse on > ~/.tmux.conf
 echo 'termcapinfo xterm* ti@:te@' > ~/.screenrc
+set -x
 pip uninstall -y -q ray
 pip install --upgrade pip
 pip install -U "$wheel"
+set +x
 
 unset RAY_ADDRESS
 OMP_NUM_THREADS=64 ray microbenchmark