wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-12 12:51:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

added Azure Resource Manager (ARM) template (#7494)

Browse Source 
* added Azure Resource Manager (ARM) template

* removed Azure doc (moved to separate PR)

* nit

* fixpaths

* nit

Co-authored-by: Richard Liaw <rliaw@berkeley.edu>
This commit is contained in:
Markus Cozowicz
2020-03-08 22:29:10 -07:00
committed by GitHub
co-authored by Richard Liaw
parent e7bc5c612d
commit 145ebe14c7
3 changed files with 616 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/azure/azure-init.sh
Executable
+93
View File
@@ -0,0 +1,93 @@
#!/bin/sh
USERNAME=$1
CONDA_ENV=$2
WHEEL=$3
RAY_HEAD_IP=$4
TYPE=$5
echo "Installing wheel..."
sudo -u $USERNAME -i /bin/bash -l -c "conda init bash"
sudo -u $USERNAME -i /bin/bash -l -c "conda activate $CONDA_ENV; pip install $WHEEL"
echo "Setting up service scripts..."
cat > /home/$USERNAME/ray-head.sh << EOM
#!/bin/bash
conda activate $CONDA_ENV
NUM_GPUS=\`nvidia-smi -L | wc -l\`
ray stop
ulimit -n 65536
ray start --head --redis-port=6379 --object-manager-port=8076 --num-gpus=\$NUM_GPUS --block --webui-host 0.0.0.0
EOM
cat > /home/$USERNAME/ray-worker.sh << EOM
#!/bin/bash
conda activate $CONDA_ENV
NUM_GPUS=\`nvidia-smi -L | wc -l\`
ray stop
ulimit -n 65536
while true
do
ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 --num-gpus=\$NUM_GPUS --block
echo Ray exited. Auto-restarting in 1 second...
sleep 1
done
EOM
cat > /home/$USERNAME/tensorboard.sh << EOM
#!/bin/bash
conda activate $CONDA_ENV
mkdir -p /home/$USERNAME/ray_results
tensorboard --bind_all --logdir=/home/$USERNAME/ray_results
EOM
chmod +x /home/$USERNAME/ray-head.sh
chmod +x /home/$USERNAME/ray-worker.sh
chmod +x /home/$USERNAME/tensorboard.sh
cat > /lib/systemd/system/ray.service << EOM
[Unit]
Description=Ray
[Service]
Type=simple
User=$USERNAME
ExecStart=/bin/bash -l /home/$USERNAME/ray-$TYPE.sh
[Install]
WantedBy=multi-user.target
EOM
cat > /lib/systemd/system/tensorboard.service << EOM
[Unit]
Description=TensorBoard
[Service]
Type=simple
User=$USERNAME
ExecStart=/bin/bash -l /home/$USERNAME/tensorboard.sh
[Install]
WantedBy=multi-user.target
EOM
echo "Configure ray to start at boot..."
systemctl enable ray
echo "Starting ray..."
systemctl start ray
if [ "$type" = "head" ]; then
echo "Configure TensorBoard to start at boot..."
systemctl enable tensorboard
echo "Starting TensorBoard..."
systemctl start tensorboard
fi
doc/azure/azure-ray-template.json
+496
View File
@@ -0,0 +1,496 @@
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"adminUsername": {
"type": "string",
"defaultValue": "ubuntu",
"metadata": {
"description": "Username for the Virtual Machine."
}
},
"publicKey": {
"type": "securestring",
"metadata": {
"description": "SSH Key for the Virtual Machine"
}
},
"adminPassword": {
"type": "securestring",
"metadata": {
"description": "Password for the Virtual Machine and JupyterLab"
}
},
"vmSizeHead": {
"type": "string",
"defaultValue": "Standard_D2s_v3",
"metadata": {
"description": "The size of the head-node Virtual Machine"
}
},
"vmSizeWorker": {
"type": "string",
"defaultValue": "Standard_D2s_v3",
"metadata": {
"description": "The size of the worker node Virtual Machine"
}
},
"workerInitial": {
"type": "int",
"defaultValue": 1,
"minValue": 0,
"metadata": {
"description": "Initial number of worker nodes"
}
},
"workerMin": {
"type": "int",
"defaultValue": 1,
"minValue": 0,
"metadata": {
"description": "Minimum number of worker nodes"
}
},
"workerMax": {
"type": "int",
"defaultValue": 1,
"minValue": 0,
"metadata": {
"description": "Maximum number of worker nodes"
}
},
"condaEnv": {
"type": "string",
"defaultValue": "py37_tensorflow",
"allowedValues": [
"azureml_py36_automl",
"azureml_py36_pytorch",
"azureml_py36_tensorflow",
"py37_default",
"py37_pytorch",
"py37_tensorflow"
],
"metadata": {
"description": "Conda environment to select (installed on DSVM)"
}
},
"PythonPackages": {
"type": "string",
"defaultValue": "ray[rllib] gym[atari]",
"metadata": {
"description": "Python packages to install (space separated)"
}
},
"PublicWebUI": {
"type": "bool",
"defaultValue": true,
"metadata": {
"description": "Open port for web UI"
}
}
},
"variables": {
"azureScriptInitUrl": "https://raw.githubusercontent.com/eisber/ray/marcozo/arm/doc/azure/azure-init.sh",
"vmName": "ray-node",
"subnetWorkers": "10.32.0.0/16",
"subnetHead": "10.33.0.0/16",
"publicIpAddressName": "[concat(variables('vmName'), '-ip' )]",
"networkIpConfig": "[guid(resourceGroup().id, variables('vmName'))]",
"subnetRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'ray-vnet', 'ray-subnet')]",
"subnetHeadRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'ray-vnet', 'ray-subnet-head')]",
"osDiskType": "Standard_LRS",
"vmNameHead": "[concat(variables('vmName'), '-head')]",
"vmNameWorker": "[concat(variables('vmName'), '-workers')]",
"networkInterfaceName": "[concat(variables('vmName'), '-nic')]",
"subnetNetwork": "[split(variables('subnetHead'), '/')[0]]",
"headInternalIP": "[concat(substring(variables('subnetNetwork'), 0, lastIndexOf(variables('subnetNetwork'), '.')), '.5')]",
"imagePublisher": "microsoft-dsvm",
"imageOffer": "ubuntu-1804",
"imageSku": "1804",
"imageVersion": "latest"
},
"resources": [
{
"type": "Microsoft.Network/networkSecurityGroups",
"apiVersion": "2019-02-01",
"name": "ray-nsg",
"location": "[resourceGroup().location]",
"properties": {
"securityRules": [
{
"name": "SSH",
"properties": {
"priority": 1000,
"protocol": "TCP",
"access": "Allow",
"direction": "Inbound",
"sourceAddressPrefix": "*",
"sourcePortRange": "*",
"destinationAddressPrefix": "*",
"destinationPortRange": "22"
}
},
{
"name": "JupyterLab",
"properties": {
"priority": 1001,
"protocol": "TCP",
"access": "Allow",
"direction": "Inbound",
"sourceAddressPrefix": "*",
"sourcePortRange": "*",
"destinationAddressPrefix": "*",
"destinationPortRange": "8000"
}
},
{
"name": "RayWebUI",
"properties": {
"priority": 1002,
"protocol": "TCP",
"access": "[if(parameters('PublicWebUI'), 'Allow', 'Deny')]",
"direction": "Inbound",
"sourceAddressPrefix": "*",
"sourcePortRange": "*",
"destinationAddressPrefix": "*",
"destinationPortRange": "8265"
}
},
{
"name": "TensorBoard",
"properties": {
"priority": 1003,
"protocol": "TCP",
"access": "[if(parameters('PublicWebUI'), 'Allow', 'Deny')]",
"direction": "Inbound",
"sourceAddressPrefix": "*",
"sourcePortRange": "*",
"destinationAddressPrefix": "*",
"destinationPortRange": "6006"
}
}
]
}
},
{
"type": "Microsoft.Network/virtualNetworks",
"apiVersion": "2019-11-01",
"name": "ray-vnet",
"location": "[resourceGroup().location]",
"properties": {
"addressSpace": {
"addressPrefixes": [
"[variables('subnetHead')]",
"[variables('subnetWorkers')]"
]
},
"subnets": [
{
"name": "ray-subnet",
"properties": {
"addressPrefix": "[variables('subnetWorkers')]"
}
},
{
"name": "ray-subnet-head",
"properties": {
"addressPrefix": "[variables('subnetHead')]"
}
}
]
}
},
{
"type": "Microsoft.Network/publicIpAddresses",
"apiVersion": "2019-02-01",
"name": "[variables('publicIpAddressName')]",
"location": "[resourceGroup().location]",
"properties": {
"publicIpAllocationMethod": "Static",
"publicIPAddressVersion": "IPv4"
},
"sku": {
"name": "Basic",
"tier": "Regional"
}
},
{
"type": "Microsoft.Network/networkInterfaces",
"apiVersion": "2018-10-01",
"name": "[variables('networkInterfaceName')]",
"location": "[resourceGroup().location]",
"dependsOn": [
"[resourceId('Microsoft.Network/publicIpAddresses/', variables('publicIpAddressName'))]",
"[resourceId('Microsoft.Network/networkSecurityGroups','ray-nsg')]"
],
"properties": {
"ipConfigurations": [
{
"name": "[variables('networkIpConfig')]",
"properties": {
"subnet": {
"id": "[variables('subnetHeadRef')]"
},
"privateIPAllocationMethod": "Static",
"privateIPAddress": "[variables('headInternalIP')]",
"publicIpAddress": {
"id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))]"
}
}
}
],
"networkSecurityGroup": {
"id": "[resourceId('Microsoft.Network/networkSecurityGroups','ray-nsg')]"
}
}
},
{
"type": "Microsoft.Compute/virtualMachines",
"apiVersion": "2019-03-01",
"name": "[variables('vmNameHead')]",
"location": "[resourceGroup().location]",
"dependsOn": [
"[resourceId('Microsoft.Network/networkInterfaces/', variables('networkInterfaceName'))]"
],
"properties": {
"hardwareProfile": {
"vmSize": "[parameters('vmSizeHead')]"
},
"storageProfile": {
"osDisk": {
"createOption": "fromImage",
"managedDisk": {
"storageAccountType": "[variables('osDiskType')]"
}
},
"imageReference": {
"publisher": "[variables('imagePublisher')]",
"offer": "[variables('imageOffer')]",
"sku": "[variables('imageSku')]",
"version": "[variables('imageVersion')]"
}
},
"networkProfile": {
"networkInterfaces": [
{
"id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]"
}
]
},
"osProfile": {
"computerName": "[variables('vmNameHead')]",
"adminUsername": "[parameters('adminUsername')]",
"adminPassword": "[parameters('adminPassword')]",
"linuxConfiguration": {
"disablePasswordAuthentication": false,
"ssh": {
"publicKeys": [
{
"path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]",
"keyData": "[parameters('publicKey')]"
}
]
}
}
}
},
"resources": [
{
"type": "Microsoft.Compute/virtualMachines/extensions",
"name": "[concat(variables('vmNameHead'), '/HeadNodeInitScript')]",
"apiVersion": "2017-03-30",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Compute/virtualMachines/', variables('vmNameHead'))]"
],
"properties": {
"publisher": "Microsoft.Azure.Extensions",
"type": "CustomScript",
"typeHandlerVersion": "2.1",
"autoUpgradeMinorVersion": true,
"settings": {
"commandToExecute": "[concat('sh azure-init.sh ', parameters('adminUsername'), ' ', parameters('condaEnv'), ' \"', parameters('PythonPackages'), '\" ignore head 2>&1 >/var/log/ray-head.log')]",
"fileUris": [
"[variables('azureScriptInitUrl')]"
]
}
}
}
]
},
{
"type": "Microsoft.Compute/virtualMachineScaleSets",
"name": "[variables('vmNameWorker')]",
"location": "[resourceGroup().location]",
"apiVersion": "2019-07-01",
"dependsOn": [
"Microsoft.Network/virtualNetworks/ray-vnet"
],
"sku": {
"name": "[parameters('vmSizeWorker')]",
"tier": "Standard",
"capacity": "[parameters('workerInitial')]"
},
"properties": {
"upgradePolicy": {
"mode": "Manual"
},
"virtualMachineProfile": {
"storageProfile": {
"osDisk": {
"createOption": "fromImage",
"managedDisk": {
"storageAccountType": "[variables('osDiskType')]"
}
},
"imageReference": {
"publisher": "[variables('imagePublisher')]",
"offer": "[variables('imageOffer')]",
"sku": "[variables('imageSku')]",
"version": "[variables('imageVersion')]"
}
},
"osProfile": {
"computerNamePrefix": "[variables('vmNameWorker')]",
"adminUsername": "[parameters('adminUsername')]",
"adminPassword": "[parameters('adminPassword')]",
"linuxConfiguration": {
"disablePasswordAuthentication": false,
"ssh": {
"publicKeys": [
{
"path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]",
"keyData": "[parameters('publicKey')]"
}
]
}
}
},
"networkProfile": {
"networkInterfaceConfigurations": [
{
"name": "[concat(variables('vmNameWorker'),'-nic')]",
"properties": {
"primary": true,
"ipConfigurations": [
{
"name": "worker-ip-config",
"properties": {
"subnet": {
"id": "[variables('subnetRef')]"
}
}
}
]
}
}
]
},
"extensionProfile": {
"extensions": [
{
"name": "RayWorkerInitScript",
"properties": {
"publisher": "Microsoft.Azure.Extensions",
"type": "CustomScript",
"typeHandlerVersion": "2.1",
"autoUpgradeMinorVersion": true,
"settings": {
"commandToExecute": "[concat('sh azure-init.sh ', parameters('adminUsername'), ' ', parameters('condaEnv'), ' \"', parameters('PythonPackages'), '\" ', variables('headInternalIP'), ' worker 2>&1 >/var/log/ray-worker.log')]",
"fileUris": [
"[variables('azureScriptInitUrl')]"
]
}
}
}
]
}
}
}
},
{
"type": "Microsoft.Insights/autoscaleSettings",
"apiVersion": "2015-04-01",
"name": "cpuautoscale",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Compute/virtualMachineScaleSets/', variables('vmNameWorker'))]"
],
"properties": {
"name": "cpuautoscale",
"targetResourceUri": "[concat('/subscriptions/',subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.Compute/virtualMachineScaleSets/', variables('vmNameWorker'))]",
"enabled": true,
"profiles": [
{
"name": "Profile1",
"capacity": {
"minimum": "[parameters('workerMin')]",
"maximum": "[parameters('workerMax')]",
"default": "[parameters('workerInitial')]"
},
"rules": [
{
"metricTrigger": {
"metricName": "Percentage CPU",
"metricNamespace": "",
"metricResourceUri": "[concat('/subscriptions/',subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.Compute/virtualMachineScaleSets/', variables('vmNameWorker'))]",
"timeGrain": "PT1M",
"statistic": "Average",
"timeWindow": "PT10M",
"timeAggregation": "Average",
"operator": "GreaterThan",
"threshold": 80
},
"scaleAction": {
"direction": "Increase",
"type": "ChangeCount",
"value": "1",
"cooldown": "PT5M"
}
},
{
"metricTrigger": {
"metricName": "Percentage CPU",
"metricNamespace": "",
"metricResourceUri": "[concat('/subscriptions/',subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.Compute/virtualMachineScaleSets/', variables('vmNameWorker'))]",
"timeGrain": "PT1M",
"statistic": "Average",
"timeWindow": "PT30M",
"timeAggregation": "Average",
"operator": "LessThan",
"threshold": 20
},
"scaleAction": {
"direction": "Decrease",
"type": "ChangeCount",
"value": "1",
"cooldown": "PT5M"
}
}
]
}
]
}
}
],
"outputs": {
"JupyterLabURL": {
"type": "string",
"value": "[concat('https://', reference(variables('publicIpAddressName')).ipAddress, ':8000')]"
},
"SSH": {
"type": "string",
"value": "[concat('ssh -t -L 8265:localhost:8265 -L 8888:localhost:8888 ', parameters('adminUsername'),'@', reference(variables('publicIpAddressName')).ipAddress)]"
},
"RayWebUIURL": {
"type": "string",
"value": "[concat('http://', reference(variables('publicIpAddressName')).ipAddress, ':8265')]",
"condition": "[parameters('PublicWebUI')]"
},
"TensorBoard": {
"type": "string",
"value": "[concat('http://', reference(variables('publicIpAddressName')).ipAddress, ':6006')]",
"condition": "[parameters('PublicWebUI')]"
}
}
}
doc/source/autoscaling.rst
+27 -1
View File
@@ -8,7 +8,7 @@ Ray comes with a built-in autoscaler that makes deploying a Ray cluster simple,
Setup
-----
This section provides instructions for configuring the autoscaler to launch a Ray cluster on AWS/GCP, an existing Kubernetes cluster, or on a private cluster of host machines.
This section provides instructions for configuring the autoscaler to launch a Ray cluster on AWS/Azure/GCP, an existing Kubernetes cluster, or on a private cluster of host machines.
Once you have finished configuring the autoscaler to create a cluster, see the Quickstart guide below for more details on how to get started running Ray programs on it.
@@ -40,6 +40,32 @@ Test that it works by running the following commands from your local machine:
.. note:: You may see a message like: ``bash: cannot set terminal process group (-1): Inappropriate ioctl for device bash: no job control in this shell`` This is a harmless error. If the cluster launcher fails, it is most likely due to some other factor.
Azure Portal
~~~~~~~~~~~~
Alternatively, you can deploy a cluster using Azure portal directly. Please note that auto scaling is done using Azure VM Scale Sets and not through
the Ray autoscaler. This will deploy `Azure Data Science VMs (DSVM) <https://azure.microsoft.com/en-us/services/virtual-machines/data-science-virtual-machines/>`_
for both the head node and an auto-scale cluster managed by `Azure Virtual Machine Scale Sets <https://azure.microsoft.com/en-us/services/virtual-machine-scale-sets/>`_.
The head node conviently exposes both SSH as well as JupyterLab.
.. image:: https://aka.ms/deploytoazurebutton
:target: https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fray-project%2Fray%2Fmaster%2Fdoc%2Fazure%2Fazure-ray-template.json
:alt: Deploy to Azure
Once the template is successfully deploy the deployment output page provides the ssh command to connect and the link to the JupyterHub on the head node (username/password as specified on the template input).
Use the following code connect to the Ray cluster.
.. code-block:: python
import ray
ray.init(address='auto')
Note that on each node the `azure-init.sh <https://github.com/ray-project/ray/blob/master/doc/azure/azure-init.sh>`_ script is executed and performs
1. activate one of the conda environments available on DSVM
2. install Ray and any other user-specified dependencies
3. setup of a systemd task (``/lib/systemd/system/ray.service``) which starting ray in head or worker mode
GCP
~~~