From 145ebe14c75d9050895ba9dd37912ac46a0891a8 Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Mon, 9 Mar 2020 06:29:10 +0100 Subject: [PATCH] added Azure Resource Manager (ARM) template (#7494) * added Azure Resource Manager (ARM) template * removed Azure doc (moved to separate PR) * nit * fixpaths * nit Co-authored-by: Richard Liaw --- doc/azure/azure-init.sh | 93 ++++++ doc/azure/azure-ray-template.json | 496 ++++++++++++++++++++++++++++++ doc/source/autoscaling.rst | 28 +- 3 files changed, 616 insertions(+), 1 deletion(-) create mode 100755 doc/azure/azure-init.sh create mode 100644 doc/azure/azure-ray-template.json diff --git a/doc/azure/azure-init.sh b/doc/azure/azure-init.sh new file mode 100755 index 000000000..5bbf5cea0 --- /dev/null +++ b/doc/azure/azure-init.sh @@ -0,0 +1,93 @@ +#!/bin/sh + +USERNAME=$1 +CONDA_ENV=$2 +WHEEL=$3 +RAY_HEAD_IP=$4 +TYPE=$5 + +echo "Installing wheel..." +sudo -u $USERNAME -i /bin/bash -l -c "conda init bash" +sudo -u $USERNAME -i /bin/bash -l -c "conda activate $CONDA_ENV; pip install $WHEEL" + +echo "Setting up service scripts..." +cat > /home/$USERNAME/ray-head.sh << EOM +#!/bin/bash +conda activate $CONDA_ENV + +NUM_GPUS=\`nvidia-smi -L | wc -l\` + +ray stop +ulimit -n 65536 +ray start --head --redis-port=6379 --object-manager-port=8076 --num-gpus=\$NUM_GPUS --block --webui-host 0.0.0.0 +EOM + +cat > /home/$USERNAME/ray-worker.sh << EOM +#!/bin/bash +conda activate $CONDA_ENV + +NUM_GPUS=\`nvidia-smi -L | wc -l\` + +ray stop +ulimit -n 65536 + +while true +do + ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 --num-gpus=\$NUM_GPUS --block + echo Ray exited. Auto-restarting in 1 second... + sleep 1 +done +EOM + +cat > /home/$USERNAME/tensorboard.sh << EOM +#!/bin/bash + +conda activate $CONDA_ENV +mkdir -p /home/$USERNAME/ray_results + +tensorboard --bind_all --logdir=/home/$USERNAME/ray_results +EOM + +chmod +x /home/$USERNAME/ray-head.sh +chmod +x /home/$USERNAME/ray-worker.sh +chmod +x /home/$USERNAME/tensorboard.sh + +cat > /lib/systemd/system/ray.service << EOM +[Unit] + Description=Ray + +[Service] + Type=simple + User=$USERNAME + ExecStart=/bin/bash -l /home/$USERNAME/ray-$TYPE.sh + +[Install] +WantedBy=multi-user.target +EOM + +cat > /lib/systemd/system/tensorboard.service << EOM +[Unit] + Description=TensorBoard + +[Service] + Type=simple + User=$USERNAME + ExecStart=/bin/bash -l /home/$USERNAME/tensorboard.sh + +[Install] +WantedBy=multi-user.target +EOM + +echo "Configure ray to start at boot..." +systemctl enable ray + +echo "Starting ray..." +systemctl start ray + +if [ "$type" = "head" ]; then + echo "Configure TensorBoard to start at boot..." + systemctl enable tensorboard + + echo "Starting TensorBoard..." + systemctl start tensorboard +fi \ No newline at end of file diff --git a/doc/azure/azure-ray-template.json b/doc/azure/azure-ray-template.json new file mode 100644 index 000000000..3e603199a --- /dev/null +++ b/doc/azure/azure-ray-template.json @@ -0,0 +1,496 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "adminUsername": { + "type": "string", + "defaultValue": "ubuntu", + "metadata": { + "description": "Username for the Virtual Machine." + } + }, + "publicKey": { + "type": "securestring", + "metadata": { + "description": "SSH Key for the Virtual Machine" + } + }, + "adminPassword": { + "type": "securestring", + "metadata": { + "description": "Password for the Virtual Machine and JupyterLab" + } + }, + "vmSizeHead": { + "type": "string", + "defaultValue": "Standard_D2s_v3", + "metadata": { + "description": "The size of the head-node Virtual Machine" + } + }, + "vmSizeWorker": { + "type": "string", + "defaultValue": "Standard_D2s_v3", + "metadata": { + "description": "The size of the worker node Virtual Machine" + } + }, + "workerInitial": { + "type": "int", + "defaultValue": 1, + "minValue": 0, + "metadata": { + "description": "Initial number of worker nodes" + } + }, + "workerMin": { + "type": "int", + "defaultValue": 1, + "minValue": 0, + "metadata": { + "description": "Minimum number of worker nodes" + } + }, + "workerMax": { + "type": "int", + "defaultValue": 1, + "minValue": 0, + "metadata": { + "description": "Maximum number of worker nodes" + } + }, + "condaEnv": { + "type": "string", + "defaultValue": "py37_tensorflow", + "allowedValues": [ + "azureml_py36_automl", + "azureml_py36_pytorch", + "azureml_py36_tensorflow", + "py37_default", + "py37_pytorch", + "py37_tensorflow" + ], + "metadata": { + "description": "Conda environment to select (installed on DSVM)" + } + }, + "PythonPackages": { + "type": "string", + "defaultValue": "ray[rllib] gym[atari]", + "metadata": { + "description": "Python packages to install (space separated)" + } + }, + "PublicWebUI": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Open port for web UI" + } + } + }, + "variables": { + "azureScriptInitUrl": "https://raw.githubusercontent.com/eisber/ray/marcozo/arm/doc/azure/azure-init.sh", + "vmName": "ray-node", + "subnetWorkers": "10.32.0.0/16", + "subnetHead": "10.33.0.0/16", + "publicIpAddressName": "[concat(variables('vmName'), '-ip' )]", + "networkIpConfig": "[guid(resourceGroup().id, variables('vmName'))]", + "subnetRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'ray-vnet', 'ray-subnet')]", + "subnetHeadRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'ray-vnet', 'ray-subnet-head')]", + "osDiskType": "Standard_LRS", + "vmNameHead": "[concat(variables('vmName'), '-head')]", + "vmNameWorker": "[concat(variables('vmName'), '-workers')]", + "networkInterfaceName": "[concat(variables('vmName'), '-nic')]", + "subnetNetwork": "[split(variables('subnetHead'), '/')[0]]", + "headInternalIP": "[concat(substring(variables('subnetNetwork'), 0, lastIndexOf(variables('subnetNetwork'), '.')), '.5')]", + "imagePublisher": "microsoft-dsvm", + "imageOffer": "ubuntu-1804", + "imageSku": "1804", + "imageVersion": "latest" + }, + "resources": [ + { + "type": "Microsoft.Network/networkSecurityGroups", + "apiVersion": "2019-02-01", + "name": "ray-nsg", + "location": "[resourceGroup().location]", + "properties": { + "securityRules": [ + { + "name": "SSH", + "properties": { + "priority": 1000, + "protocol": "TCP", + "access": "Allow", + "direction": "Inbound", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "22" + } + }, + { + "name": "JupyterLab", + "properties": { + "priority": 1001, + "protocol": "TCP", + "access": "Allow", + "direction": "Inbound", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "8000" + } + }, + { + "name": "RayWebUI", + "properties": { + "priority": 1002, + "protocol": "TCP", + "access": "[if(parameters('PublicWebUI'), 'Allow', 'Deny')]", + "direction": "Inbound", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "8265" + } + }, + { + "name": "TensorBoard", + "properties": { + "priority": 1003, + "protocol": "TCP", + "access": "[if(parameters('PublicWebUI'), 'Allow', 'Deny')]", + "direction": "Inbound", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "6006" + } + } + ] + } + }, + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2019-11-01", + "name": "ray-vnet", + "location": "[resourceGroup().location]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[variables('subnetHead')]", + "[variables('subnetWorkers')]" + ] + }, + "subnets": [ + { + "name": "ray-subnet", + "properties": { + "addressPrefix": "[variables('subnetWorkers')]" + } + }, + { + "name": "ray-subnet-head", + "properties": { + "addressPrefix": "[variables('subnetHead')]" + } + } + ] + } + }, + { + "type": "Microsoft.Network/publicIpAddresses", + "apiVersion": "2019-02-01", + "name": "[variables('publicIpAddressName')]", + "location": "[resourceGroup().location]", + "properties": { + "publicIpAllocationMethod": "Static", + "publicIPAddressVersion": "IPv4" + }, + "sku": { + "name": "Basic", + "tier": "Regional" + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2018-10-01", + "name": "[variables('networkInterfaceName')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/publicIpAddresses/', variables('publicIpAddressName'))]", + "[resourceId('Microsoft.Network/networkSecurityGroups','ray-nsg')]" + ], + "properties": { + "ipConfigurations": [ + { + "name": "[variables('networkIpConfig')]", + "properties": { + "subnet": { + "id": "[variables('subnetHeadRef')]" + }, + "privateIPAllocationMethod": "Static", + "privateIPAddress": "[variables('headInternalIP')]", + "publicIpAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))]" + } + } + } + ], + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups','ray-nsg')]" + } + } + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2019-03-01", + "name": "[variables('vmNameHead')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/networkInterfaces/', variables('networkInterfaceName'))]" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('vmSizeHead')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "fromImage", + "managedDisk": { + "storageAccountType": "[variables('osDiskType')]" + } + }, + "imageReference": { + "publisher": "[variables('imagePublisher')]", + "offer": "[variables('imageOffer')]", + "sku": "[variables('imageSku')]", + "version": "[variables('imageVersion')]" + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + } + ] + }, + "osProfile": { + "computerName": "[variables('vmNameHead')]", + "adminUsername": "[parameters('adminUsername')]", + "adminPassword": "[parameters('adminPassword')]", + "linuxConfiguration": { + "disablePasswordAuthentication": false, + "ssh": { + "publicKeys": [ + { + "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]", + "keyData": "[parameters('publicKey')]" + } + ] + } + } + } + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "name": "[concat(variables('vmNameHead'), '/HeadNodeInitScript')]", + "apiVersion": "2017-03-30", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[concat('Microsoft.Compute/virtualMachines/', variables('vmNameHead'))]" + ], + "properties": { + "publisher": "Microsoft.Azure.Extensions", + "type": "CustomScript", + "typeHandlerVersion": "2.1", + "autoUpgradeMinorVersion": true, + "settings": { + "commandToExecute": "[concat('sh azure-init.sh ', parameters('adminUsername'), ' ', parameters('condaEnv'), ' \"', parameters('PythonPackages'), '\" ignore head 2>&1 >/var/log/ray-head.log')]", + "fileUris": [ + "[variables('azureScriptInitUrl')]" + ] + } + } + } + ] + }, + { + "type": "Microsoft.Compute/virtualMachineScaleSets", + "name": "[variables('vmNameWorker')]", + "location": "[resourceGroup().location]", + "apiVersion": "2019-07-01", + "dependsOn": [ + "Microsoft.Network/virtualNetworks/ray-vnet" + ], + "sku": { + "name": "[parameters('vmSizeWorker')]", + "tier": "Standard", + "capacity": "[parameters('workerInitial')]" + }, + "properties": { + "upgradePolicy": { + "mode": "Manual" + }, + "virtualMachineProfile": { + "storageProfile": { + "osDisk": { + "createOption": "fromImage", + "managedDisk": { + "storageAccountType": "[variables('osDiskType')]" + } + }, + "imageReference": { + "publisher": "[variables('imagePublisher')]", + "offer": "[variables('imageOffer')]", + "sku": "[variables('imageSku')]", + "version": "[variables('imageVersion')]" + } + }, + "osProfile": { + "computerNamePrefix": "[variables('vmNameWorker')]", + "adminUsername": "[parameters('adminUsername')]", + "adminPassword": "[parameters('adminPassword')]", + "linuxConfiguration": { + "disablePasswordAuthentication": false, + "ssh": { + "publicKeys": [ + { + "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]", + "keyData": "[parameters('publicKey')]" + } + ] + } + } + }, + "networkProfile": { + "networkInterfaceConfigurations": [ + { + "name": "[concat(variables('vmNameWorker'),'-nic')]", + "properties": { + "primary": true, + "ipConfigurations": [ + { + "name": "worker-ip-config", + "properties": { + "subnet": { + "id": "[variables('subnetRef')]" + } + } + } + ] + } + } + ] + }, + "extensionProfile": { + "extensions": [ + { + "name": "RayWorkerInitScript", + "properties": { + "publisher": "Microsoft.Azure.Extensions", + "type": "CustomScript", + "typeHandlerVersion": "2.1", + "autoUpgradeMinorVersion": true, + "settings": { + "commandToExecute": "[concat('sh azure-init.sh ', parameters('adminUsername'), ' ', parameters('condaEnv'), ' \"', parameters('PythonPackages'), '\" ', variables('headInternalIP'), ' worker 2>&1 >/var/log/ray-worker.log')]", + "fileUris": [ + "[variables('azureScriptInitUrl')]" + ] + } + } + } + ] + } + } + } + }, + { + "type": "Microsoft.Insights/autoscaleSettings", + "apiVersion": "2015-04-01", + "name": "cpuautoscale", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[concat('Microsoft.Compute/virtualMachineScaleSets/', variables('vmNameWorker'))]" + ], + "properties": { + "name": "cpuautoscale", + "targetResourceUri": "[concat('/subscriptions/',subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.Compute/virtualMachineScaleSets/', variables('vmNameWorker'))]", + "enabled": true, + "profiles": [ + { + "name": "Profile1", + "capacity": { + "minimum": "[parameters('workerMin')]", + "maximum": "[parameters('workerMax')]", + "default": "[parameters('workerInitial')]" + }, + "rules": [ + { + "metricTrigger": { + "metricName": "Percentage CPU", + "metricNamespace": "", + "metricResourceUri": "[concat('/subscriptions/',subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.Compute/virtualMachineScaleSets/', variables('vmNameWorker'))]", + "timeGrain": "PT1M", + "statistic": "Average", + "timeWindow": "PT10M", + "timeAggregation": "Average", + "operator": "GreaterThan", + "threshold": 80 + }, + "scaleAction": { + "direction": "Increase", + "type": "ChangeCount", + "value": "1", + "cooldown": "PT5M" + } + }, + { + "metricTrigger": { + "metricName": "Percentage CPU", + "metricNamespace": "", + "metricResourceUri": "[concat('/subscriptions/',subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.Compute/virtualMachineScaleSets/', variables('vmNameWorker'))]", + "timeGrain": "PT1M", + "statistic": "Average", + "timeWindow": "PT30M", + "timeAggregation": "Average", + "operator": "LessThan", + "threshold": 20 + }, + "scaleAction": { + "direction": "Decrease", + "type": "ChangeCount", + "value": "1", + "cooldown": "PT5M" + } + } + ] + } + ] + } + } + ], + "outputs": { + "JupyterLabURL": { + "type": "string", + "value": "[concat('https://', reference(variables('publicIpAddressName')).ipAddress, ':8000')]" + }, + "SSH": { + "type": "string", + "value": "[concat('ssh -t -L 8265:localhost:8265 -L 8888:localhost:8888 ', parameters('adminUsername'),'@', reference(variables('publicIpAddressName')).ipAddress)]" + }, + "RayWebUIURL": { + "type": "string", + "value": "[concat('http://', reference(variables('publicIpAddressName')).ipAddress, ':8265')]", + "condition": "[parameters('PublicWebUI')]" + }, + "TensorBoard": { + "type": "string", + "value": "[concat('http://', reference(variables('publicIpAddressName')).ipAddress, ':6006')]", + "condition": "[parameters('PublicWebUI')]" + } + } +} \ No newline at end of file diff --git a/doc/source/autoscaling.rst b/doc/source/autoscaling.rst index e5b3d6d22..d4f7b6021 100644 --- a/doc/source/autoscaling.rst +++ b/doc/source/autoscaling.rst @@ -8,7 +8,7 @@ Ray comes with a built-in autoscaler that makes deploying a Ray cluster simple, Setup ----- -This section provides instructions for configuring the autoscaler to launch a Ray cluster on AWS/GCP, an existing Kubernetes cluster, or on a private cluster of host machines. +This section provides instructions for configuring the autoscaler to launch a Ray cluster on AWS/Azure/GCP, an existing Kubernetes cluster, or on a private cluster of host machines. Once you have finished configuring the autoscaler to create a cluster, see the Quickstart guide below for more details on how to get started running Ray programs on it. @@ -40,6 +40,32 @@ Test that it works by running the following commands from your local machine: .. note:: You may see a message like: ``bash: cannot set terminal process group (-1): Inappropriate ioctl for device bash: no job control in this shell`` This is a harmless error. If the cluster launcher fails, it is most likely due to some other factor. +Azure Portal +~~~~~~~~~~~~ + +Alternatively, you can deploy a cluster using Azure portal directly. Please note that auto scaling is done using Azure VM Scale Sets and not through +the Ray autoscaler. This will deploy `Azure Data Science VMs (DSVM) `_ +for both the head node and an auto-scale cluster managed by `Azure Virtual Machine Scale Sets `_. +The head node conviently exposes both SSH as well as JupyterLab. + +.. image:: https://aka.ms/deploytoazurebutton + :target: https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fray-project%2Fray%2Fmaster%2Fdoc%2Fazure%2Fazure-ray-template.json + :alt: Deploy to Azure + +Once the template is successfully deploy the deployment output page provides the ssh command to connect and the link to the JupyterHub on the head node (username/password as specified on the template input). +Use the following code connect to the Ray cluster. + +.. code-block:: python + + import ray + ray.init(address='auto') + +Note that on each node the `azure-init.sh `_ script is executed and performs + +1. activate one of the conda environments available on DSVM +2. install Ray and any other user-specified dependencies +3. setup of a systemd task (``/lib/systemd/system/ray.service``) which starting ray in head or worker mode + GCP ~~~