From 08707f940806e08734e31e4e8309e5f03c41e97e Mon Sep 17 00:00:00 2001 From: Wapaul1 Date: Thu, 17 Nov 2016 22:33:29 -0800 Subject: [PATCH] Integration of Webui with Ray (#32) * Initial integration of webui with ray * Re-organized calling of build-webui in setup.py * Fixed Lint comments on js code * Fixed more lint issues * Fixed various issues * Fixed directory in services.py * Small changes. * Changes to match lint --- build-webui.sh | 32 ++++ doc/install-on-macosx.md | 2 +- doc/install-on-ubuntu.md | 2 +- install-dependencies.sh | 4 +- lib/python/MANIFEST.in | 1 + lib/python/ray/services.py | 17 ++ lib/python/setup.py | 6 + lib/python/webui/.gitkeep | 0 webui/.babelrc | 3 + webui/client/app/index.jsx | 305 ++++++++++++++++++++++++++++++++++ webui/client/index.html | 10 ++ webui/client/static/rayui.css | 84 ++++++++++ webui/index.js | 112 +++++++++++++ webui/package.json | 32 ++++ webui/task.js | 68 ++++++++ webui/webpack.config.js | 19 +++ 16 files changed, 693 insertions(+), 4 deletions(-) create mode 100755 build-webui.sh create mode 100644 lib/python/MANIFEST.in create mode 100644 lib/python/webui/.gitkeep create mode 100644 webui/.babelrc create mode 100644 webui/client/app/index.jsx create mode 100644 webui/client/index.html create mode 100644 webui/client/static/rayui.css create mode 100644 webui/index.js create mode 100644 webui/package.json create mode 100644 webui/task.js create mode 100644 webui/webpack.config.js diff --git a/build-webui.sh b/build-webui.sh new file mode 100755 index 000000000..1c7a726ef --- /dev/null +++ b/build-webui.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + platform="linux" +elif [[ "$unamestr" == "Darwin" ]]; then + platform="macosx" +else + echo "Unrecognized platform." + exit 1 +fi + +WEBUI_DIR="$ROOT_DIR/webui" + +PYTHON_DIR="$ROOT_DIR/lib/python" +PYTHON_WEBUI_DIR="$PYTHON_DIR/webui" + +pushd "$WEBUI_DIR" + npm install + if [[ $platform == "linux" ]]; then + nodejs ./node_modules/.bin/webpack -g + elif [[ $platform == "macosx" ]]; then + node ./node_modules/.bin/webpack -g + fi + pushd node_modules + rm -rf react* babel* classnames dom-helpers jsesc webpack .bin + popd +popd + +cp -r $WEBUI_DIR/* $PYTHON_WEBUI_DIR/ diff --git a/doc/install-on-macosx.md b/doc/install-on-macosx.md index 721c9f4e3..bc208c9f0 100644 --- a/doc/install-on-macosx.md +++ b/doc/install-on-macosx.md @@ -9,7 +9,7 @@ To install Ray, first install the following dependencies. We recommend using ``` brew update -brew install cmake automake autoconf libtool boost +brew install cmake automake autoconf libtool boost node sudo easy_install pip # If you're using Anaconda, then this is unnecessary. pip install numpy funcsigs colorama psutil redis --ignore-installed six diff --git a/doc/install-on-ubuntu.md b/doc/install-on-ubuntu.md index 625fed7da..fc4638dcc 100644 --- a/doc/install-on-ubuntu.md +++ b/doc/install-on-ubuntu.md @@ -10,7 +10,7 @@ To install Ray, first install the following dependencies. We recommend using ``` sudo apt-get update -sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-pip libboost-all-dev unzip # If you're using Anaconda, then python-dev and python-pip are unnecessary. +sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-pip libboost-all-dev unzip nodejs npm # If you're using Anaconda, then python-dev and python-pip are unnecessary. pip install numpy funcsigs colorama psutil redis pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4 # We use the latest version of cloudpickle because it can serialize named tuples. diff --git a/install-dependencies.sh b/install-dependencies.sh index a9045801a..d6104f50b 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -18,7 +18,7 @@ fi if [[ $platform == "linux" ]]; then # These commands must be kept in sync with the installation instructions. sudo apt-get update - sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip + sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip nodejs npm sudo pip install funcsigs colorama psutil redis elif [[ $platform == "macosx" ]]; then # check that brew is installed @@ -31,7 +31,7 @@ elif [[ $platform == "macosx" ]]; then brew update fi # These commands must be kept in sync with the installation instructions. - brew install cmake automake autoconf libtool boost + brew install cmake automake autoconf libtool boost node sudo easy_install pip sudo pip install numpy funcsigs colorama psutil redis --ignore-installed six fi diff --git a/lib/python/MANIFEST.in b/lib/python/MANIFEST.in new file mode 100644 index 000000000..8708b58cb --- /dev/null +++ b/lib/python/MANIFEST.in @@ -0,0 +1 @@ +recursive-include webui * diff --git a/lib/python/ray/services.py b/lib/python/ray/services.py index 3f1da3165..8d92e502e 100644 --- a/lib/python/ray/services.py +++ b/lib/python/ray/services.py @@ -161,6 +161,22 @@ def start_worker(address_info, worker_path, cleanup=True): if cleanup: all_processes.append(p) +def start_webui(redis_port, cleanup=True): + """This method starts the web interface. + + Args: + redis_port (int): The redis server's port + cleanup (bool): True if using Ray in local mode. If cleanup is true, then + this process will be killed by services.cleanup() when the Python process + that imported services exits. This is True by default. + """ + executable = "nodejs" if sys.platform == "linux" or sys.platform == "linux2" else "node" + command = [executable, os.path.join(os.path.abspath(os.path.dirname(__file__)), "../webui/index.js"), str(redis_port)] + with open("/tmp/webui_out.txt", "wb") as out: + p = subprocess.Popen(command, stdout=out) + if cleanup: + all_processes.append(p) + def start_ray_local(node_ip_address="127.0.0.1", num_workers=0, worker_path=None): """Start Ray in local mode. @@ -196,4 +212,5 @@ def start_ray_local(node_ip_address="127.0.0.1", num_workers=0, worker_path=None start_worker(address_info, worker_path, cleanup=True) time.sleep(0.3) # Return the addresses of the relevant processes. + start_webui(redis_port) return address_info diff --git a/lib/python/setup.py b/lib/python/setup.py index d17ace9a2..e1dc92b76 100644 --- a/lib/python/setup.py +++ b/lib/python/setup.py @@ -1,10 +1,15 @@ from __future__ import print_function +import os import subprocess from setuptools import setup, find_packages import setuptools.command.install as _install +subprocess.check_call(["../../build-webui.sh"]) +datafiles = [(root, [os.path.join(root, f) for f in files]) + for root, dirs, files in os.walk("./webui")] + class install(_install.install): def run(self): subprocess.check_call(["../../build.sh"]) @@ -22,6 +27,7 @@ setup(name="ray", "lib/python/libplasma.so"], "photon": ["build/photon_scheduler", "libphoton.so"]}, + data_files=datafiles, cmdclass={"install": install}, install_requires=["numpy", "funcsigs", diff --git a/lib/python/webui/.gitkeep b/lib/python/webui/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/webui/.babelrc b/webui/.babelrc new file mode 100644 index 000000000..86c445f54 --- /dev/null +++ b/webui/.babelrc @@ -0,0 +1,3 @@ +{ + "presets": ["es2015", "react"] +} diff --git a/webui/client/app/index.jsx b/webui/client/app/index.jsx new file mode 100644 index 000000000..0c4952978 --- /dev/null +++ b/webui/client/app/index.jsx @@ -0,0 +1,305 @@ +import React from 'react'; +import {render} from 'react-dom'; +import {AutoSizer, InfiniteLoader, List} from 'react-virtualized'; +import classNames from 'classnames/bind'; +import io from 'socket.io-client'; +import scrollbarSize from 'dom-helpers/util/scrollbarSize'; + +class RayUI extends React.Component { + + constructor(props) { + super(props); + this.state = { + table_one: "object_table", + table_two: "task_table", + table_three: "failure_table", + table_four: "Remote_table", + table_one_channel:"object", + table_two_channel:"task", + table_three_channel:"failure", + table_four_channel:"remote", + websocket_connection: io() + }; + } + + render() { + return ( +
+ + + + +
+ ); + } +} +class TableView extends React.Component { + + constructor(props) { + super(props); + this.state= { + press: false + }; + this._toggle=this._toggle.bind(this) + } + + _toggle(e){ + this.setState({press: !this.state.press}); + } + + render() { + return ( +
+ + +
+ ); + } +} +class TableScroll extends React.Component{ + constructor(props) { + super(props); + this.state = { + messages: [], + filteredmsg: [], + loadedRowsMap: {}, + content: "This is the view pane.", + atEnd: true, + currentpos: 0, + filter: "" + }; + this._onfilter = this._onfilter.bind(this); + this.filterdata = this.filterdata.bind(this); + this._isRowLoaded = this._isRowLoaded.bind(this); + this._rowRenderer = this._rowRenderer.bind(this); + this._loadMoreRows = this._loadMoreRows.bind(this); + this.objectselect = this.objectselect.bind(this); + this._objectrenderer = this._objectrenderer.bind(this); + this.taskselect = this.taskselect.bind(this); + this._taskrenderer = this._taskrenderer.bind(this); + this.computationselect = this.computationselect.bind(this); + this._computationrenderer = this._computationrenderer.bind(this); + this.failureselect = this.failureselect.bind(this); + this._failurerenderer = this._failurerenderer.bind(this); + this.remoteselect = this.remoteselect.bind(this); + this._remoterenderer = this._remoterenderer.bind(this); + this.scrollcontroller = this.scrollcontroller.bind(this); + switch (this.props.channel) { + case "object": this.renderfunction = this._objectrenderer; + this.header = (
+
{"Object ID"}
+
{"Plasma Store ID"}
+
); + break; + case "failure": this.renderfunction = this._failurerenderer; + this.header = (
+
{"Failed Function"}
+
); + break; + case "computation": this.renderfunction = this._computationrenderer; + this.header = (
+
{"Task iid"}
+
{"Function_id"}
+
); + break; + case "task": this.renderfunction = this._taskrenderer; + this.header = (
+
{"Node id"}
+
{"Function_id"}
+
); + break; + case "remote": this.renderfunction = this._remoterenderer; + this.header = (
+
{"Function id"}
+
{"Module"}
+
{"Function Name"}
+
); + break; + default: break; + } + } + componentDidMount() { + var self = this; + console.log("port" + location.port); + this.props.socket.emit('getall', {table:this.props.channel}); + var arr = []; + this.props.socket.on(this.props.channel, function(message) { + console.log("got message " + message); + var filteredarray = self.state.filteredmsg; + message.forEach(function(msg,i){ + // console.log("Content is " + JSON.stringify(msg)); + arr.push(msg); + if (self.filterdata(msg, self.state.filter)) {filteredarray.push(msg);} + }); + self.setState({messages: arr, filteredmsg: filteredarray}); + }); + } + filterdata(data, filter) { + console.log(data); + var self = this; + return filter != "" ? Object.values(data).filter(function(data){return data === Object(data) ? self.filterdata(data, filter) : String(data).includes(filter)}).length != 0 : true; + } + _onfilter(e) { + var self = this; + this.setState({filteredmsg: e.target.value != "" ? self.state.messages.filter(function(data){return self.filterdata(data, e.target.value)}) : self.state.messages, filter:e.target.value}); + } + _isRowLoaded({ index }) { + return !!this.state.loadedRowsMap[index]; + } + objectselect(data, e) { + console.log(data); + this.setState({content:JSON.stringify(data)}) + } + _objectrenderer(record, key, style) { + const className = classNames("evenRow", "cell", "centeredCell"); + return ( + + ); + + } + failureselect(data, e) { + console.log(data); + this.setState({content:data.error}) + } + _failurerenderer(record, key, style) { + const className = classNames("evenRow", "cell", "centeredCell"); + return ( + + ); + + } + computationselect(data, e) { + console.log(data); + this.setState({content: JSON.stringify}); + } + + _computationrenderer(record, key, style) { + const className = classNames("evenRow", "cell", "centeredCell"); + return ( + + ); + } + taskselect(data, e) { + console.log(data); + this.setState({content: JSON.stringify(data)}); + } + + _taskrenderer(record, key, style) { + const className = classNames("evenRow", "cell", "centeredCell"); + return ( + + ); + + } + remoteselect(data, e) { + console.log(data); + this.setState({content: JSON.stringify(data)}); + } + + _remoterenderer(record, key, style) { + const className = classNames("evenRow", "cell", "centeredCell"); + return ( + + ); + } + + + _rowRenderer({ index, key, style, isScrolling}){ + const record = this.state.filteredmsg[index]; + return this.renderfunction(record, key, style); + } + + _loadMoreRows({ startIndex, stopIndex }) { + for (var i = startIndex; i <= stopIndex; i++) { + this.state.loadedRowsMap[i] = 1; + } + let promiseResolver; + return new Promise(resolve => { + promiseResolver = resolve; + }) + } + + scrollcontroller({clientHeight, scrollHeight, scrollTop}){ + this.setState({atEnd:scrollTop >= scrollHeight- clientHeight, currentpos: Math.floor(scrollTop/20)-1}); + } + + render() { + if (!this.props.press) { + var style = {display:'none'} + } + var self = this; + return ( + + {({ width }) => ( +
+ +
{this.header}
+ + {({ onRowsRendered, registerChild }) => ( + + )} + + +
+ )} + + );} +} +render(, document.getElementById('mount-point')); diff --git a/webui/client/index.html b/webui/client/index.html new file mode 100644 index 000000000..3b77be84a --- /dev/null +++ b/webui/client/index.html @@ -0,0 +1,10 @@ + + Ray UI + + + + +
+ + + diff --git a/webui/client/static/rayui.css b/webui/client/static/rayui.css new file mode 100644 index 000000000..65ea4aa59 --- /dev/null +++ b/webui/client/static/rayui.css @@ -0,0 +1,84 @@ +.GridContainer { + margin-top: 15px; + border: 1px solid #e0e0e0; +} + +.GridRow { + margin-top: 15px; + display: flex; + flex-direction: row; +} +.GridColumn { + display: flex; + flex-direction: column; + flex: 1 1 auto; +} +.LeftSideGridContainer { + flex: 0 0 50px; +} + +.BodyGrid { + width: 100%; + border: 1px solid #e0e0e0; +} + +.evenRow, +.oddRow { + border-bottom: 1px solid #e0e0e0; +} +.oddRow { + background-color: #fafafa; +} + +.cell, +.headerCell { + width: 100%; + height: 100%; + display: flex; + flex-direction: column; + justify-content: center; + padding: 0 .5em; +} +.cell { + border-right: 1px solid #e0e0e0; + border-bottom: 1px solid #e0e0e0; + display: flex; + flex-direction: row; + background-color: white; +} +.rowbutton:hover { + background-color: blue; +} +.rowbutton:hover *{ + background-color: gold; +} +.headerCell { + font-weight: bold; + border-right: 1px solid #e0e0e0; +} +.centeredCell { + align-items: center; + text-align: center; +} +.table { + display: flex; + flex-direction: row; +} +.letterCell { + font-size: 1.5em; + color: #fff; + text-align: center; +} + +.noCells { + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + display: flex; + align-items: center; + justify-content: center; + font-size: 1em; + color: #bdbdbd; +} diff --git a/webui/index.js b/webui/index.js new file mode 100644 index 000000000..84ba90721 --- /dev/null +++ b/webui/index.js @@ -0,0 +1,112 @@ +var express = require('express'); +var app = express(); +var http = require('http').Server(app); +var io = require('socket.io')(http); +var redis = require("redis"); +var task = require('./task.js'); + +if (process.argv.length > 2) { + var port = process.argv[2]; + var sub = redis.createClient(port, {return_buffers: true}); + var db = redis.createClient(port, {return_buffers: true}); +} else { + var sub = redis.createClient({return_buffers: true}); + var db = redis.createClient({return_buffers: true}); +} +const assert = require('assert'); + +app.use(express.static(__dirname + '/client')); +app.get('/', function(req, res) { + res.sendFile(__dirname + '/client/index.html'); +}); + +sub.config("SET", "notify-keyspace-events", "AKE"); +sub.psubscribe("task_log:*"); +sub.psubscribe("__keyspace@0__:obj:*"); +sub.psubscribe("__keyspace@0__:Failures*"); +sub.psubscribe("__keyspace@0__:RemoteFunction*"); +io.on('connection', function(socket) { + console.log('a user connected'); + socket.on('disconnect', function() { console.log('user disconnected'); }); + sub.on('psubscribe', function(channel, count) { console.log("Subscribed"); }); +}); + +backlogobject = []; +backlogtask = []; +backlogfailures = []; +backlogremotefunction = []; +var failureindex; +db.llen("Failures", function(err, result) { failureindex = result; }); +sub.on('pmessage', function(pattern, channel, message) { + if (channel.toString().split(":")[0] === "__keyspace@0__") { + console.log(channel.toString()); + switch (channel.toString().split(":")[1]) { + case "Failures": + db.lindex("Failures", failureindex++, function(err, result) { + backlogfailures.push({ + "functionname": result.toString().split(" ")[2].slice(5, -5), + "error": result.toString() + }); + }); + break; + case "obj": + db.smembers(channel.slice(15), function(err, result) { + console.log(result); + backlogobject.push({ + "ObjectId": channel.slice(19).toString('hex'), + "PlasmaStoreId": result[0].toString() + }); + }); + break; + case "RemoteFunction": + db.hgetall(channel.slice(15), function(err, result) { + backlogremotefunction.push({ + "function_id": result.function_id.toString('hex'), + "module": result.module.toString(), + "name": result.name.toString() + }); + }); + break; + default: + console.log(channel.toString()); + break; + } + } else { + backlogtask.push(task.parse_task_instance(message)); + } +}); + + + +setInterval(function() { + if (backlogfailures.length > 0) { + console.log("Sending ", backlogfailures.length, " objects on failure"); + console.log(backlogfailures); + io.sockets.emit('failure', backlogfailures); + } + backlogfailures = []; +}, 30); +setInterval(function() { + if (backlogobject.length > 0) { + console.log("Sending ", backlogobject.length, " objects on object"); + console.log(backlogobject); + io.sockets.emit('object', backlogobject); + } + backlogobject = []; +}, 30); +setInterval(function() { + if (backlogtask.length > 0) { + console.log("Sending ", backlogtask.length, " objects on task"); + io.sockets.emit('task', backlogtask); + } + backlogtask = []; +}, 30); +setInterval(function() { + if (backlogremotefunction.length > 0) { + console.log("Sending ", backlogremotefunction.length, " objects on remote"); + console.log(backlogremotefunction); + io.sockets.emit('remote', backlogremotefunction); + } + backlogremotefunction = []; +}, 30); +http.listen(3000, function() { console.log('listening on *:3000'); }); diff --git a/webui/package.json b/webui/package.json new file mode 100644 index 000000000..15e00e514 --- /dev/null +++ b/webui/package.json @@ -0,0 +1,32 @@ +{ + "name": "RayWebUI", + "version": "0.0.1", + "description": "A Web UI for Ray", + "repository": { + "type": "git", + "url": "git://github.com/ray-project/ray.git" + }, + "private": true, + "dependencies": { + "babel-core": "~6.17.0", + "babel-loader": "~6.2.5", + "babel-preset-es2015": "~6.16.0", + "babel-preset-react": "~6.16.0", + "classnames": "~2.2.5", + "express": "^4.10.2", + "jbinary": "^2.1.3", + "react": "~15.3.2", + "react-addons-shallow-compare": "~15.3.2", + "react-dom": "~15.3.2", + "react-virtualized": "~8.0.12", + "redis": "^2.6.2", + "socket.io": "^1.5.0", + "socket.io-client": "~1.5.0", + "webpack": "~1.13.2", + "dom-helpers": "~3.0.0", + "jsesc": "~2.2.0" + }, + "devDependencies": { + "webpack": "~1.13.3" + } +} diff --git a/webui/task.js b/webui/task.js new file mode 100644 index 000000000..bcfa95dd9 --- /dev/null +++ b/webui/task.js @@ -0,0 +1,68 @@ +var jb = require('jbinary'); +var stream = require('stream'); + +var task_argument = { + 'jBinary.littleEndian': true, + is_ref: ['enum', 'int8', [true, false]], + padding: ['array', 'int8', 7], + reference: [ + 'if', 'is_ref', + {object_id: ['array', 'uint8', 20], padding: ['array', 'int8', 4]} + ], + value: [ + 'if_not', 'is_ref', + {offset: 'int64', length: 'int64', padding: ['array', 'int8', 8]} + ] +}; + +var task_spec_header = { + 'jBinary.littleEndian': true, + function_id: ['array', 'uint8', 20], + padding: ['array', 'uint8', 4], + num_args: ['int64'], + arg_index: ['int64'], + num_returns: ['int64'], + args_value_size: ['int64'], + args_value_offset: ['int64'], + arguments: ['array', task_argument, 'num_args'] +}; + +var task_instance = { + 'jBinary.littleEndian': true, + task_iid: ['array', 'uint8', 20], + state: 'int32', + node_id: ['array', 'uint8', 20], + padding: ['array', 'uint8', 4], + task_spec_header: ['object', task_spec_header], +}; + + + +// Convert string or byte buffer of an object ID to hex string. +function id_to_hex(id) { + return new Buffer(id).toString('hex'); +} + +module.exports = { + parse_task_instance: function(buffer) { + var binary = new jb(buffer, task_instance); + binary.read('padding'); + var task_spec = binary.read('task_spec_header'); + var arguments = []; + for (var i = 0; i < task_spec['arguments'].length; i++) { + var arg = task_spec['arguments'][i]; + if (arg['is_ref']) { + console.log(arg['reference']['object_id']); + arguments.push(id_to_hex(arg['reference']['object_id'])); + } else { + arguments.push("value"); + } + } + var state = binary.read('state'); + var node_id = binary.read('node_id'); + return { + state: state, node_id: id_to_hex(node_id), + function_id: id_to_hex(task_spec['function_id']), arguments: arguments + } + } +} diff --git a/webui/webpack.config.js b/webui/webpack.config.js new file mode 100644 index 000000000..4a0acbc2c --- /dev/null +++ b/webui/webpack.config.js @@ -0,0 +1,19 @@ +var webpack = require('webpack'); +var path = require('path'); + +var BUILD_DIR = path.resolve(__dirname, 'client/public'); +var APP_DIR = path.resolve(__dirname, 'client/app'); +var config = { + entry: APP_DIR + '/index.jsx', + output: {path: BUILD_DIR, filename: 'bundle.js'}, + module: { + loaders: [{ + test: /\.jsx?/, + include: APP_DIR, + loader: 'babel', + query: {presets: ['react']} + }] + } +}; + +module.exports = config;