From 101255f782674f70aa2d5e8e714155f1d37aef1b Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Tue, 28 Apr 2020 22:24:55 -0700 Subject: [PATCH] [Serve] RayServe TF, PyTorch, Sklearn Examples (#8156) --- ci/travis/install-dependencies.sh | 11 ++- doc/source/index.rst | 3 + doc/source/rayserve/overview.rst | 18 +++- .../rayserve/tutorials/pytorch-tutorial.rst | 47 ++++++++++ .../rayserve/tutorials/sklearn-tutorial.rst | 51 +++++++++++ .../tutorials/tensorflow-tutorial.rst | 54 ++++++++++++ python/ray/serve/BUILD | 24 +++++ .../serve/examples/doc/tutorial_pytorch.py | 63 ++++++++++++++ .../serve/examples/doc/tutorial_sklearn.py | 87 +++++++++++++++++++ .../serve/examples/doc/tutorial_tensorflow.py | 86 ++++++++++++++++++ 10 files changed, 441 insertions(+), 3 deletions(-) create mode 100644 doc/source/rayserve/tutorials/pytorch-tutorial.rst create mode 100644 doc/source/rayserve/tutorials/sklearn-tutorial.rst create mode 100644 doc/source/rayserve/tutorials/tensorflow-tutorial.rst create mode 100644 python/ray/serve/examples/doc/tutorial_pytorch.py create mode 100644 python/ray/serve/examples/doc/tutorial_sklearn.py create mode 100644 python/ray/serve/examples/doc/tutorial_tensorflow.py diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 3cd8b2eb5..4f094dd19 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -132,10 +132,19 @@ install_dependencies() { if [ -n "${PYTHON-}" ]; then install_miniconda + + # PyTorch is installed first since we are using a "-f" directive to find the wheels. + # We want to install the CPU version only. + case "${OSTYPE}" in + linux*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html;; + darwin*) pip install torch torchvision;; + msys*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html;; + esac + pip_packages=(scipy tensorflow=="${TF_VERSION:-2.0.0b1}" cython==0.29.0 gym opencv-python-headless pyyaml \ pandas==0.24.2 requests feather-format lxml openpyxl xlrd py-spy pytest pytest-timeout networkx tabulate aiohttp \ uvicorn dataclasses pygments werkzeug kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio \ - scikit-learn numba) + scikit-learn numba Pillow) if [ "${OSTYPE}" != msys ]; then # These packages aren't Windows-compatible pip_packages+=(blist) # https://github.com/DanielStutzbach/blist/issues/81#issue-391460716 diff --git a/doc/source/index.rst b/doc/source/index.rst index a24fcfef5..68a1bb2e3 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -285,6 +285,9 @@ Getting Involved :caption: RayServe rayserve/overview.rst + rayserve/tutorials/tensorflow-tutorial.rst + rayserve/tutorials/pytorch-tutorial.rst + rayserve/tutorials/sklearn-tutorial.rst .. toctree:: :maxdepth: -1 diff --git a/doc/source/rayserve/overview.rst b/doc/source/rayserve/overview.rst index 350bb7ac1..70a74b975 100644 --- a/doc/source/rayserve/overview.rst +++ b/doc/source/rayserve/overview.rst @@ -8,6 +8,8 @@ RayServe: Scalable and Programmable Serving :height: 250px :width: 400px +.. _rayserve-overview: + Overview -------- @@ -92,6 +94,7 @@ To follow along, you'll need to make the necessary imports. from ray import serve serve.init() # initializes serve and Ray +.. _serve-endpoint: Endpoints ~~~~~~~~~ @@ -105,6 +108,8 @@ model that you'll be serving. To create one, we'll simply specify the name, rout serve.create_endpoint("simple_endpoint", "/simple") +.. _serve-backend: + Backends ~~~~~~~~ @@ -209,6 +214,15 @@ You can also have RayServe batch requests for performance. You'll configure this serve.link("counter1", "counter1") Other Resources ----------------- +--------------- -More coming soon! +.. _serve_frameworks: + +Frameworks +~~~~~~~~~~ +RayServe makes it easy to deploy models from all popular frameworks. +Learn more about how to deploy your model in the following tutorials: + +- :ref:`Tensorflow & Keras ` +- :ref:`PyTorch ` +- :ref:`Scikit-Learn ` diff --git a/doc/source/rayserve/tutorials/pytorch-tutorial.rst b/doc/source/rayserve/tutorials/pytorch-tutorial.rst new file mode 100644 index 000000000..0c5dfe73f --- /dev/null +++ b/doc/source/rayserve/tutorials/pytorch-tutorial.rst @@ -0,0 +1,47 @@ +.. _serve-pytorch-tutorial: + +PyTorch Tutorial +================ + +In this guide, we will load and serve a PyTorch Resnet Model. +In particular, we show: + +- How to load the model from PyTorch's pre-trained modelzoo. +- How to parse the JSON request, transform the payload and evaluated in the model. + +Please see the :ref:`overview ` to learn more general information about RayServe. + +This tutorial requires Pytorch and Torchvision installed in your system. RayServe +is :ref:`framework agnostic ` and work with any version of PyTorch. + +.. code-block:: bash + + pip install torch torchvision + +Let's import RayServe and some other helpers. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_pytorch.py + :start-after: __doc_import_begin__ + :end-before: __doc_import_end__ + + +Services are just defined as normal classes with ``__init__`` and ``__call__`` methods. +The ``__call__`` method will be invoked per request. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_pytorch.py + :start-after: __doc_define_servable_begin__ + :end-before: __doc_define_servable_end__ + +Now that we've defined our services, let's deploy the model to RayServe. We will +define an :ref:`endpoint ` for the route representing the digit classifier task, a +:ref:`backend ` correspond the physical implementation, and connect them together. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_pytorch.py + :start-after: __doc_deploy_begin__ + :end-before: __doc_deploy_end__ + +Let's query it! + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_pytorch.py + :start-after: __doc_query_begin__ + :end-before: __doc_query_end__ \ No newline at end of file diff --git a/doc/source/rayserve/tutorials/sklearn-tutorial.rst b/doc/source/rayserve/tutorials/sklearn-tutorial.rst new file mode 100644 index 000000000..3f14543f1 --- /dev/null +++ b/doc/source/rayserve/tutorials/sklearn-tutorial.rst @@ -0,0 +1,51 @@ +.. _serve-sklearn-tutorial: + +Scikit-Learn Tutorial +===================== + +In this guide, we will train and deploy a simple Scikit-Learn classifier. +In particular, we show: + +- How to load the model from file system in your RayServe definition +- How to parse the JSON request and evaluated in sklearn model + +Please see the :ref:`overview ` to learn more general information about RayServe. + +RayServe supports :ref:`arbitrary frameworks `. You can use any version of sklearn. + +.. code-block:: bash + + pip install scikit-learn + +Let's import RayServe and some other helpers. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_sklearn.py + :start-after: __doc_import_begin__ + :end-before: __doc_import_end__ + +We will train a logistic regression with the iris dataset. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_sklearn.py + :start-after: __doc_train_model_begin__ + :end-before: __doc_train_model_end__ + +Services are just defined as normal classes with ``__init__`` and ``__call__`` methods. +The ``__call__`` method will be invoked per request. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_sklearn.py + :start-after: __doc_define_servable_begin__ + :end-before: __doc_define_servable_end__ + +Now that we've defined our services, let's deploy the model to RayServe. We will +define an :ref:`endpoint ` for the route representing the classifier task, a +:ref:`backend ` correspond the physical implementation, and connect them together. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_sklearn.py + :start-after: __doc_deploy_begin__ + :end-before: __doc_deploy_end__ + +Let's query it! + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_sklearn.py + :start-after: __doc_query_begin__ + :end-before: __doc_query_end__ \ No newline at end of file diff --git a/doc/source/rayserve/tutorials/tensorflow-tutorial.rst b/doc/source/rayserve/tutorials/tensorflow-tutorial.rst new file mode 100644 index 000000000..7039b72b9 --- /dev/null +++ b/doc/source/rayserve/tutorials/tensorflow-tutorial.rst @@ -0,0 +1,54 @@ +.. _serve-tensorflow-tutorial: + +Keras and Tensorflow Tutorial +============================= + +In this guide, we will train and deploy a simple Tensorflow neural net. +In particular, we show: + +- How to load the model from file system in your RayServe definition +- How to parse the JSON request and evaluated in Tensorflow + +Please see the :ref:`overview ` to learn more general information about RayServe. + +RayServe makes it easy to deploy models from :ref:`all popular frameworks `. +However, for this tutorial, we use Tensorflow 2 and Keras. Please make sure you have +Tensorflow 2 installed. + + +.. code-block:: bash + + pip install "tensorflow>=2.0" + +Let's import RayServe and some other helpers. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_tensorflow.py + :start-after: __doc_import_begin__ + :end-before: __doc_import_end__ + +We will train a simple MNIST model using Keras. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_tensorflow.py + :start-after: __doc_train_model_begin__ + :end-before: __doc_train_model_end__ + +Services are just defined as normal classes with ``__init__`` and ``__call__`` methods. +The ``__call__`` method will be invoked per request. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_tensorflow.py + :start-after: __doc_define_servable_begin__ + :end-before: __doc_define_servable_end__ + +Now that we've defined our services, let's deploy the model to RayServe. We will +define an :ref:`endpoint ` for the route representing the digit classifier task, a +:ref:`backend ` correspond the physical implementation, and connect them together. + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_tensorflow.py + :start-after: __doc_deploy_begin__ + :end-before: __doc_deploy_end__ + +Let's query it! + +.. literalinclude:: ../../../../python/ray/serve/examples/doc/tutorial_tensorflow.py + :start-after: __doc_query_begin__ + :end-before: __doc_query_end__ \ No newline at end of file diff --git a/python/ray/serve/BUILD b/python/ray/serve/BUILD index b818e06bb..41f77bd6b 100644 --- a/python/ray/serve/BUILD +++ b/python/ray/serve/BUILD @@ -63,3 +63,27 @@ py_test( tags = ["exclusive"], deps = [":serve_lib"] ) + +py_test( + name = "tutorial_tensorflow", + size = "small", + srcs = glob(["examples/doc/*.py"]), + tags = ["exclusive"], + deps = [":serve_lib"] +) + +py_test( + name = "tutorial_pytorch", + size = "small", + srcs = glob(["examples/doc/*.py"]), + tags = ["exclusive"], + deps = [":serve_lib"] +) + +py_test( + name = "tutorial_sklearn", + size = "small", + srcs = glob(["examples/doc/*.py"]), + tags = ["exclusive"], + deps = [":serve_lib"] +) \ No newline at end of file diff --git a/python/ray/serve/examples/doc/tutorial_pytorch.py b/python/ray/serve/examples/doc/tutorial_pytorch.py new file mode 100644 index 000000000..42d634acd --- /dev/null +++ b/python/ray/serve/examples/doc/tutorial_pytorch.py @@ -0,0 +1,63 @@ +# yapf: disable +# __doc_import_begin__ +from ray import serve + +from io import BytesIO +from PIL import Image +import requests + +import torch +from torchvision import transforms +from torchvision.models import resnet18 +# __doc_import_end__ +# yapf: enable + + +# __doc_define_servable_begin__ +class ImageModel: + def __init__(self): + self.model = resnet18(pretrained=True) + self.preprocessor = transforms.Compose([ + transforms.Resize(224), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Lambda(lambda t: t[:3, ...]), # remove alpha channel + ]) + + def __call__(self, flask_request): + image_payload_bytes = flask_request.data + pil_image = Image.open(BytesIO(image_payload_bytes)) + print("[1/3] Parsed image data: {}".format(pil_image)) + + pil_images = [pil_image] # Our current batch size is one + input_tensor = torch.cat( + [self.preprocessor(i).unsqueeze(0) for i in pil_images]) + print("[2/3] Images transformed, tensor shape {}".format( + input_tensor.shape)) + + with torch.no_grad(): + output_tensor = self.model(input_tensor) + print("[3/3] Inference done!") + return {"class_index": int(torch.argmax(output_tensor[0]))} + + +# __doc_define_servable_end__ + +# __doc_deploy_begin__ +serve.init() +serve.create_endpoint("predictor", "/image_predict", methods=["POST"]) +serve.create_backend(ImageModel, "resnet18:v0") +serve.set_traffic("predictor", {"resnet18:v0": 1}) +# __doc_deploy_end__ + +# __doc_query_begin__ +ray_logo_bytes = requests.get( + "https://github.com/ray-project/ray/raw/" + "master/doc/source/images/ray_header_logo.png").content + +resp = requests.post( + "http://localhost:8000/image_predict", data=ray_logo_bytes) +print(resp.json()) +# Output +# {'class_index': 463} +# __doc_query_end__ diff --git a/python/ray/serve/examples/doc/tutorial_sklearn.py b/python/ray/serve/examples/doc/tutorial_sklearn.py new file mode 100644 index 000000000..8569a1f2a --- /dev/null +++ b/python/ray/serve/examples/doc/tutorial_sklearn.py @@ -0,0 +1,87 @@ +# yapf: disable +# __doc_import_begin__ +from ray import serve + +import pickle +import json +import numpy as np +import requests + +from sklearn.datasets import load_iris +from sklearn.ensemble import GradientBoostingClassifier +from sklearn.metrics import mean_squared_error +# __doc_import_end__ +# yapf: enable + +# __doc_train_model_begin__ +# Load data +data, target, target_names, description, feature_names, _ = load_iris().values( +) + +# Instantiate model +model = GradientBoostingClassifier() + +# Training and validation split +np.random.shuffle(data), np.random.shuffle(target) +train_x, train_y = data[:100], target[:100] +val_x, val_y = data[100:], target[100:] + +# Train and evaluate models +model.fit(train_x, train_y) +print("MSE:", mean_squared_error(model.predict(val_x), val_y)) + +# Save the model and label to file +with open("/tmp/iris_model_logistic_regression.pkl", "wb") as f: + pickle.dump(model, f) +with open("/tmp/iris_labels.json", "w") as f: + json.dump(target_names.tolist(), f) +# __doc_train_model_end__ + + +# __doc_define_servable_begin__ +class BoostingModel: + def __init__(self): + with open("/tmp/iris_model_logistic_regression.pkl", "rb") as f: + self.model = pickle.load(f) + with open("/tmp/iris_labels.json") as f: + self.label_list = json.load(f) + + def __call__(self, flask_request): + payload = flask_request.json + print("Worker: received flask request with data", payload) + + input_vector = [ + payload["sepal length"], + payload["sepal width"], + payload["petal length"], + payload["petal width"], + ] + prediction = self.model.predict([input_vector])[0] + human_name = self.label_list[prediction] + return {"result": human_name} + + +# __doc_define_servable_end__ + +# __doc_deploy_begin__ +serve.init() +serve.create_endpoint("iris_classifier", "/regressor") +serve.create_backend(BoostingModel, "lr:v1") +serve.set_traffic("iris_classifier", {"lr:v1": 1}) +# __doc_deploy_end__ + +# __doc_query_begin__ +sample_request_input = { + "sepal length": 1.2, + "sepal width": 1.0, + "petal length": 1.1, + "petal width": 0.9, +} +response = requests.get( + "http://localhost:8000/regressor", json=sample_request_input) +print(response.text) +# Result: +# { +# "result": "versicolor" +# } +# __doc_query_end__ diff --git a/python/ray/serve/examples/doc/tutorial_tensorflow.py b/python/ray/serve/examples/doc/tutorial_tensorflow.py new file mode 100644 index 000000000..e6922c5bd --- /dev/null +++ b/python/ray/serve/examples/doc/tutorial_tensorflow.py @@ -0,0 +1,86 @@ +# yapf: disable +# __doc_import_begin__ +from ray import serve + +import os +import numpy as np +import requests +# __doc_import_end__ +# yapf: enable + +# __doc_train_model_begin__ +TRAINED_MODEL_PATH = "/tmp/mnist_model.h5" + + +def train_and_save_model(): + import tensorflow as tf + # Load mnist dataset + mnist = tf.keras.datasets.mnist + (x_train, y_train), (x_test, y_test) = mnist.load_data() + x_train, x_test = x_train / 255.0, x_test / 255.0 + + # Train a simple neural net model + model = tf.keras.models.Sequential([ + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128, activation="relu"), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10) + ]) + loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + model.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"]) + model.fit(x_train, y_train, epochs=1) + + model.evaluate(x_test, y_test, verbose=2) + model.summary() + + # Save the model in h5 format in local file system + model.save(TRAINED_MODEL_PATH) + + +if not os.path.exists(TRAINED_MODEL_PATH): + train_and_save_model() +# __doc_train_model_end__ + + +# __doc_define_servable_begin__ +class TFMnistModel: + def __init__(self, model_path): + import tensorflow as tf + self.model_path = model_path + self.model = tf.keras.models.load_model(model_path) + + def __call__(self, flask_request): + # Step 1: transform HTTP request -> tensorflow input + # Here we define the request schema to be a json array. + input_array = np.array(flask_request.json["array"]) + reshaped_array = input_array.reshape((1, 28, 28)) + + # Step 2: tensorflow input -> tensorflow output + prediction = self.model(reshaped_array) + + # Step 3: tensorflow output -> web output + return { + "prediction": prediction.numpy().tolist(), + "file": self.model_path + } + + +# __doc_define_servable_end__ + +# __doc_deploy_begin__ +serve.init() +serve.create_endpoint(endpoint_name="tf_classifier", route="/mnist") +serve.create_backend(TFMnistModel, "tf:v1", "/tmp/mnist_model.h5") +serve.set_traffic("tf_classifier", {"tf:v1": 1}) +# __doc_deploy_end__ + +# __doc_query_begin__ +resp = requests.get( + "http://localhost:8000/mnist", + json={"array": np.random.randn(28 * 28).tolist()}) +print(resp.json()) +# { +# "prediction": [[-1.504277229309082, ..., -6.793371200561523]], +# "file": "/tmp/mnist_model.h5" +# } +# __doc_query_end__