diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt index 6de01b0c9..cb2c358fa 100644 --- a/doc/requirements-doc.txt +++ b/doc/requirements-doc.txt @@ -25,6 +25,7 @@ sphinx-jsonschema sphinx-tabs sphinx-version-warning sphinx-book-theme +starlette tabulate uvicorn werkzeug diff --git a/doc/source/serve/index.rst b/doc/source/serve/index.rst index 6d4503bb7..b9c0d1497 100644 --- a/doc/source/serve/index.rst +++ b/doc/source/serve/index.rst @@ -37,7 +37,7 @@ Since Serve is built on Ray, it also allows you to scale to many machines, in yo Installation ============ -Ray Serve supports Python versions 3.6 and higher. To install Ray Serve: +Ray Serve supports Python versions 3.6 through 3.8. To install Ray Serve: .. code-block:: bash diff --git a/doc/source/serve/key-concepts.rst b/doc/source/serve/key-concepts.rst index d15a142c8..deada7233 100644 --- a/doc/source/serve/key-concepts.rst +++ b/doc/source/serve/key-concepts.rst @@ -19,7 +19,11 @@ Backends Backends define the implementation of your business logic or models that will handle requests when queries come in to :ref:`serve-endpoint`. In order to support seamless scalability backends can have many replicas, which are individual processes running in the Ray cluster to handle requests. To define a backend, first you must define the "handler" or the business logic you'd like to respond with. -The handler should take as input a `Flask Request object `_ and return any JSON-serializable object as output. +The handler should take as input a `Flask Request object `_. +The handler should return any JSON-serializable object as output. For a more customizable response type, the handler may return a +`Starlette Response object `_. +In the future, Ray Serve will support `Starlette Request objects `_ as input as well. + A backend is defined using :mod:`client.create_backend `, and the implementation can be defined as either a function or a class. Use a function when your response is stateless and a class when you might need to maintain some state (like a model). When using a class, you can specify arguments to be passed to the constructor in :mod:`client.create_backend `, shown below. diff --git a/python/ray/serve/api.py b/python/ray/serve/api.py index b6ac3c0ee..bd7b7d41a 100644 --- a/python/ray/serve/api.py +++ b/python/ray/serve/api.py @@ -255,7 +255,8 @@ class Client: Args: backend_tag (str): a unique tag assign to identify this backend. func_or_class (callable, class): a function or a class implementing - __call__. + __call__, returning a JSON-serializable object or a + Starlette Response object. actor_init_args (optional): the arguments to pass to the class. initialization method. ray_actor_options (optional): options to be passed into the diff --git a/python/ray/serve/http_proxy.py b/python/ray/serve/http_proxy.py index 01c8d8e02..00c59ceae 100644 --- a/python/ray/serve/http_proxy.py +++ b/python/ray/serve/http_proxy.py @@ -3,6 +3,7 @@ import socket from typing import List import uvicorn +import starlette.responses import ray from ray.exceptions import RayTaskError @@ -126,6 +127,18 @@ class HTTPProxy: if isinstance(result, RayTaskError): error_message = "Task Error. Traceback: {}.".format(result) await error_sender(error_message, 500) + elif isinstance(result, starlette.responses.Response): + if isinstance(result, starlette.responses.StreamingResponse): + raise TypeError("Starlette StreamingResponse returned by " + f"backend for endpoint {endpoint_name}. " + "StreamingResponse is unserializable and not " + "supported by Ray Serve. Consider using " + "another Starlette response type such as " + "Response, HTMLResponse, PlainTextResponse, " + "or JSONResponse. If support for " + "StreamingResponse is desired, please let " + "the Ray team know by making a Github issue!") + await result(scope, receive, send) else: await Response(result).send(scope, receive, send) diff --git a/python/ray/serve/http_util.py b/python/ray/serve/http_util.py index da41c10eb..1a057a88e 100644 --- a/python/ray/serve/http_util.py +++ b/python/ray/serve/http_util.py @@ -117,7 +117,7 @@ class Response: elif content_type == "json": self.raw_headers.append([b"content-type", b"application/json"]) else: - raise ValueError("Invalid content type {}".foramt(content_type)) + raise ValueError("Invalid content type {}".format(content_type)) async def send(self, scope, receive, send): await send({ diff --git a/python/ray/serve/tests/test_api.py b/python/ray/serve/tests/test_api.py index 4ad1cb554..778aef4b9 100644 --- a/python/ray/serve/tests/test_api.py +++ b/python/ray/serve/tests/test_api.py @@ -4,6 +4,7 @@ import time import os import pytest import requests +import starlette.responses import ray from ray import serve @@ -32,6 +33,63 @@ def test_e2e(serve_instance): assert resp == "POST" +def test_starlette_response(serve_instance): + client = serve_instance + + def basic_response(_): + return starlette.responses.Response( + "Hello, world!", media_type="text/plain") + + client.create_backend("basic_response", basic_response) + client.create_endpoint( + "basic_response", backend="basic_response", route="/basic_response") + assert requests.get( + "http://127.0.0.1:8000/basic_response").text == "Hello, world!" + + def html_response(_): + return starlette.responses.HTMLResponse( + "

Hello, world!

") + + client.create_backend("html_response", html_response) + client.create_endpoint( + "html_response", backend="html_response", route="/html_response") + assert requests.get( + "http://127.0.0.1:8000/html_response" + ).text == "

Hello, world!

" + + def plain_text_response(_): + return starlette.responses.PlainTextResponse("Hello, world!") + + client.create_backend("plain_text_response", plain_text_response) + client.create_endpoint( + "plain_text_response", + backend="plain_text_response", + route="/plain_text_response") + assert requests.get( + "http://127.0.0.1:8000/plain_text_response").text == "Hello, world!" + + def json_response(_): + return starlette.responses.JSONResponse({"hello": "world"}) + + client.create_backend("json_response", json_response) + client.create_endpoint( + "json_response", backend="json_response", route="/json_response") + assert requests.get("http://127.0.0.1:8000/json_response").json()[ + "hello"] == "world" + + def redirect_response(_): + return starlette.responses.RedirectResponse( + url="http://127.0.0.1:8000/basic_response") + + client.create_backend("redirect_response", redirect_response) + client.create_endpoint( + "redirect_response", + backend="redirect_response", + route="/redirect_response") + assert requests.get( + "http://127.0.0.1:8000/redirect_response").text == "Hello, world!" + + def test_backend_user_config(serve_instance): client = serve_instance diff --git a/python/requirements.txt b/python/requirements.txt index 76d0799fe..28c387fde 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -38,6 +38,7 @@ tensorboardX uvicorn pydantic<1.7 dataclasses; python_version < '3.7' +starlette # Requirements for running tests blist; platform_system != "Windows" diff --git a/python/setup.py b/python/setup.py index 09327ffa7..62fc4b694 100644 --- a/python/setup.py +++ b/python/setup.py @@ -97,7 +97,7 @@ ray_files += [ extras = { "serve": [ "uvicorn", "flask", "requests", "pydantic<1.7", - "dataclasses; python_version < '3.7'" + "dataclasses; python_version < '3.7'", "starlette" ], "tune": [ "dataclasses; python_version < '3.7'", "pandas", "tabulate",