[Serve] Support basic Starlette response types (#12811)

doc/requirements-doc.txt

@@ -25,6 +25,7 @@ sphinx-jsonschema
 sphinx-tabs
 sphinx-version-warning
 sphinx-book-theme
+starlette
 tabulate
 uvicorn
 werkzeug

doc/source/serve/index.rst

@@ -37,7 +37,7 @@ Since Serve is built on Ray, it also allows you to scale to many machines, in yo
 Installation
 ============
 
-Ray Serve supports Python versions 3.6 and higher. To install Ray Serve:
+Ray Serve supports Python versions 3.6 through 3.8. To install Ray Serve:
 
 .. code-block:: bash
 

doc/source/serve/key-concepts.rst

@@ -19,7 +19,11 @@ Backends
 Backends define the implementation of your business logic or models that will handle requests when queries come in to :ref:`serve-endpoint`.
 In order to support seamless scalability backends can have many replicas, which are individual processes running in the Ray cluster to handle requests.
 To define a backend, first you must define the "handler" or the business logic you'd like to respond with.
-The handler should take as input a `Flask Request object <https://flask.palletsprojects.com/en/1.1.x/api/?highlight=request#flask.Request>`_ and return any JSON-serializable object as output.
+The handler should take as input a `Flask Request object <https://flask.palletsprojects.com/en/1.1.x/api/?highlight=request#flask.Request>`_. 
+The handler should return any JSON-serializable object as output.  For a more customizable response type, the handler may return a 
+`Starlette Response object <https://www.starlette.io/responses/>`_.  
+In the future, Ray Serve will support `Starlette Request objects <https://www.starlette.io/requests/>`_ as input as well.
+
 A backend is defined using :mod:`client.create_backend <ray.serve.api.Client.create_backend>`, and the implementation can be defined as either a function or a class.
 Use a function when your response is stateless and a class when you might need to maintain some state (like a model).
 When using a class, you can specify arguments to be passed to the constructor in :mod:`client.create_backend <ray.serve.api.Client.create_backend>`, shown below.

python/ray/serve/api.py

@@ -255,7 +255,8 @@ class Client:
         Args:
             backend_tag (str): a unique tag assign to identify this backend.
             func_or_class (callable, class): a function or a class implementing
-                __call__.
+                __call__, returning a JSON-serializable object or a
+                Starlette Response object.
             actor_init_args (optional): the arguments to pass to the class.
                 initialization method.
             ray_actor_options (optional): options to be passed into the

python/ray/serve/http_proxy.py

@@ -3,6 +3,7 @@ import socket
 from typing import List
 
 import uvicorn
+import starlette.responses
 
 import ray
 from ray.exceptions import RayTaskError
@@ -126,6 +127,18 @@ class HTTPProxy:
         if isinstance(result, RayTaskError):
             error_message = "Task Error. Traceback: {}.".format(result)
             await error_sender(error_message, 500)
+        elif isinstance(result, starlette.responses.Response):
+            if isinstance(result, starlette.responses.StreamingResponse):
+                raise TypeError("Starlette StreamingResponse returned by "
+                                f"backend for endpoint {endpoint_name}. "
+                                "StreamingResponse is unserializable and not "
+                                "supported by Ray Serve.  Consider using "
+                                "another Starlette response type such as "
+                                "Response, HTMLResponse, PlainTextResponse, "
+                                "or JSONResponse.  If support for "
+                                "StreamingResponse is desired, please let "
+                                "the Ray team know by making a Github issue!")
+            await result(scope, receive, send)
         else:
             await Response(result).send(scope, receive, send)
 

python/ray/serve/http_util.py

@@ -117,7 +117,7 @@ class Response:
         elif content_type == "json":
             self.raw_headers.append([b"content-type", b"application/json"])
         else:
-            raise ValueError("Invalid content type {}".foramt(content_type))
+            raise ValueError("Invalid content type {}".format(content_type))
 
     async def send(self, scope, receive, send):
         await send({

python/ray/serve/tests/test_api.py

@@ -4,6 +4,7 @@ import time
 import os
 import pytest
 import requests
+import starlette.responses
 
 import ray
 from ray import serve
@@ -32,6 +33,63 @@ def test_e2e(serve_instance):
     assert resp == "POST"
 
 
+def test_starlette_response(serve_instance):
+    client = serve_instance
+
+    def basic_response(_):
+        return starlette.responses.Response(
+            "Hello, world!", media_type="text/plain")
+
+    client.create_backend("basic_response", basic_response)
+    client.create_endpoint(
+        "basic_response", backend="basic_response", route="/basic_response")
+    assert requests.get(
+        "http://127.0.0.1:8000/basic_response").text == "Hello, world!"
+
+    def html_response(_):
+        return starlette.responses.HTMLResponse(
+            "<html><body><h1>Hello, world!</h1></body></html>")
+
+    client.create_backend("html_response", html_response)
+    client.create_endpoint(
+        "html_response", backend="html_response", route="/html_response")
+    assert requests.get(
+        "http://127.0.0.1:8000/html_response"
+    ).text == "<html><body><h1>Hello, world!</h1></body></html>"
+
+    def plain_text_response(_):
+        return starlette.responses.PlainTextResponse("Hello, world!")
+
+    client.create_backend("plain_text_response", plain_text_response)
+    client.create_endpoint(
+        "plain_text_response",
+        backend="plain_text_response",
+        route="/plain_text_response")
+    assert requests.get(
+        "http://127.0.0.1:8000/plain_text_response").text == "Hello, world!"
+
+    def json_response(_):
+        return starlette.responses.JSONResponse({"hello": "world"})
+
+    client.create_backend("json_response", json_response)
+    client.create_endpoint(
+        "json_response", backend="json_response", route="/json_response")
+    assert requests.get("http://127.0.0.1:8000/json_response").json()[
+        "hello"] == "world"
+
+    def redirect_response(_):
+        return starlette.responses.RedirectResponse(
+            url="http://127.0.0.1:8000/basic_response")
+
+    client.create_backend("redirect_response", redirect_response)
+    client.create_endpoint(
+        "redirect_response",
+        backend="redirect_response",
+        route="/redirect_response")
+    assert requests.get(
+        "http://127.0.0.1:8000/redirect_response").text == "Hello, world!"
+
+
 def test_backend_user_config(serve_instance):
     client = serve_instance
 

python/requirements.txt

@@ -38,6 +38,7 @@ tensorboardX
 uvicorn
 pydantic<1.7
 dataclasses; python_version < '3.7'
+starlette
 
 # Requirements for running tests
 blist; platform_system != "Windows"

python/setup.py

@@ -97,7 +97,7 @@ ray_files += [
 extras = {
     "serve": [
         "uvicorn", "flask", "requests", "pydantic<1.7",
-        "dataclasses; python_version < '3.7'"
+        "dataclasses; python_version < '3.7'", "starlette"
     ],
     "tune": [
         "dataclasses; python_version < '3.7'", "pandas", "tabulate",