From 2e82e05e4b1dfbe4f1ec3fcb4f665a5e8d2db43d Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 2 Jun 2020 17:14:10 -0500 Subject: [PATCH] [serve] Add list_backends and list_endpoints (#8737) --- doc/source/serve/key-concepts.rst | 24 +++++++++++- python/ray/serve/__init__.py | 24 ++++++++---- python/ray/serve/api.py | 19 +++++++++ python/ray/serve/master.py | 18 +++++++-- python/ray/serve/tests/test_api.py | 62 ++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 12 deletions(-) diff --git a/doc/source/serve/key-concepts.rst b/doc/source/serve/key-concepts.rst index c1f5c1d86..1fd2d853d 100644 --- a/doc/source/serve/key-concepts.rst +++ b/doc/source/serve/key-concepts.rst @@ -23,7 +23,14 @@ model that you'll be serving. To create one, we'll simply specify the name, rout .. code-block:: python - serve.create_endpoint("simple_endpoint", "/simple") + serve.create_endpoint("simple_endpoint", "/simple", methods=["GET"]) + +To view all of the existing endpoints that have created, use `serve.list_endpoints`. + +.. code-block:: python + + >>> serve.list_endpoints() + {'simple_endpoint': {'route': '/simple', 'methods': ['GET'], 'traffic': {}}} You can also delete an endpoint using ``serve.delete_endpoint``. Note that this will not delete any associated backends, which can be reused for other endpoints. @@ -67,6 +74,21 @@ It's important to note that Ray Serve places these backends in individual worker # If we call this backend, it will respond with "hello, world!". serve.create_backend("simple_backend_class", RequestHandler, "hello, world!") +We can also list all available backends and delete them to reclaim resources. + +.. code-block:: python + + >> serve.list_backends() + { + 'simple_backend': {'accepts_batches': False, 'num_replicas': 1, 'max_batch_size': None}, + 'simple_backend_class': {'accepts_batches': False, 'num_replicas': 1, 'max_batch_size': None}, + } + >> serve.delete_backend("simple_backend") + >> serve.list_backends() + { + 'simple_backend_class': {'accepts_batches': False, 'num_replicas': 1, 'max_batch_size': None}, + } + Setting Traffic =============== diff --git a/python/ray/serve/__init__.py b/python/ray/serve/__init__.py index a6300880f..7bc2ff580 100644 --- a/python/ray/serve/__init__.py +++ b/python/ray/serve/__init__.py @@ -1,10 +1,20 @@ -from ray.serve.api import (init, create_backend, delete_backend, - create_endpoint, delete_endpoint, set_traffic, - get_handle, stat, update_backend_config, - get_backend_config, accept_batch) # noqa: E402 +from ray.serve.api import ( + init, create_backend, delete_backend, create_endpoint, delete_endpoint, + set_traffic, get_handle, stat, update_backend_config, get_backend_config, + accept_batch, list_backends, list_endpoints) # noqa: E402 __all__ = [ - "init", "create_backend", "delete_backend", "create_endpoint", - "delete_endpoint", "set_traffic", "get_handle", "stat", - "update_backend_config", "get_backend_config", "accept_batch" + "init", + "create_backend", + "delete_backend", + "create_endpoint", + "delete_endpoint", + "set_traffic", + "get_handle", + "stat", + "update_backend_config", + "get_backend_config", + "accept_batch", + "list_backends", + "list_endpoints", ] diff --git a/python/ray/serve/api.py b/python/ray/serve/api.py index eb7712bb6..026b99f48 100644 --- a/python/ray/serve/api.py +++ b/python/ray/serve/api.py @@ -141,6 +141,16 @@ def delete_endpoint(endpoint): retry_actor_failures(master_actor.delete_endpoint, endpoint) +@_ensure_connected +def list_endpoints(): + """Returns a dictionary of all registered endpoints. + + The dictionary keys are endpoint names and values are dictionaries + of the form: {"methods": List[str], "traffic": Dict[str, float]}. + """ + return retry_actor_failures(master_actor.get_all_endpoints) + + @_ensure_connected def update_backend_config(backend_tag, config_options): """Update a backend configuration for a backend tag. @@ -200,6 +210,15 @@ def create_backend(backend_tag, backend_config, replica_config) +@_ensure_connected +def list_backends(): + """Returns a dictionary of all registered backends. + + Dictionary maps backend tags to backend configs. + """ + return retry_actor_failures(master_actor.get_all_backends) + + @_ensure_connected def delete_backend(backend_tag): """Delete the given backend. diff --git a/python/ray/serve/master.py b/python/ray/serve/master.py index 07605da1e..798e40b94 100644 --- a/python/ray/serve/master.py +++ b/python/ray/serve/master.py @@ -446,12 +446,22 @@ class ServeMaster: return self.workers def get_all_backends(self): - """Used for validation by the API client.""" - return list(self.backends.keys()) + """Returns a dictionary of backend tag to backend config dict.""" + backends = {} + for backend_tag, (_, config, _) in self.backends.items(): + backends[backend_tag] = config.__dict__ + return backends def get_all_endpoints(self): - """Used for validation by the API client.""" - return [endpoint for endpoint, methods in self.routes.values()] + """Returns a dictionary of endpoint to endpoint config.""" + endpoints = {} + for route, (endpoint, methods) in self.routes.items(): + endpoints[endpoint] = { + "route": route if route.startswith("/") else None, + "methods": methods, + "traffic": self.traffic_policies.get(endpoint, {}) + } + return endpoints async def set_traffic(self, endpoint_name, traffic_policy_dictionary): """Sets the traffic policy for the specified endpoint.""" diff --git a/python/ray/serve/tests/test_api.py b/python/ray/serve/tests/test_api.py index 7cc8de202..2c197b9f3 100644 --- a/python/ray/serve/tests/test_api.py +++ b/python/ray/serve/tests/test_api.py @@ -426,3 +426,65 @@ def test_parallel_start(serve_instance): handle = serve.get_handle("test-parallel") ray.get(handle.remote(), timeout=10) + + +def test_list_endpoints(serve_instance): + serve.init() + + def f(): + pass + + serve.create_endpoint("endpoint", "/api", methods=["GET", "POST"]) + serve.create_endpoint("endpoint2", methods=["POST"]) + serve.create_backend("backend", f) + serve.set_traffic("endpoint2", {"backend": 1.0}) + + endpoints = serve.list_endpoints() + assert "endpoint" in endpoints + assert endpoints["endpoint"] == { + "route": "/api", + "methods": ["GET", "POST"], + "traffic": {} + } + + assert "endpoint2" in endpoints + assert endpoints["endpoint2"] == { + "route": None, + "methods": ["POST"], + "traffic": { + "backend": 1.0 + } + } + + serve.delete_endpoint("endpoint") + assert "endpoint2" in serve.list_endpoints() + + serve.delete_endpoint("endpoint2") + assert len(serve.list_endpoints()) == 0 + + +def test_list_backends(serve_instance): + serve.init() + + @serve.accept_batch + def f(): + pass + + serve.create_backend("backend", f, config={"max_batch_size": 10}) + backends = serve.list_backends() + assert len(backends) == 1 + assert "backend" in backends + assert backends["backend"]["max_batch_size"] == 10 + + serve.create_backend("backend2", f, config={"num_replicas": 10}) + backends = serve.list_backends() + assert len(backends) == 2 + assert backends["backend2"]["num_replicas"] == 10 + + serve.delete_backend("backend") + backends = serve.list_backends() + assert len(backends) == 1 + assert "backend2" in backends + + serve.delete_backend("backend2") + assert len(serve.list_backends()) == 0