[Serve] Allow multiple HTTP servers. (#9523)

python/ray/serve/http_proxy.py

@@ -1,5 +1,6 @@
 import asyncio
 from urllib.parse import parse_qs
+import socket
 
 import uvicorn
 
@@ -30,7 +31,7 @@ class HTTPProxy:
         assert ray.is_initialized()
         controller = serve.api._get_controller()
 
-        self.route_table = await controller.get_http_proxy_config.remote()
+        self.route_table = await controller.get_router_config.remote()
 
         # The exporter is required to return results for /-/metrics endpoint.
         [self.metric_exporter] = await controller.get_metric_exporter.remote()
@@ -179,7 +180,19 @@ class HTTPProxyActor:
         # Start running the HTTP server on the event loop.
         asyncio.get_event_loop().create_task(self.run())
 
+    def ready(self):
+        return True
+
     async def run(self):
+        sock = socket.socket()
+        # These two socket options will allow multiple process to bind the the
+        # same port. Kernel will evenly load balance among the port listeners.
+        # Note: this will only work on Linux.
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        if hasattr(socket, "SO_REUSEPORT"):
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        sock.bind((self.host, self.port))
+
         # Note(simon): we have to use lower level uvicorn Config and Server
         # class because we want to run the server as a coroutine. The only
         # alternative is to call uvicorn.run which is blocking.
@@ -194,7 +207,7 @@ class HTTPProxyActor:
         # because the existing implementation fails if it isn't running in
         # the main thread and uvicorn doesn't expose a way to configure it.
         server.install_signal_handlers = lambda: None
-        await server.serve()
+        await server.serve(sockets=[sock])
 
     async def set_route_table(self, route_table):
         self.app.set_route_table(route_table)