[Serve] Performance: Use uvloop when possible (#9216)

python/ray/serve/benchmarks/baselines/Makefile

@@ -0,0 +1,12 @@
+noop:
+	@echo "please specify which baseline to run"
+
+uvicorn:
+	uvicorn uvicorn_app:app --no-access-log --workers 1
+
+fastapi:
+	uvicorn fastapi_app:app --no-access-log --workers 1
+
+bench:
+	wrk -c 100 -t 10 -d 10s http://127.0.0.1:8000
+

python/ray/serve/benchmarks/baselines/fastapi_app.py

@@ -0,0 +1,8 @@
+from fastapi import FastAPI
+
+app = FastAPI()
+
+
+@app.get("/")
+async def read_root():
+    return "Hello world"

python/ray/serve/benchmarks/baselines/uvicorn_app.py

@@ -0,0 +1,13 @@
+async def app(scope, receive, send):
+    assert scope["type"] == "http"
+    await send({
+        "type": "http.response.start",
+        "status": 200,
+        "headers": [
+            [b"content-type", b"text/plain"],
+        ]
+    })
+    await send({
+        "type": "http.response.body",
+        "body": b"Hello, world!",
+    })

python/ray/serve/benchmarks/noop_latency.py

@@ -1,35 +1,70 @@
-from ray import serve
-from ray.serve.constants import DEFAULT_HTTP_ADDRESS
-import requests
 import time
+from typing import Optional
+
+import requests
 import pandas as pd
 from tqdm import tqdm
+import click
 
-serve.init()
+from ray import serve
+from ray.serve.constants import DEFAULT_HTTP_ADDRESS
+from ray.serve import master
+
+master._TRACING_ENABLED = True
 
 
-def noop(_):
-    return ""
+def block_until_ready(url):
+    while requests.get(url).status_code == 404:
+        time.sleep(1)
+        print("Waiting for noop route to showup.")
 
 
-serve.create_backend("noop", noop)
-serve.create_endpoint("noop", backend="noop", route="/noop")
+def run_http_benchmark(url, num_queries):
+    latency = []
+    for _ in tqdm(range(num_queries + 200)):
+        start = time.perf_counter()
+        requests.get(url)
+        end = time.perf_counter()
+        latency.append(end - start)
 
-url = "{}/noop".format(DEFAULT_HTTP_ADDRESS)
-while requests.get(url).status_code == 404:
-    time.sleep(1)
-    print("Waiting for noop route to showup.")
+    # Remove initial samples
+    latency = latency[200:]
 
-latency = []
-for _ in tqdm(range(5200)):
-    start = time.perf_counter()
-    resp = requests.get(url)
-    end = time.perf_counter()
-    latency.append(end - start)
+    series = pd.Series(latency) * 1000
+    print("Latency for single noop backend (ms)")
+    print(series.describe(percentiles=[0.5, 0.9, 0.95, 0.99]))
 
-# Remove initial samples
-latency = latency[200:]
 
-series = pd.Series(latency) * 1000
-print("Latency for single noop backend (ms)")
-print(series.describe(percentiles=[0.5, 0.9, 0.95, 0.99]))
+@click.command()
+@click.option("--blocking", is_flag=True, required=False, help="Block forever")
+@click.option("--num-queries", type=int, required=False)
+@click.option("--num-replicas", type=int, default=1)
+@click.option("--max-concurrent-queries", type=int, required=False)
+def main(num_replicas: int, num_queries: Optional[int],
+         max_concurrent_queries: Optional[int], blocking: bool):
+    serve.init()
+
+    def noop(_):
+        return "hello world"
+
+    config = {
+        "num_replicas": num_replicas,
+        "max_concurrent_queries": max_concurrent_queries
+    }
+    print("Using config", config)
+    serve.create_backend("noop", noop, config=config)
+    serve.create_endpoint("noop", backend="noop", route="/noop")
+
+    url = "{}/noop".format(DEFAULT_HTTP_ADDRESS)
+    block_until_ready(url)
+
+    if num_queries:
+        run_http_benchmark(url, num_queries)
+    if blocking:
+        print("Endpoint {} is ready.".format(url))
+        while True:
+            time.sleep(5)
+
+
+if __name__ == "__main__":
+    main()