[Metric] custom metrics refinement (#10861)

* In progress

* In Progress.

* Addressed code review.

* Add unit tests.

* Add a simple doc.

* Fixed test failure.

* Fix all test failures from serve.

* Addressed code review.

python/ray/serve/router.py

@@ -9,7 +9,7 @@ from dataclasses import dataclass, field
 from ray.exceptions import RayTaskError
 
 import ray
-from ray.experimental import metrics
+from ray.util import metrics
 from ray.serve.context import TaskContext
 from ray.serve.endpoint_policy import RandomEndpointPolicy
 from ray.serve.utils import logger, chain_future
@@ -139,21 +139,25 @@ class Router:
         # -- Metrics Registration -- #
         self.num_router_requests = metrics.Count(
             "num_router_requests",
-            "Number of requests processed by the router.", "requests",
-            ["endpoint"])
+            description="Number of requests processed by the router.",
+            tag_keys=("endpoint", ))
         self.num_error_endpoint_requests = metrics.Count(
             "num_error_endpoint_requests",
-            ("Number of requests that errored when getting results "
-             "for the endpoint."), "requests", ["endpoint"])
+            description=(
+                "Number of requests that errored when getting results "
+                "for the endpoint."),
+            tag_keys=("endpoint", ))
         self.num_error_backend_requests = metrics.Count(
             "num_error_backend_requests",
-            ("Number of requests that errored when getting result "
-             "from the backend."), "requests", ["backend"])
+            description=("Number of requests that errored when getting result "
+                         "from the backend."),
+            tag_keys=("backend", ))
 
         self.backend_queue_size = metrics.Gauge(
             "backend_queued_queries",
-            "Current number of queries queued in the router for a backend",
-            "requests", ["backend"])
+            description=("Current number of queries queued "
+                         "in the router for a backend"),
+            tag_keys=("backend", ))
 
         asyncio.get_event_loop().create_task(self.report_queue_lengths())
 
@@ -161,7 +165,7 @@ class Router:
                               **request_kwargs):
         endpoint = request_meta.endpoint
         logger.debug("Received a request for endpoint {}".format(endpoint))
-        self.num_router_requests.record(1, {"endpoint": endpoint})
+        self.num_router_requests.record(1, tags={"endpoint": endpoint})
 
         request_context = request_meta.request_context
         query = Query(
@@ -177,7 +181,8 @@ class Router:
         try:
             result = await query.async_future
         except RayTaskError as e:
-            self.num_error_endpoint_requests.record(1, {"endpoint": endpoint})
+            self.num_error_endpoint_requests.record(
+                1, tags={"endpoint": endpoint})
             result = e
         return result
 
@@ -301,7 +306,8 @@ class Router:
             else:
                 result = await object_ref
         except RayTaskError as error:
-            self.num_error_backend_requests.record(1, {"backend": backend})
+            self.num_error_backend_requests.record(
+                1, tags={"backend": backend})
             result = error
         self.queries_counter[backend][backend_replica_tag] -= 1
         await self.mark_worker_idle(backend, backend_replica_tag)
@@ -358,6 +364,7 @@ class Router:
                 self.name, queue_lengths)
 
             for backend, length in queue_lengths.items():
-                self.backend_queue_size.record(length, {"backend": backend})
+                self.backend_queue_size.record(
+                    length, tags={"backend": backend})
 
             await asyncio.sleep(REPORT_QUEUE_LENGTH_PERIOD_S)