[Stats] Metrics Export User Interface Part 2 (Prometheus Service Discovery) (#9970)

* In progress.

* In Progress.

* Finish the working version.

* Write a documentation.

* Addressed code review.

* Fix lint error.

* Lint.

* Addressed code review. Make test less flaky.

* Use a random port for ray start.

* Modify doc.

* Make write atomic.

python/ray/tests/test_metrics_agent.py

@@ -1,3 +1,4 @@
+import json
 import pytest
 
 from collections import defaultdict
@@ -7,9 +8,12 @@ import requests
 from opencensus.tags import tag_key as tag_key_module
 from prometheus_client.parser import text_string_to_metric_families
 
+import ray
+
 from ray.core.generated.common_pb2 import MetricPoint
 from ray.dashboard.util import get_unused_port
-from ray.metrics_agent import Gauge, MetricsAgent
+from ray.metrics_agent import (Gauge, MetricsAgent,
+                               PrometheusServiceDiscoveryWriter)
 
 
 def generate_metrics_point(name: str,
@@ -180,6 +184,38 @@ def test_multiple_record(cleanup_agent):
     assert sample_values == [point.value for point in points]
 
 
+def test_prometheus_file_based_service_discovery(ray_start_cluster):
+    # Make sure Prometheus service discovery file is correctly written
+    # when number of nodes are dynamically changed.
+    NUM_NODES = 5
+    cluster = ray_start_cluster
+    nodes = [cluster.add_node() for _ in range(NUM_NODES)]
+    cluster.wait_for_nodes()
+    addr = ray.init(address=cluster.address)
+    redis_address = addr["redis_address"]
+    writer = PrometheusServiceDiscoveryWriter(
+        redis_address, ray.ray_constants.REDIS_DEFAULT_PASSWORD, "/tmp/ray")
+
+    def get_metrics_export_address_from_node(nodes):
+        return [
+            "{}:{}".format(node.node_ip_address, node.metrics_export_port)
+            for node in nodes
+        ]
+
+    loaded_json_data = json.loads(writer.get_file_discovery_content())[0]
+    assert (set(get_metrics_export_address_from_node(nodes)) == set(
+        loaded_json_data["targets"]))
+
+    # Let's update nodes.
+    for _ in range(3):
+        nodes.append(cluster.add_node())
+
+    # Make sure service discovery file content is correctly updated.
+    loaded_json_data = json.loads(writer.get_file_discovery_content())[0]
+    assert (set(get_metrics_export_address_from_node(nodes)) == set(
+        loaded_json_data["targets"]))
+
+
 if __name__ == "__main__":
     import sys
     sys.exit(pytest.main(["-v", __file__]))