wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-11 22:17:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Export Metrics in OpenCensus Protobuf Format (#10080)

Browse Source
This commit is contained in:
Simon Mo
2020-08-18 11:32:42 -07:00
committed by GitHub
parent 8d06e30a06
commit bedc2c24c8
13 changed files with 329 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tests/test_metrics_agent.py
+82 -225
View File
@@ -1,190 +1,16 @@
import asyncio
import json
import time
from collections import defaultdict
from pprint import pformat
import requests
import pytest
from opencensus.tags import tag_key as tag_key_module
from prometheus_client.parser import text_string_to_metric_families
import ray
from ray.core.generated.common_pb2 import MetricPoint
from ray.dashboard.util import get_unused_port
from ray.metrics_agent import (Gauge, MetricsAgent,
PrometheusServiceDiscoveryWriter)
from ray.metrics_agent import PrometheusServiceDiscoveryWriter
from ray.experimental.metrics import Count, Histogram
from ray.test_utils import wait_for_condition, SignalActor
def generate_metrics_point(name: str,
value: float,
timestamp: int,
tags: dict,
description: str = None,
units: str = None):
return MetricPoint(
metric_name=name,
timestamp=timestamp,
value=value,
tags=tags,
description=description,
units=units)
# NOTE: Opencensus metrics is a singleton per process.
# That says, we should re-use the same agent for all tests.
# Please be careful when you add new tests here. If each
# test doesn't use different metrics, it can have some confliction.
metrics_agent = None
@pytest.fixture
def cleanup_agent():
global metrics_agent
if not metrics_agent:
metrics_agent = MetricsAgent(get_unused_port())
yield
metrics_agent._registry = defaultdict(lambda: None)
def test_gauge():
tags = [tag_key_module.TagKey(str(i)) for i in range(10)]
name = "name"
description = "description"
units = "units"
gauge = Gauge(name, description, units, tags)
assert gauge.__dict__()["name"] == name
assert gauge.__dict__()["description"] == description
assert gauge.__dict__()["units"] == units
assert gauge.__dict__()["tags"] == tags
def test_basic_e2e(cleanup_agent):
# Test the basic end to end workflow. This includes.
# - Metrics are reported.
# - Metrics are dynamically registered to registry.
# - Metrics are accessbiel from Prometheus.
POINTS_DEF = [0, 1, 2]
tag = {"TAG_KEY": "TAG_VALUE"}
metrics_points = [
generate_metrics_point(
str(i), float(i), i, tag, description=str(i), units=str(i))
for i in POINTS_DEF
]
metrics_points_dict = {
metric_point.metric_name: metric_point
for metric_point in metrics_points
}
assert metrics_agent.record_metrics_points(metrics_points) is False
# Make sure all metrics are registered.
for i, metric_entry in zip(POINTS_DEF, metrics_agent.registry.items()):
metric_name, metric_entry = metric_entry
assert metric_name == metric_entry.name
assert metric_entry.name == str(i)
assert metric_entry.description == str(i)
assert metric_entry.units == str(i)
assert metric_entry.tags == [tag_key_module.TagKey(key) for key in tag]
# Make sure all metrics are available through a port.
response = requests.get("http://localhost:{}".format(
metrics_agent.metrics_export_port))
response.raise_for_status()
for line in response.text.split("\n"):
for family in text_string_to_metric_families(line):
metric_name = family.name
if metric_name not in metrics_points_dict:
continue
if line.startswith("# HELP"):
# description
assert (family.documentation == metrics_points_dict[
metric_name].description)
else:
for sample in family.samples:
metrics_points_dict[metric_name].value == sample.value
def test_missing_def(cleanup_agent):
# Make sure when metrics with description and units are reported,
# agent updates its registry to include them.
POINTS_DEF = [4, 5, 6]
tag = {"TAG_KEY": "TAG_VALUE"}
metrics_points = [
generate_metrics_point(
str(i),
float(i),
i,
tag,
) for i in POINTS_DEF
]
# At first, metrics shouldn't have description and units.
assert metrics_agent.record_metrics_points(metrics_points) is True
for i, metric_entry in zip(POINTS_DEF, metrics_agent.registry.items()):
metric_name, metric_entry = metric_entry
assert metric_name == metric_entry.name
assert metric_entry.name == str(i)
assert metric_entry.description == ""
assert metric_entry.units == ""
assert metric_entry.tags == [tag_key_module.TagKey(key) for key in tag]
# The points are coming again with description and units.
# Make sure they are updated.
metrics_points = [
generate_metrics_point(
str(i), float(i), i, tag, description=str(i), units=str(i))
for i in POINTS_DEF
]
assert metrics_agent.record_metrics_points(metrics_points) is False
for i, metric_entry in zip(POINTS_DEF, metrics_agent.registry.items()):
metric_name, metric_entry = metric_entry
assert metric_name == metric_entry.name
assert metric_entry.name == str(i)
assert metric_entry.description == str(i)
assert metric_entry.units == str(i)
assert metric_entry.tags == [tag_key_module.TagKey(key) for key in tag]
def test_multiple_record(cleanup_agent):
# Make sure prometheus export data properly when multiple points with
# the same name is reported.
TOTAL_POINTS = 10
NAME = "TEST"
values = list(range(TOTAL_POINTS))
tags = [{"TAG_KEY": str(i)} for i in range(TOTAL_POINTS)]
timestamps = list(range(TOTAL_POINTS))
points = []
for i in range(TOTAL_POINTS):
points.append(
generate_metrics_point(
name=NAME,
value=values[i],
timestamp=timestamps[i],
tags=tags[i]))
for point in points:
metrics_agent.record_metrics_points([point])
# Make sure data is available at prometheus.
response = requests.get("http://localhost:{}".format(
metrics_agent.metrics_export_port))
response.raise_for_status()
sample_values = []
for line in response.text.split("\n"):
for family in text_string_to_metric_families(line):
metric_name = family.name
name_without_prefix = metric_name.split("_")[1]
if name_without_prefix != NAME:
continue
# Lines for recorded metrics values.
for sample in family.samples:
sample_values.append(sample.value)
assert sample_values == [point.value for point in points]
def test_prometheus_file_based_service_discovery(ray_start_cluster):
# Make sure Prometheus service discovery file is correctly written
# when number of nodes are dynamically changed.
@@ -217,8 +43,8 @@ def test_prometheus_file_based_service_discovery(ray_start_cluster):
loaded_json_data["targets"]))
@pytest.mark.skip("This test is flaky right now. Will be fixed in #10080")
def test_metrics_export_end_to_end(ray_start_cluster):
@pytest.fixture
def _setup_cluster_for_test(ray_start_cluster):
NUM_NODES = 2
cluster = ray_start_cluster
# Add a head node.
@@ -231,36 +57,25 @@ def test_metrics_export_end_to_end(ray_start_cluster):
cluster.wait_for_nodes()
ray.init(address=cluster.address)
signal = SignalActor.remote()
worker_should_exit = SignalActor.remote()
# Generate some metrics around actor & tasks.
# Generate some metrics from actor & tasks.
@ray.remote
def f():
counter = Count("test_counter", "desc", "unit", [])
ray.get(signal.send.remote())
while True:
counter.record(1, {})
time.sleep(0.1)
counter = Count(f"test_counter", "desc", "unit", [])
counter.record(1, {})
ray.get(worker_should_exit.wait.remote())
@ray.remote
class A:
async def ready(self):
pass
async def ping(self):
histogram = Histogram("test_histogram", "desc", "unit", [0, 1, 2],
histogram = Histogram("test_histogram", "desc", "unit", [0.1, 1.6],
[])
while True:
histogram.record(1, {})
await asyncio.sleep(0.1)
histogram.record(1.5, {})
ray.get(worker_should_exit.wait.remote())
obj_refs = [f.remote() for _ in range(30)]
a = A.remote()
obj_refs.append(a.ping.remote())
# Make sure both histogram and counter are created
ray.get(a.ready.remote())
ray.get(signal.wait.remote())
obj_refs = [f.remote(), a.ping.remote()]
node_info_list = ray.nodes()
prom_addresses = []
@@ -269,62 +84,104 @@ def test_metrics_export_end_to_end(ray_start_cluster):
addr = node_info["NodeManagerAddress"]
prom_addresses.append(f"{addr}:{metrics_export_port}")
yield prom_addresses
ray.get(worker_should_exit.send.remote())
ray.get(obj_refs)
ray.shutdown()
cluster.shutdown()
def test_metrics_export_end_to_end(_setup_cluster_for_test):
TEST_TIMEOUT_S = 20
prom_addresses = _setup_cluster_for_test
# Make sure we can ping Prometheus endpoints.
def fetch_prometheus(prom_addresses):
components_dict = {}
metric_names = set()
metric_samples = []
for address in prom_addresses:
if address not in components_dict:
components_dict[address] = set()
try:
response = requests.get(
"http://localhost:{}".format(metrics_export_port))
response = requests.get(f"http://{address}/metrics")
except requests.exceptions.ConnectionError:
return components_dict, metric_names
continue
for line in response.text.split("\n"):
for family in text_string_to_metric_families(line):
for sample in family.samples:
# print(sample)
metric_names.add(sample.name)
metric_samples.append(sample)
if "Component" in sample.labels:
components_dict[address].add(
sample.labels["Component"])
return components_dict, metric_names
return components_dict, metric_names, metric_samples
def test_prometheus_endpoint():
# TODO(Simon): Add a gcs_server after fixing metrics.
components_dict, metric_names = fetch_prometheus(prom_addresses)
def test_cases():
components_dict, metric_names, metric_samples = fetch_prometheus(
prom_addresses)
# Raylet should be on every node
expected_components = {"raylet"}
components_found = all(
expected_components.issubset(components)
for components in components_dict.values())
assert all(
"raylet" in components for components in components_dict.values())
# Core worker should be on at least one node
components_found = components_found and any(
"core_worker" in components
for components in components_dict.values())
# GCS server should be on one node
assert any("gcs_server" in components
for components in components_dict.values())
expected_metric_names = {"ray_test_counter", "ray_test_histogram_max"}
metric_names_found = expected_metric_names.issubset(metric_names)
# Core worker should be on at least on node
assert any("core_worker" in components
for components in components_dict.values())
return components_found and metric_names_found
# Make sure our user defined metrics exist
for metric_name in ["test_counter", "test_histogram"]:
assert any(metric_name in full_name for full_name in metric_names)
# Make sure the numeric value is correct
test_counter_sample = [
m for m in metric_samples if "test_counter" in m.name
][0]
assert test_counter_sample.value == 1.0
# Make sure the numeric value is correct
test_histogram_samples = [
m for m in metric_samples if "test_histogram" in m.name
]
buckets = {
m.labels["le"]: m.value
for m in test_histogram_samples if "_bucket" in m.name
}
# We recorded value 1.5 for the histogram. In Prometheus data model
# the histogram is cumulative. So we expect the count to appear in
# <1.1 and <+Inf buckets.
assert buckets == {"0.1": 0.0, "1.6": 1.0, "+Inf": 1.0}
hist_count = [m for m in test_histogram_samples
if "_count" in m.name][0].value
hist_sum = [m for m in test_histogram_samples
if "_sum" in m.name][0].value
assert hist_count == 1
assert hist_sum == 1.5
def wrap_test_case_for_retry():
try:
test_cases()
return True
except AssertionError:
return False
try:
wait_for_condition(
test_prometheus_endpoint,
timeout=20,
wrap_test_case_for_retry,
timeout=TEST_TIMEOUT_S,
retry_interval_ms=1000, # Yield resource for other processes
)
except RuntimeError:
# This is for debugging when test failed.
raise RuntimeError(
"All components were not visible to "
"prometheus endpoints on time. "
f"The compoenents are {fetch_prometheus(prom_addresses)}")
ray.shutdown()
print(
f"The compoenents are {pformat(fetch_prometheus(prom_addresses))}")
test_cases() # Should fail assert
if __name__ == "__main__":