[Stats] Basic Metrics Infrastructure (Metrics Agent + Prometheus Exporter) (#9607)

src/ray/protobuf/common.proto

@@ -334,8 +334,16 @@ message CoreWorkerStats {
 }
 
 message MetricPoint {
+  // Name of the metric.
   string metric_name = 1;
+  // Timestamp when metric is exported.
   int64 timestamp = 2;
+  // Value of the metric point.
   double value = 3;
+  // Tags of the metric.
   map<string, string> tags = 4;
+  // [Optional] Description of the metric.
+  string description = 5;
+  // [Optional] Unit of the metric.
+  string units = 6;
 }

src/ray/protobuf/reporter.proto

@@ -35,10 +35,11 @@ message GetProfilingStatsReply {
 }
 
 message ReportMetricsRequest {
-  repeated MetricPoint metrics_point = 1;
+  repeated MetricPoint metrics_points = 1;
 }
 
 message ReportMetricsReply {
+  bool metrcs_description_required = 1;
 }
 
 // Service for communicating with the reporter.py process on a remote node.

src/ray/stats/metric.cc

@@ -36,6 +36,10 @@ static void RegisterAsView(opencensus::stats::ViewDescriptor view_descriptor,
   view_descriptor.RegisterForExport();
 }
 
+///
+/// Stats Config
+///
+
 StatsConfig &StatsConfig::instance() {
   static StatsConfig instance;
   return instance;
@@ -71,6 +75,9 @@ void StatsConfig::SetIsInitialized(bool initialized) { is_initialized_ = initial
 
 bool StatsConfig::IsInitialized() const { return is_initialized_; }
 
+///
+/// Metric
+///
 void Metric::Record(double value, const TagsType &tags) {
   if (StatsConfig::instance().IsStatsDisabled()) {
     return;

src/ray/stats/metric.h

@@ -185,7 +185,9 @@ struct MetricPoint {
   int64_t timestamp;
   double value;
   std::unordered_map<std::string, std::string> tags;
+  const opencensus::stats::MeasureDescriptor &measure_descriptor;
 };
+
 }  // namespace stats
 
 }  // namespace ray

src/ray/stats/metric_exporter.cc

@@ -23,13 +23,15 @@ template <>
 void MetricExporter::ExportToPoints(
     const opencensus::stats::ViewData::DataMap<opencensus::stats::Distribution>
         &view_data,
-    const std::string &metric_name, std::vector<std::string> &keys,
-    std::vector<MetricPoint> &points) {
+    const opencensus::stats::MeasureDescriptor &measure_descriptor,
+    std::vector<std::string> &keys, std::vector<MetricPoint> &points) {
   // Return if no raw data found in view map.
   if (view_data.size() == 0) {
     return;
   }
 
+  const auto &metric_name = measure_descriptor.name();
+
   // NOTE(lingxuan.zlx): No sampling in histogram data, so all points all be filled in.
   std::unordered_map<std::string, std::string> tags;
   for (size_t i = 0; i < view_data.begin()->first.size(); ++i) {
@@ -53,10 +55,12 @@ void MetricExporter::ExportToPoints(
     }
   }
   hist_mean /= view_data.size();
-  MetricPoint mean_point = {metric_name + ".mean", current_sys_time_ms(), hist_mean,
-                            tags};
-  MetricPoint max_point = {metric_name + ".max", current_sys_time_ms(), hist_max, tags};
-  MetricPoint min_point = {metric_name + ".min", current_sys_time_ms(), hist_min, tags};
+  MetricPoint mean_point = {metric_name + ".mean", current_sys_time_ms(), hist_mean, tags,
+                            measure_descriptor};
+  MetricPoint max_point = {metric_name + ".max", current_sys_time_ms(), hist_max, tags,
+                           measure_descriptor};
+  MetricPoint min_point = {metric_name + ".min", current_sys_time_ms(), hist_min, tags,
+                           measure_descriptor};
   points.push_back(std::move(mean_point));
   points.push_back(std::move(max_point));
   points.push_back(std::move(min_point));
@@ -85,17 +89,17 @@ void MetricExporter::ExportViewData(
     for (size_t i = 0; i < descriptor.columns().size(); ++i) {
       keys.push_back(descriptor.columns()[i].name());
     }
-    auto &metric_name = descriptor.name();
+    const auto &measure_descriptor = descriptor.measure_descriptor();
     switch (view_data.type()) {
     case opencensus::stats::ViewData::Type::kDouble:
-      ExportToPoints<double>(view_data.double_data(), metric_name, keys, points);
+      ExportToPoints<double>(view_data.double_data(), measure_descriptor, keys, points);
       break;
     case opencensus::stats::ViewData::Type::kInt64:
-      ExportToPoints<int64_t>(view_data.int_data(), metric_name, keys, points);
+      ExportToPoints<int64_t>(view_data.int_data(), measure_descriptor, keys, points);
       break;
     case opencensus::stats::ViewData::Type::kDistribution:
       ExportToPoints<opencensus::stats::Distribution>(view_data.distribution_data(),
-                                                      metric_name, keys, points);
+                                                      measure_descriptor, keys, points);
       break;
     default:
       RAY_LOG(FATAL) << "Unknown view data type.";

src/ray/stats/metric_exporter.h

@@ -57,8 +57,9 @@ class MetricExporter final : public opencensus::stats::StatsExporter::Handler {
   /// \param keys, metric tags map
   /// \param points, memory metric vector instance
   void ExportToPoints(const opencensus::stats::ViewData::DataMap<DTYPE> &view_data,
-                      const std::string &metric_name, std::vector<std::string> &keys,
-                      std::vector<MetricPoint> &points) {
+                      const opencensus::stats::MeasureDescriptor &measure_descriptor,
+                      std::vector<std::string> &keys, std::vector<MetricPoint> &points) {
+    const auto &metric_name = measure_descriptor.name();
     for (const auto &row : view_data) {
       std::unordered_map<std::string, std::string> tags;
       for (size_t i = 0; i < keys.size(); ++i) {
@@ -66,7 +67,7 @@ class MetricExporter final : public opencensus::stats::StatsExporter::Handler {
       }
       // Current timestamp is used for point not view data time.
       MetricPoint point{metric_name, current_sys_time_ms(),
-                        static_cast<double>(row.second), tags};
+                        static_cast<double>(row.second), tags, measure_descriptor};
       RAY_LOG(DEBUG) << "Metric name " << metric_name << ", value " << point.value;
       points.push_back(std::move(point));
       if (points.size() >= report_batch_size_) {

src/ray/stats/metric_exporter_client.cc

@@ -54,7 +54,7 @@ void MetricsAgentExporter::ReportMetrics(const std::vector<MetricPoint> &points)
   MetricExporterDecorator::ReportMetrics(points);
   rpc::ReportMetricsRequest request;
   for (auto point : points) {
-    auto metric_point = request.add_metrics_point();
+    auto metric_point = request.add_metrics_points();
     metric_point->set_metric_name(point.metric_name);
     metric_point->set_timestamp(point.timestamp);
     metric_point->set_value(point.value);
@@ -62,10 +62,22 @@ void MetricsAgentExporter::ReportMetrics(const std::vector<MetricPoint> &points)
     for (auto &tag : point.tags) {
       (*mutable_tags)[tag.first] = tag.second;
     }
+    // If description and units information is requested from
+    // the metrics agent, append the information.
+    // TODO(sang): It can be inefficient if there are lots of new registered metrics.
+    // We should make it more efficient if there's compelling use cases.
+    if (should_update_description_) {
+      metric_point->set_description(point.measure_descriptor.description());
+      metric_point->set_units(point.measure_descriptor.units());
+    }
   }
+  should_update_description_ = false;
 
   // TODO(sang): Should retry metrics report if it fails.
-  client_->ReportMetrics(request, nullptr);
+  client_->ReportMetrics(
+      request, [this](const Status &status, const rpc::ReportMetricsReply &reply) {
+        should_update_description_ = reply.metrcs_description_required();
+      });
 }
 
 }  // namespace stats

src/ray/stats/metric_exporter_client.h

@@ -69,6 +69,8 @@ class MetricsAgentExporter : public MetricExporterDecorator {
   std::unique_ptr<rpc::MetricsAgentClient> client_;
   /// Call Manager for gRPC client.
   rpc::ClientCallManager client_call_manager_;
+  /// Whether or not description and units information for metrics should be updated.
+  bool should_update_description_ = true;
 };
 
 }  // namespace stats