Export Metrics in OpenCensus Protobuf Format (#10080)

src/ray/protobuf/BUILD

@@ -56,7 +56,10 @@ python_grpc_compile(
 proto_library(
     name = "reporter_proto",
     srcs = ["reporter.proto"],
-    deps = [":common_proto"],
+    deps = [
+        ":common_proto",
+        "@io_opencensus_proto//opencensus/proto/metrics/v1:metrics_proto"
+    ],
 )
 
 cc_proto_library(

src/ray/protobuf/reporter.proto

@@ -16,6 +16,8 @@ syntax = "proto3";
 
 package ray.rpc;
 
+import "opencensus/proto/metrics/v1/metrics.proto";
+
 import "src/ray/protobuf/common.proto";
 
 message GetProfilingStatsRequest {
@@ -42,10 +44,19 @@ message ReportMetricsReply {
   bool metrcs_description_required = 1;
 }
 
+message ReportOCMetricsRequest {
+  repeated opencensus.proto.metrics.v1.Metric metrics = 1;
+}
+
+message ReportOCMetricsReply {
+}
+
 // Service for communicating with the reporter.py process on a remote node.
 service ReporterService {
   // Get the profiling stats.
   rpc GetProfilingStats(GetProfilingStatsRequest) returns (GetProfilingStatsReply);
   // Report metrics to the local metrics agents.
   rpc ReportMetrics(ReportMetricsRequest) returns (ReportMetricsReply);
+  // Report OpenCensus metrics to the local metrics agent.
+  rpc ReportOCMetrics(ReportOCMetricsRequest) returns (ReportOCMetricsReply);
 }

src/ray/rpc/metrics_agent_client.h

@@ -47,6 +47,12 @@ class MetricsAgentClient {
   /// \param[in] callback The callback function that handles reply.
   VOID_RPC_CLIENT_METHOD(ReporterService, ReportMetrics, grpc_client_, )
 
+  /// Report open census protobuf metrics to metrics agent.
+  ///
+  /// \param[in] request The request message.
+  /// \param[in] callback The callback function that handles reply.
+  VOID_RPC_CLIENT_METHOD(ReporterService, ReportOCMetrics, grpc_client_, )
+
  private:
   /// The RPC client.
   std::unique_ptr<GrpcClient<ReporterService>> grpc_client_;

src/ray/stats/metric_exporter.cc

@@ -20,7 +20,7 @@ namespace ray {
 namespace stats {
 
 template <>
-void MetricExporter::ExportToPoints(
+void MetricPointExporter::ExportToPoints(
     const opencensus::stats::ViewData::DataMap<opencensus::stats::Distribution>
         &view_data,
     const opencensus::stats::MeasureDescriptor &measure_descriptor,
@@ -75,7 +75,7 @@ void MetricExporter::ExportToPoints(
   }
 }
 
-void MetricExporter::ExportViewData(
+void MetricPointExporter::ExportViewData(
     const std::vector<std::pair<opencensus::stats::ViewDescriptor,
                                 opencensus::stats::ViewData>> &data) {
   std::vector<MetricPoint> points;
@@ -110,5 +110,108 @@ void MetricExporter::ExportViewData(
   metric_exporter_client_->ReportMetrics(points);
 }
 
+OpenCensusProtoExporter::OpenCensusProtoExporter(const int port,
+                                                 boost::asio::io_service &io_service,
+                                                 const std::string address)
+    : client_call_manager_(io_service) {
+  client_.reset(new rpc::MetricsAgentClient(address, port, client_call_manager_));
+};
+
+void OpenCensusProtoExporter::ExportViewData(
+    const std::vector<std::pair<opencensus::stats::ViewDescriptor,
+                                opencensus::stats::ViewData>> &data) {
+  // Start converting opencensus data into their protobuf format.
+  // The format can be found here
+  // https://github.com/census-instrumentation/opencensus-proto/blob/master/src/opencensus/proto/metrics/v1/metrics.proto
+  rpc::ReportOCMetricsRequest request_proto;
+
+  for (const auto &datum : data) {
+    // Unpack the fields we need for in memory data structure.
+    auto &view_descriptor = datum.first;
+    auto &view_data = datum.second;
+    auto &measure_descriptor = view_descriptor.measure_descriptor();
+
+    // Create one metric `Point` in protobuf.
+    auto request_point_proto = request_proto.add_metrics();
+
+    // Write the `MetricDescriptor`.
+    auto metric_descriptor_proto = request_point_proto->mutable_metric_descriptor();
+    metric_descriptor_proto->set_name(measure_descriptor.name());
+    metric_descriptor_proto->set_description(measure_descriptor.description());
+    metric_descriptor_proto->set_unit(measure_descriptor.units());
+    for (const auto &tag_key : view_descriptor.columns()) {
+      metric_descriptor_proto->add_label_keys()->set_key(tag_key.name());
+    };
+
+    // Helpers for writing the actual `TimeSeries`.
+    auto start_time = absl::ToUnixSeconds(view_data.start_time());
+    auto end_time = absl::ToUnixSeconds(view_data.end_time());
+    auto make_new_data_point_proto = [&request_point_proto, start_time, end_time](
+                                         const std::vector<std::string> &tag_values) {
+      auto metric_timeseries_proto = request_point_proto->add_timeseries();
+      metric_timeseries_proto->mutable_start_timestamp()->set_seconds(start_time);
+
+      for (const auto &value : tag_values) {
+        metric_timeseries_proto->add_label_values()->set_value(value);
+      };
+
+      auto point_proto = metric_timeseries_proto->add_points();
+      point_proto->mutable_timestamp()->set_seconds(end_time);
+      return point_proto;
+    };
+
+    // Write the `TimeSeries` for the given aggregated data type.
+    switch (view_data.type()) {
+    case opencensus::stats::ViewData::Type::kDouble:
+      for (const auto &row : view_data.double_data()) {
+        auto point_proto = make_new_data_point_proto(row.first /*tag_values*/);
+        point_proto->set_double_value(row.second);
+      }
+      break;
+    case opencensus::stats::ViewData::Type::kInt64:
+      for (const auto &row : view_data.int_data()) {
+        auto point_proto = make_new_data_point_proto(row.first /*tag_values*/);
+        point_proto->set_int64_value(row.second);
+      }
+      break;
+    case opencensus::stats::ViewData::Type::kDistribution:
+      for (const auto &row : view_data.distribution_data()) {
+        opencensus::stats::Distribution dist_value = row.second;
+
+        auto point_proto = make_new_data_point_proto(row.first /*tag_values*/);
+
+        // Copy in memory data into `DistributionValue` protobuf.
+        auto distribution_proto = point_proto->mutable_distribution_value();
+        distribution_proto->set_count(dist_value.count());
+        distribution_proto->set_sum(dist_value.count() * dist_value.mean());
+        distribution_proto->set_sum_of_squared_deviation(
+            dist_value.sum_of_squared_deviation());
+
+        // Write the `BucketOption` and `Bucket` data.
+        auto bucket_opt_proto =
+            distribution_proto->mutable_bucket_options()->mutable_explicit_();
+        for (const auto &bound : dist_value.bucket_boundaries().lower_boundaries()) {
+          bucket_opt_proto->add_bounds(bound);
+        }
+        for (const auto &count : dist_value.bucket_counts()) {
+          distribution_proto->add_buckets()->set_count(count);
+        }
+      }
+      break;
+    default:
+      RAY_LOG(FATAL) << "Unknown view data type.";
+      break;
+    }
+  }
+
+  client_->ReportOCMetrics(
+      request_proto, [](const Status &status, const rpc::ReportOCMetricsReply &reply) {
+        RAY_UNUSED(reply);
+        if (!status.ok()) {
+          RAY_LOG(WARNING) << "Export metrics to agent failed: " << status;
+        }
+      });
+}
+
 }  // namespace stats
 }  // namespace ray

src/ray/stats/metric_exporter.h

@@ -13,9 +13,11 @@
 // limitations under the License.
 
 #pragma once
+#include <boost/asio.hpp>
 #include "absl/memory/memory.h"
 #include "opencensus/stats/stats.h"
 #include "opencensus/tags/tag_key.h"
+#include "ray/rpc/client_call.h"
 #include "ray/stats/metric.h"
 #include "ray/stats/metric_exporter_client.h"
 #include "ray/util/logging.h"
@@ -28,19 +30,21 @@ namespace stats {
 /// opencensus data view, and sends it to the remote (for example
 /// sends metrics to dashboard agents through RPC). How to use it? Register metrics
 /// exporter after a main thread launched.
-class MetricExporter final : public opencensus::stats::StatsExporter::Handler {
+class MetricPointExporter final : public opencensus::stats::StatsExporter::Handler {
  public:
-  explicit MetricExporter(std::shared_ptr<MetricExporterClient> metric_exporter_client,
-                          size_t report_batch_size = kDefaultBatchSize)
+  explicit MetricPointExporter(
+      std::shared_ptr<MetricExporterClient> metric_exporter_client,
+      size_t report_batch_size = kDefaultBatchSize)
       : metric_exporter_client_(metric_exporter_client),
         report_batch_size_(report_batch_size) {}
 
-  ~MetricExporter() = default;
+  ~MetricPointExporter() = default;
 
   static void Register(std::shared_ptr<MetricExporterClient> metric_exporter_client,
                        size_t report_batch_size) {
     opencensus::stats::StatsExporter::RegisterPushHandler(
-        absl::make_unique<MetricExporter>(metric_exporter_client, report_batch_size));
+        absl::make_unique<MetricPointExporter>(metric_exporter_client,
+                                               report_batch_size));
   }
 
   void ExportViewData(
@@ -84,5 +88,29 @@ class MetricExporter final : public opencensus::stats::StatsExporter::Handler {
   size_t report_batch_size_;
 };
 
+class OpenCensusProtoExporter final : public opencensus::stats::StatsExporter::Handler {
+ public:
+  OpenCensusProtoExporter(const int port, boost::asio::io_service &io_service,
+                          const std::string address);
+
+  ~OpenCensusProtoExporter() = default;
+
+  static void Register(const int port, boost::asio::io_service &io_service,
+                       const std::string address) {
+    opencensus::stats::StatsExporter::RegisterPushHandler(
+        absl::make_unique<OpenCensusProtoExporter>(port, io_service, address));
+  }
+
+  void ExportViewData(
+      const std::vector<std::pair<opencensus::stats::ViewDescriptor,
+                                  opencensus::stats::ViewData>> &data) override;
+
+ private:
+  /// Call Manager for gRPC client.
+  rpc::ClientCallManager client_call_manager_;
+  /// Client to call a metrics agent gRPC server.
+  std::unique_ptr<rpc::MetricsAgentClient> client_;
+};
+
 }  // namespace stats
 }  // namespace ray

src/ray/stats/metric_exporter_client.cc

@@ -42,45 +42,11 @@ void MetricExporterDecorator::ReportMetrics(const std::vector<MetricPoint> &poin
 ///
 /// Metrics Agent Exporter
 ///
-MetricsAgentExporter::MetricsAgentExporter(std::shared_ptr<MetricExporterClient> exporter,
-                                           const int port,
-                                           boost::asio::io_service &io_service,
-                                           const std::string address)
-    : MetricExporterDecorator(exporter), client_call_manager_(io_service) {
-  client_.reset(new rpc::MetricsAgentClient(address, port, client_call_manager_));
-}
+MetricsAgentExporter::MetricsAgentExporter(std::shared_ptr<MetricExporterClient> exporter)
+    : MetricExporterDecorator(exporter) {}
 
 void MetricsAgentExporter::ReportMetrics(const std::vector<MetricPoint> &points) {
   MetricExporterDecorator::ReportMetrics(points);
-  rpc::ReportMetricsRequest request;
-  for (auto point : points) {
-    auto metric_point = request.add_metrics_points();
-    metric_point->set_metric_name(point.metric_name);
-    metric_point->set_timestamp(point.timestamp);
-    metric_point->set_value(point.value);
-    auto mutable_tags = metric_point->mutable_tags();
-    for (auto &tag : point.tags) {
-      (*mutable_tags)[tag.first] = tag.second;
-    }
-    // If description and units information is requested from
-    // the metrics agent, append the information.
-    // TODO(sang): It can be inefficient if there are lots of new registered metrics.
-    // We should make it more efficient if there's compelling use cases.
-    if (should_update_description_) {
-      metric_point->set_description(point.measure_descriptor.description());
-      metric_point->set_units(point.measure_descriptor.units());
-    }
-  }
-  should_update_description_ = false;
-
-  // TODO(sang): Should retry metrics report if it fails.
-  client_->ReportMetrics(
-      request, [this](const Status &status, const rpc::ReportMetricsReply &reply) {
-        if (!status.ok()) {
-          RAY_LOG(WARNING) << "ReportMetrics failed with status " << status;
-        }
-        should_update_description_ = reply.metrcs_description_required();
-      });
 }
 
 }  // namespace stats

src/ray/stats/metric_exporter_client.h

@@ -14,9 +14,6 @@
 
 #pragma once
 
-#include <boost/asio.hpp>
-
-#include "ray/rpc/client_call.h"
 #include "ray/rpc/metrics_agent_client.h"
 #include "ray/stats/metric.h"
 
@@ -57,20 +54,11 @@ class MetricExporterDecorator : public MetricExporterClient {
 
 class MetricsAgentExporter : public MetricExporterDecorator {
  public:
-  MetricsAgentExporter(std::shared_ptr<MetricExporterClient> exporter, const int port,
-                       boost::asio::io_service &io_service, const std::string address);
+  MetricsAgentExporter(std::shared_ptr<MetricExporterClient> exporter);
 
   ~MetricsAgentExporter() {}
 
   void ReportMetrics(const std::vector<MetricPoint> &points) override;
-
- private:
-  /// Client to call a metrics agent gRPC server.
-  std::unique_ptr<rpc::MetricsAgentClient> client_;
-  /// Call Manager for gRPC client.
-  rpc::ClientCallManager client_call_manager_;
-  /// Whether or not description and units information for metrics should be updated.
-  bool should_update_description_ = true;
 };
 
 }  // namespace stats

src/ray/stats/stats.h

@@ -83,8 +83,7 @@ static inline void Init(const TagsType &global_tags, const int metrics_agent_por
   // Default exporter is a metrics agent exporter.
   if (exporter_to_use == nullptr) {
     std::shared_ptr<MetricExporterClient> stdout_exporter(new StdoutExporterClient());
-    exporter.reset(new MetricsAgentExporter(stdout_exporter, metrics_agent_port,
-                                            (*metrics_io_service), "127.0.0.1"));
+    exporter.reset(new MetricsAgentExporter(stdout_exporter));
   } else {
     exporter = exporter_to_use;
   }
@@ -96,7 +95,9 @@ static inline void Init(const TagsType &global_tags, const int metrics_agent_por
       absl::Milliseconds(std::max(RayConfig::instance().metrics_report_interval_ms() / 2,
                                   static_cast<uint64_t>(500))));
 
-  MetricExporter::Register(exporter, metrics_report_batch_size);
+  MetricPointExporter::Register(exporter, metrics_report_batch_size);
+  OpenCensusProtoExporter::Register(metrics_agent_port, (*metrics_io_service),
+                                    "127.0.0.1");
   opencensus::stats::StatsExporter::SetInterval(
       StatsConfig::instance().GetReportInterval());
   opencensus::stats::DeltaProducer::Get()->SetHarvestInterval(