[Stats] Make metrics report time configurable (#10036)

* Done.

* Lint.

* Address code review.

* Address code review.

* Remove wrong commit.

* Fix a test error.

src/ray/common/ray_config_def.h

@@ -346,3 +346,6 @@ RAY_CONFIG(bool, enable_multi_tenancy, false)
 
 /// Whether start the Plasma Store as a Raylet thread.
 RAY_CONFIG(bool, ownership_based_object_directory_enabled, false)
+
+// The interval where metrics are exported in milliseconds.
+RAY_CONFIG(uint64_t, metrics_report_interval_ms, 10000)

src/ray/gcs/gcs_server/gcs_server_main.cc

@@ -57,7 +57,7 @@ int main(int argc, char *argv[]) {
   }
 
   RayConfig::instance().initialize(config_map);
-  const ray::stats::TagsType global_tags = {{ray::stats::JobNameKey, "gcs_server"},
+  const ray::stats::TagsType global_tags = {{ray::stats::ComponentKey, "gcs_server"},
                                             {ray::stats::VersionKey, "0.9.0.dev0"}};
   ray::stats::Init(global_tags, metrics_agent_port);
 

src/ray/stats/metric.h

@@ -76,12 +76,12 @@ class StatsConfig final {
   /// If true, don't collect metrics in this process.
   bool is_stats_disabled_ = true;
   // Regular reporting interval for all reporters.
-  absl::Duration report_interval_ = absl::Seconds(10);
+  absl::Duration report_interval_ = absl::Milliseconds(10000);
   // Time interval for periodic aggregation.
   // Exporter may capture empty collection if harvest interval is longer than
   // report interval. So harvest interval is suggusted to be half of report
   // interval.
-  absl::Duration harvest_interval_ = absl::Seconds(5);
+  absl::Duration harvest_interval_ = absl::Milliseconds(5000);
   // Whether or not if the stats has been initialized.
   bool is_initialized_ = false;
 };

src/ray/stats/stats.h

@@ -89,6 +89,13 @@ static inline void Init(const TagsType &global_tags, const int metrics_agent_por
     exporter = exporter_to_use;
   }
 
+  // Set interval.
+  StatsConfig::instance().SetReportInterval(absl::Milliseconds(std::max(
+      RayConfig::instance().metrics_report_interval_ms(), static_cast<uint64_t>(1000))));
+  StatsConfig::instance().SetHarvestInterval(
+      absl::Milliseconds(std::max(RayConfig::instance().metrics_report_interval_ms() / 2,
+                                  static_cast<uint64_t>(500))));
+
   MetricExporter::Register(exporter, metrics_report_batch_size);
   opencensus::stats::StatsExporter::SetInterval(
       StatsConfig::instance().GetReportInterval());
@@ -101,7 +108,6 @@ static inline void Init(const TagsType &global_tags, const int metrics_agent_por
 /// Shutdown the initialized stats library.
 /// This cleans up various threads and metadata for stats library.
 static inline void Shutdown() {
-  // TODO(sang): Harvest thread is not currently cleaned up.
   absl::MutexLock lock(&stats_mutex);
   if (!StatsConfig::instance().IsInitialized()) {
     // Return if stats had never been initialized.