[Stats] Metrics Export User Interface Part 1 (#9913)

* Metrics export port expose done.

* Support exposing metrics port + metrics agent service discovery through ray.nodes()

* Formatting.

* Added a doc.

* Linting.

* Change the location of metrics agent port.

* Addressed code review.

* Addressed code review.

src/ray/common/ray_config_def.h

@@ -325,10 +325,6 @@ RAY_CONFIG(int64_t, enable_metrics_collection, true)
 /// Whether start the Plasma Store as a Raylet thread.
 RAY_CONFIG(bool, put_small_object_in_memory_store, false)
 
-/// Metric agent port for reporting, default -1 means no such agent will be
-/// listening.
-RAY_CONFIG(int, metrics_agent_port, -1)
-
 /// Maximum number of tasks that can be in flight between an owner and a worker for which
 /// the owner has been granted a lease. A value >1 is used when we want to enable
 /// pipelining task submission.

src/ray/core_worker/core_worker.cc

@@ -141,7 +141,7 @@ CoreWorkerProcess::CoreWorkerProcess(const CoreWorkerOptions &options)
   // NOTE(lingxuan.zlx): We assume RayConfig is initialized before it's used.
   // RayConfig is generated in Java_io_ray_runtime_RayNativeRuntime_nativeInitialize
   // for java worker or in constructor of CoreWorker for python worker.
-  ray::stats::Init(global_tags, RayConfig::instance().metrics_agent_port());
+  ray::stats::Init(global_tags, options_.metrics_agent_port);
 }
 
 CoreWorkerProcess::~CoreWorkerProcess() {

src/ray/core_worker/core_worker.h

@@ -121,6 +121,9 @@ struct CoreWorkerOptions {
   std::function<void()> terminate_asyncio_thread;
   /// Serialized representation of JobConfig.
   std::string serialized_job_config;
+  /// The port number of a metrics agent that imports metrics from core workers.
+  /// -1 means there's no such agent.
+  int metrics_agent_port;
 };
 
 /// Lifecycle management of one or more `CoreWorker` instances in a process.

src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc

@@ -198,7 +198,7 @@ JNIEXPORT void JNICALL Java_io_ray_runtime_RayNativeRuntime_nativeInitialize(
       RAY_LOG(INFO) << "Calling System.gc() ...";
       env->CallStaticObjectMethod(java_system_class, java_system_gc);
       last_gc_time_ms = current_time_ms();
-      RAY_LOG(INFO) << "GC finished in " << (double) (last_gc_time_ms - start) / 1000
+      RAY_LOG(INFO) << "GC finished in " << (double)(last_gc_time_ms - start) / 1000
                     << " seconds.";
     }
   };
@@ -232,6 +232,7 @@ JNIEXPORT void JNICALL Java_io_ray_runtime_RayNativeRuntime_nativeInitialize(
       static_cast<int>(numWorkersPerProcess),        // num_workers
       nullptr,                                       // terminate_asyncio_thread
       serialized_job_config,                         // serialized_job_config
+      -1,                                            // metrics_agent_port
   };
 
   ray::CoreWorkerProcess::Initialize(options);

src/ray/core_worker/test/core_worker_test.cc

@@ -163,6 +163,9 @@ class CoreWorkerTest : public ::testing::Test {
           true,                           // ref_counting_enabled
           false,                          // is_local_mode
           1,                              // num_workers
+          nullptr,                        // terminate_asyncio_thread
+          "",                             // serialized_job_config
+          -1,                             // metrics_agent_port
       };
       CoreWorkerProcess::Initialize(options);
     }

src/ray/core_worker/test/mock_worker.cc

@@ -59,6 +59,9 @@ class MockWorker {
         true,           // ref_counting_enabled
         false,          // is_local_mode
         1,              // num_workers
+        nullptr,        // terminate_asyncio_thread
+        "",             // serialized_job_config
+        -1,             // metrics_agent_port
     };
     CoreWorkerProcess::Initialize(options);
   }

src/ray/protobuf/gcs.proto

@@ -249,6 +249,9 @@ message GcsNodeInfo {
 
   // The Hostname address of the node manager.
   string node_manager_hostname = 8;
+
+  // The port at which the node will expose metrics to.
+  int32 metrics_export_port = 9;
 }
 
 // Represents the demand for a particular resource shape.

src/ray/raylet/main.cc

@@ -28,6 +28,7 @@ DEFINE_string(store_socket_name, "", "The socket name of object store.");
 DEFINE_int32(object_manager_port, -1, "The port of object manager.");
 DEFINE_int32(node_manager_port, -1, "The port of node manager.");
 DEFINE_int32(metrics_agent_port, -1, "The port of metrics agent.");
+DEFINE_int32(metrics_export_port, 1, "Maximum startup concurrency");
 DEFINE_string(node_ip_address, "", "The ip address of this node.");
 DEFINE_string(redis_address, "", "The ip address of redis server.");
 DEFINE_int32(redis_port, -1, "The port of redis server.");
@@ -88,6 +89,7 @@ int main(int argc, char *argv[]) {
   const int64_t object_store_memory = FLAGS_object_store_memory;
   const std::string plasma_directory = FLAGS_plasma_directory;
   const bool huge_pages = FLAGS_huge_pages;
+  const int metrics_export_port = FLAGS_metrics_export_port;
   gflags::ShutDownCommandLineFlags();
 
   // Configuration for the node manager.
@@ -231,7 +233,8 @@ int main(int argc, char *argv[]) {
         // Initialize the node manager.
         server.reset(new ray::raylet::Raylet(
             main_service, raylet_socket_name, node_ip_address, redis_address, redis_port,
-            redis_password, node_manager_config, object_manager_config, gcs_client));
+            redis_password, node_manager_config, object_manager_config, gcs_client,
+            metrics_export_port));
 
         server->Start();
       }));

src/ray/raylet/raylet.cc

@@ -59,7 +59,7 @@ Raylet::Raylet(boost::asio::io_service &main_service, const std::string &socket_
                int redis_port, const std::string &redis_password,
                const NodeManagerConfig &node_manager_config,
                const ObjectManagerConfig &object_manager_config,
-               std::shared_ptr<gcs::GcsClient> gcs_client)
+               std::shared_ptr<gcs::GcsClient> gcs_client, int metrics_export_port)
     : self_node_id_(ClientID::FromRandom()),
       gcs_client_(gcs_client),
       object_directory_(std::make_shared<ObjectDirectory>(main_service, gcs_client_)),
@@ -78,6 +78,7 @@ Raylet::Raylet(boost::asio::io_service &main_service, const std::string &socket_
   self_node_info_.set_object_manager_port(object_manager_.GetServerPort());
   self_node_info_.set_node_manager_port(node_manager_.GetServerPort());
   self_node_info_.set_node_manager_hostname(boost::asio::ip::host_name());
+  self_node_info_.set_metrics_export_port(metrics_export_port);
 }
 
 Raylet::~Raylet() {}

src/ray/raylet/raylet.h

@@ -49,12 +49,13 @@ class Raylet {
   /// \param object_manager_config Configuration to initialize the object
   /// manager.
   /// \param gcs_client A client connection to the GCS.
+  /// \param metrics_export_port A port at which metrics are exposed to.
   Raylet(boost::asio::io_service &main_service, const std::string &socket_name,
          const std::string &node_ip_address, const std::string &redis_address,
          int redis_port, const std::string &redis_password,
          const NodeManagerConfig &node_manager_config,
          const ObjectManagerConfig &object_manager_config,
-         std::shared_ptr<gcs::GcsClient> gcs_client);
+         std::shared_ptr<gcs::GcsClient> gcs_client, int metrics_export_port);
 
   /// Start this raylet.
   void Start();