diff --git a/BUILD.bazel b/BUILD.bazel
index 8a209a8ad..dc93384f1 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -553,7 +553,7 @@ cc_library(
         ":gcs_service_rpc",
         ":gcs_table_storage_lib",
         ":node_manager_rpc",
-        ":raylet_lib",
+        ":raylet_client_lib",
         ":worker_rpc",
     ],
 )
@@ -662,6 +662,33 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "raylet_client_lib",
+    srcs = glob([
+        "src/ray/raylet_client/*.cc",
+    ]),
+    hdrs = glob([
+        "src/ray/raylet_client/*.h",
+    ]),
+    copts = COPTS,
+    linkopts = select({
+        "@bazel_tools//src/conditions:windows": [
+        ],
+        "//conditions:default": [
+            "-lpthread",
+        ],
+    }),
+    visibility = ["//streaming:__subpackages__"],
+    deps = [
+        ":gcs_cc_proto",
+        ":node_manager_fbs",
+        ":node_manager_rpc",
+        ":ray_common",
+        ":ray_util",
+        "@boost//:asio",
+    ],
+)
+
 cc_library(
     name = "core_worker_lib",
     srcs = glob(
@@ -686,17 +713,17 @@ cc_library(
     copts = COPTS,
     visibility = ["//visibility:public"],
     deps = [
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/container:flat_hash_set",
-        ":worker_cc_proto",
+        ":gcs",
+        ":plasma_client",
         ":ray_common",
         ":ray_util",
-        # TODO(hchen): After `raylet_client` is migrated to gRPC, `core_worker_lib`
-        # should only depend on `raylet_client`, instead of the whole `raylet_lib`.
-        ":raylet_lib",
+        ":raylet_client_lib",
+        ":service_based_gcs_client_lib",
+        ":worker_cc_proto",
         ":worker_rpc",
-        ":gcs",
         "@boost//:fiber",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
diff --git a/src/ray/core_worker/common.h b/src/ray/core_worker/common.h
index 5111f398d..b8afe83ea 100644
--- a/src/ray/core_worker/common.h
+++ b/src/ray/core_worker/common.h
@@ -19,7 +19,7 @@
 #include "ray/common/id.h"
 #include "ray/common/ray_object.h"
 #include "ray/common/task/task_spec.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 #include "ray/util/util.h"
 
 namespace ray {
diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h
index 65db7724c..b0ca19539 100644
--- a/src/ray/core_worker/core_worker.h
+++ b/src/ray/core_worker/core_worker.h
@@ -33,7 +33,7 @@
 #include "ray/core_worker/transport/raylet_transport.h"
 #include "ray/gcs/redis_gcs_client.h"
 #include "ray/gcs/subscription_executor.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 #include "ray/rpc/node_manager/node_manager_client.h"
 #include "ray/rpc/worker/core_worker_client.h"
 #include "ray/rpc/worker/core_worker_server.h"
diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc
index fabc01fe1..1fdc00c17 100644
--- a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc
+++ b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc
@@ -18,7 +18,7 @@
 #include "ray/core_worker/common.h"
 #include "ray/core_worker/core_worker.h"
 #include "ray/core_worker/lib/java/jni_utils.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/src/ray/core_worker/object_recovery_manager.h b/src/ray/core_worker/object_recovery_manager.h
index 9e8769f36..fb19c4553 100644
--- a/src/ray/core_worker/object_recovery_manager.h
+++ b/src/ray/core_worker/object_recovery_manager.h
@@ -20,7 +20,7 @@
 #include "ray/core_worker/reference_count.h"
 #include "ray/core_worker/store_provider/memory_store/memory_store.h"
 #include "ray/core_worker/task_manager.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 
 namespace ray {
 
diff --git a/src/ray/core_worker/store_provider/plasma_store_provider.h b/src/ray/core_worker/store_provider/plasma_store_provider.h
index 47fb54144..b8a0fd143 100644
--- a/src/ray/core_worker/store_provider/plasma_store_provider.h
+++ b/src/ray/core_worker/store_provider/plasma_store_provider.h
@@ -23,7 +23,7 @@
 #include "ray/core_worker/context.h"
 #include "ray/core_worker/reference_count.h"
 #include "ray/object_manager/plasma/client.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 
 namespace ray {
 
diff --git a/src/ray/core_worker/test/core_worker_test.cc b/src/ray/core_worker/test/core_worker_test.cc
index 18ecec934..b4727cf17 100644
--- a/src/ray/core_worker/test/core_worker_test.cc
+++ b/src/ray/core_worker/test/core_worker_test.cc
@@ -33,7 +33,7 @@
 #include "ray/core_worker/transport/direct_actor_transport.h"
 #include "ray/protobuf/core_worker.pb.h"
 #include "ray/protobuf/gcs.pb.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 #include "ray/util/filesystem.h"
 
 namespace {
diff --git a/src/ray/core_worker/test/direct_actor_transport_test.cc b/src/ray/core_worker/test/direct_actor_transport_test.cc
index 00a6ae163..c5421f340 100644
--- a/src/ray/core_worker/test/direct_actor_transport_test.cc
+++ b/src/ray/core_worker/test/direct_actor_transport_test.cc
@@ -18,7 +18,7 @@
 #include "ray/common/test_util.h"
 #include "ray/core_worker/store_provider/memory_store/memory_store.h"
 #include "ray/core_worker/transport/direct_task_transport.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 #include "ray/rpc/worker/core_worker_client.h"
 
 namespace ray {
diff --git a/src/ray/core_worker/test/direct_task_transport_test.cc b/src/ray/core_worker/test/direct_task_transport_test.cc
index 07dbc8476..871550a1e 100644
--- a/src/ray/core_worker/test/direct_task_transport_test.cc
+++ b/src/ray/core_worker/test/direct_task_transport_test.cc
@@ -19,7 +19,7 @@
 #include "ray/common/task/task_util.h"
 #include "ray/common/test_util.h"
 #include "ray/core_worker/store_provider/memory_store/memory_store.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 #include "ray/rpc/worker/core_worker_client.h"
 
 namespace ray {
diff --git a/src/ray/core_worker/test/object_recovery_manager_test.cc b/src/ray/core_worker/test/object_recovery_manager_test.cc
index fe8d6a64f..f0003a3f6 100644
--- a/src/ray/core_worker/test/object_recovery_manager_test.cc
+++ b/src/ray/core_worker/test/object_recovery_manager_test.cc
@@ -20,7 +20,7 @@
 #include "ray/common/test_util.h"
 #include "ray/core_worker/store_provider/memory_store/memory_store.h"
 #include "ray/core_worker/transport/direct_task_transport.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 
 namespace ray {
 
diff --git a/src/ray/core_worker/transport/direct_task_transport.h b/src/ray/core_worker/transport/direct_task_transport.h
index ea80109fb..2d57adfff 100644
--- a/src/ray/core_worker/transport/direct_task_transport.h
+++ b/src/ray/core_worker/transport/direct_task_transport.h
@@ -25,7 +25,7 @@
 #include "ray/core_worker/task_manager.h"
 #include "ray/core_worker/transport/dependency_resolver.h"
 #include "ray/core_worker/transport/direct_actor_transport.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 #include "ray/rpc/worker/core_worker_client.h"
 
 namespace ray {
diff --git a/src/ray/core_worker/transport/raylet_transport.h b/src/ray/core_worker/transport/raylet_transport.h
index 39c39e364..e05b1827b 100644
--- a/src/ray/core_worker/transport/raylet_transport.h
+++ b/src/ray/core_worker/transport/raylet_transport.h
@@ -18,7 +18,7 @@
 
 #include "ray/common/ray_object.h"
 #include "ray/core_worker/reference_count.h"
-#include "ray/raylet/raylet_client.h"
+#include "ray/raylet_client/raylet_client.h"
 #include "ray/rpc/worker/core_worker_server.h"
 
 namespace ray {
diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h
index f3371ceb6..5dc57d2e0 100644
--- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h
+++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h
@@ -19,7 +19,7 @@
 #include <ray/common/task/task_spec.h>
 #include <ray/gcs/accessor.h>
 #include <ray/protobuf/gcs_service.pb.h>
-#include <ray/raylet/raylet_client.h>
+#include <ray/raylet_client/raylet_client.h>
 #include <ray/rpc/node_manager/node_manager_client.h>
 #include <ray/rpc/worker/core_worker_client.h>
 #include <queue>
diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h
index 7fdcf0582..081281305 100644
--- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h
+++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h
@@ -16,7 +16,7 @@
 #include <ray/common/id.h>
 #include <ray/gcs/accessor.h>
 #include <ray/protobuf/gcs_service.pb.h>
-#include <ray/raylet/raylet_client.h>
+#include <ray/raylet_client/raylet_client.h>
 #include <ray/rpc/node_manager/node_manager_client.h>
 #include <ray/rpc/worker/core_worker_client.h>
 #include <queue>
diff --git a/src/ray/raylet/raylet_client.cc b/src/ray/raylet_client/raylet_client.cc
similarity index 100%
rename from src/ray/raylet/raylet_client.cc
rename to src/ray/raylet_client/raylet_client.cc
diff --git a/src/ray/raylet/raylet_client.h b/src/ray/raylet_client/raylet_client.h
similarity index 100%
rename from src/ray/raylet/raylet_client.h
rename to src/ray/raylet_client/raylet_client.h