diff --git a/java/BUILD.bazel b/java/BUILD.bazel
index 258215738..a5fd3df8b 100644
--- a/java/BUILD.bazel
+++ b/java/BUILD.bazel
@@ -264,6 +264,13 @@ genrule(
         WORK_DIR="$$(pwd)"
         rm -rf "$$WORK_DIR/python/ray/jars" && mkdir -p "$$WORK_DIR/python/ray/jars"
         cp -f $(location //java:ray_dist_deploy.jar) "$$WORK_DIR/python/ray/jars/ray_dist.jar"
+        chmod +w "$$WORK_DIR/python/ray/jars/ray_dist.jar"
+        zip -d "$$WORK_DIR/python/ray/jars/ray_dist.jar" \
+            native/*/gcs_server \
+            native/*/libray_redis_module.so \
+            native/*/plasma_store_server \
+            native/*/raylet \
+            native/*/redis-server
         date > $@
     """,
     local = 1,