diff --git a/java/runtime/src/main/java/org/ray/runtime/task/ArgumentsBuilder.java b/java/runtime/src/main/java/org/ray/runtime/task/ArgumentsBuilder.java
index 03486c835..750cd08e7 100644
--- a/java/runtime/src/main/java/org/ray/runtime/task/ArgumentsBuilder.java
+++ b/java/runtime/src/main/java/org/ray/runtime/task/ArgumentsBuilder.java
@@ -2,8 +2,13 @@ package org.ray.runtime.task;
 
 import java.util.ArrayList;
 import java.util.List;
+
+import org.ray.api.Ray;
 import org.ray.api.RayObject;
 import org.ray.api.id.ObjectId;
+import org.ray.api.runtime.RayRuntime;
+import org.ray.runtime.AbstractRayRuntime;
+import org.ray.runtime.RayMultiWorkerNativeRuntime;
 import org.ray.runtime.generated.Common.Language;
 import org.ray.runtime.object.NativeRayObject;
 import org.ray.runtime.object.ObjectSerializer;
@@ -38,7 +43,13 @@ public class ArgumentsBuilder {
       } else {
         value = ObjectSerializer.serialize(arg);
         if (value.data.length > LARGEST_SIZE_PASS_BY_VALUE) {
-          // Do nothing since we are not support pass by reference in direct call.
+          RayRuntime runtime = Ray.internal();
+          if (runtime instanceof RayMultiWorkerNativeRuntime) {
+            runtime = ((RayMultiWorkerNativeRuntime) runtime).getCurrentRuntime();
+          }
+          id = ((AbstractRayRuntime) runtime).getObjectStore()
+            .putRaw(value);
+          value = null;
         }
       }
       if (language == Language.PYTHON) {