Improve cross language serialization (#10490)

java/runtime/src/main/java/io/ray/runtime/serializer/MessagePackSerializer.java

@@ -208,7 +208,15 @@ public class MessagePackSerializer {
         }
       }
     }
-    typePacker.pack(object, packer, javaSerializer);
+    try {
+      typePacker.pack(object, packer, javaSerializer);
+    } catch (Exception e) {
+      if (typePacker != EXTENSION_PACKER) {
+        EXTENSION_PACKER.pack(object, packer, javaSerializer);
+      } else {
+        throw e;
+      }
+    }
   }
 
   private static Object unpack(Value v, Class<?> type, JavaDeserializer javaDeserializer) {

java/runtime/src/test/java/io/ray/runtime/serializer/SerializerTest.java

@@ -1,5 +1,6 @@
 package io.ray.runtime.serializer;
 
+import java.math.BigInteger;
 import java.util.ArrayList;
 import org.apache.commons.lang3.tuple.Pair;
 import org.testng.Assert;
@@ -45,5 +46,18 @@ public class SerializerTest {
       Assert.assertFalse(serialized.getRight());
       Assert.assertEquals(foo.get(0), bar.get(0));
     }
+    // Test BigInteger.
+    {
+      BigInteger bi = BigInteger.valueOf(Long.MAX_VALUE);
+      Pair<byte[], Boolean> serialized = Serializer.encode(bi);
+      BigInteger newBi = Serializer.decode(serialized.getLeft(), BigInteger.class);
+      Assert.assertTrue(serialized.getRight());
+      Assert.assertEquals(bi, newBi);
+      bi = bi.pow(2);
+      serialized = Serializer.encode(bi);
+      newBi = Serializer.decode(serialized.getLeft(), BigInteger.class);
+      Assert.assertFalse(serialized.getRight());
+      Assert.assertEquals(bi, newBi);
+    }
   }
 }