diff --git a/CMakeLists.txt b/CMakeLists.txt
index 881f93350..d02e88a5c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -154,5 +154,5 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
 
   # copy libplasma_java files
   add_custom_command(TARGET copy_ray POST_BUILD
-    COMMAND bash -c "cp ${ARROW_HOME}/lib/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma")
+    COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma")
 endif()
diff --git a/cmake/Modules/ThirdpartyToolchain.cmake b/cmake/Modules/ThirdpartyToolchain.cmake
index 43a0a162e..11d7428ae 100644
--- a/cmake/Modules/ThirdpartyToolchain.cmake
+++ b/cmake/Modules/ThirdpartyToolchain.cmake
@@ -179,6 +179,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   # clean the arrow_ep/python/build/lib.xxxxx directory,
   # or when you build with another python version, it creates multiple lib.xxxx directories
   set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${ARROW_SOURCE_DIR}/python/build/")
+  set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${CMAKE_SOURCE_DIR}/python/ray/pyarrow_files/pyarrow")
 
   # here we use externalProject to process pyarrow building
   # add_custom_command would have problem with setup.py
@@ -189,6 +190,6 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
     BUILD_IN_SOURCE 1
     CONFIGURE_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build
     BUILD_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build_ext
-    INSTALL_COMMAND bash -c "cp -r \$(find ${ARROW_SOURCE_DIR}/python/build/ -maxdepth 1 -type d -print | grep -m1 'lib')/pyarrow ${CMAKE_SOURCE_DIR}/python/ray/pyarrow_files/")
+    INSTALL_COMMAND bash -c "cp -rf \$(find ${ARROW_SOURCE_DIR}/python/build/ -maxdepth 1 -type d -print | grep -m1 'lib')/pyarrow ${CMAKE_SOURCE_DIR}/python/ray/pyarrow_files/")
 
 endif ()
diff --git a/test/runtest.py b/test/runtest.py
index 512d9ef59..eac25e283 100644
--- a/test/runtest.py
+++ b/test/runtest.py
@@ -2524,6 +2524,8 @@ class GlobalStateAPI(unittest.TestCase):
 
             if tables_ready:
                 break
+            # this test case is blocked sometimes, add this may fix the problem
+            time.sleep(0.1)
 
         # Flush the tables.
         ray.experimental.flush_redis_unsafe()
@@ -2537,6 +2539,7 @@ class GlobalStateAPI(unittest.TestCase):
         ray.get([f.remote() for _ in range(10)])
 
         while len(ray.global_state.task_table()) != 0:
+            time.sleep(0.1)
             ray.experimental.flush_finished_tasks_unsafe()
 
         # Make sure that we can call this method (but it won't do anything in