Adds Valgrind tests for multi-threaded object manager. (#1890)

* adds valgrind to new object manager.

* Add some comments.

* Update run_object_manager_valgrind.sh

typo

* Update run_object_manager_tests.sh

* update tests to reflect changes in #1891.

* reduce # tests.

.travis.yml

@@ -61,6 +61,7 @@ matrix:
         - bash ../../../src/common/test/run_valgrind.sh
         - bash ../../../src/plasma/test/run_valgrind.sh
         - bash ../../../src/local_scheduler/test/run_valgrind.sh
+        - bash ../../../src/ray/test/run_object_manager_valgrind.sh
         - cd ../../..
 
         - python ./python/ray/plasma/test/test.py valgrind

src/ray/object_manager/test/object_manager_stress_test.cc

@@ -121,8 +121,8 @@ class TestObjectManagerBase : public ::testing::Test {
     ObjectManagerConfig om_config_1;
     om_config_1.store_socket_name = store_sock_1;
     om_config_1.num_threads = 4;
-    om_config_1.max_sends = 20;
-    om_config_1.max_receives = 20;
+    om_config_1.max_sends = 2;
+    om_config_1.max_receives = 2;
     server1.reset(new MockServer(main_service, std::move(object_manager_service_1),
                                  om_config_1, gcs_client_1));
 
@@ -131,8 +131,8 @@ class TestObjectManagerBase : public ::testing::Test {
     ObjectManagerConfig om_config_2;
     om_config_2.store_socket_name = store_sock_2;
     om_config_2.num_threads = 4;
-    om_config_2.max_sends = 20;
-    om_config_2.max_receives = 20;
+    om_config_2.max_sends = 2;
+    om_config_2.max_receives = 2;
     server2.reset(new MockServer(main_service, std::move(object_manager_service_2),
                                  om_config_2, gcs_client_2));
 
@@ -261,7 +261,7 @@ class StressTestObjectManager : public TestObjectManagerBase {
     async_loop_index += 1;
     if ((uint)async_loop_index < async_loop_patterns.size()) {
       TransferPattern pattern = async_loop_patterns[async_loop_index];
-      TransferTestExecute(1000, 100, pattern);
+      TransferTestExecute(100, 100, pattern);
     } else {
       main_service.stop();
     }

src/ray/test/run_object_manager_tests.sh

@@ -9,6 +9,7 @@ set -x
 # Get the directory in which this script is executing.
 SCRIPT_DIR="`dirname \"$0\"`"
 RAY_ROOT="$SCRIPT_DIR/../../.."
+# Makes $RAY_ROOT an absolute path.
 RAY_ROOT="`( cd \"$RAY_ROOT\" && pwd )`"
 if [ -z "$RAY_ROOT" ] ; then
   exit 1

src/ray/test/run_object_manager_valgrind.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This needs to be run in the build tree, which is normally ray/python/ray/core
+
+# Cause the script to exit if a single command fails.
+set -e
+set -x
+
+# Get the directory in which this script is executing.
+SCRIPT_DIR="`dirname \"$0\"`"
+RAY_ROOT="$SCRIPT_DIR/../../.."
+# Makes $RAY_ROOT an absolute path.
+RAY_ROOT="`( cd \"$RAY_ROOT\" && pwd )`"
+if [ -z "$RAY_ROOT" ] ; then
+  exit 1
+fi
+# Ensure we're in the right directory.
+if [ ! -d "$RAY_ROOT/python" ]; then
+  echo "Unable to find root Ray directory. Has this script moved?"
+  exit 1
+fi
+
+CORE_DIR="$RAY_ROOT/python/ray/core"
+REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src"
+REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so"
+STORE_EXEC="$CORE_DIR/src/plasma/plasma_store"
+VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1"
+
+echo "$STORE_EXEC"
+echo "$REDIS_DIR/redis-server --loglevel warning --loadmodule $REDIS_MODULE --port 6379"
+echo "$REDIS_DIR/redis-cli -p 6379 shutdown"
+
+# Allow cleanup commands to fail.
+killall plasma_store || true
+$REDIS_DIR/redis-cli -p 6379 shutdown || true
+sleep 1s
+$REDIS_DIR/redis-server --loglevel warning --loadmodule $REDIS_MODULE --port 6379 &
+sleep 1s
+
+# Run tests.
+$VALGRIND_CMD $CORE_DIR/src/ray/object_manager/object_manager_test $STORE_EXEC
+sleep 1s
+$VALGRIND_CMD $CORE_DIR/src/ray/object_manager/object_manager_stress_test $STORE_EXEC
+$REDIS_DIR/redis-cli -p 6379 shutdown
+sleep 1s
+
+# Include raylet integration test once it's ready.
+# $CORE_DIR/src/ray/raylet/object_manager_integration_test $STORE_EXEC