Fix Travis failure detection on Mac + testing enhancements (#358)

* properly handle failure in continuous integration

* add stack overflow reference

.travis.yml

@@ -16,6 +16,8 @@ install:
   - ./test/travis-ci/install.sh
 
 script:
-  - ./test/travis-ci/base_test.sh
-  - ./test/travis-ci/hyperopt_example.sh
-  - ./test/travis-ci/lbfgs_example.sh
+  - ./test/travis-ci/run_test.sh --docker-image=amplab/ray:test-base 'source setup-env.sh && cd test && python runtest.py'
+  - ./test/travis-ci/run_test.sh --docker-image=amplab/ray:test-base 'source setup-env.sh && cd test && python array_test.py'
+  - ./test/travis-ci/run_test.sh --docker-image=amplab/ray:test-base 'source setup-env.sh && cd test && python microbenchmarks.py'
+  - ./test/travis-ci/run_test.sh --docker-only --shm-size=500m --docker-image=amplab/ray:test-examples 'source setup-env.sh && cd examples/hyperopt && python driver.py'
+  - ./test/travis-ci/run_test.sh --docker-only --shm-size=500m --docker-image=amplab/ray:test-examples 'source setup-env.sh && cd examples/lbfgs && python driver.py'

test/travis-ci/base_test.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-if [[ $TRAVIS_OS_NAME == 'linux' ]]; then
-  # Linux test uses Docker
-  docker run amplab/ray:test-base bash -c 'source setup-env.sh && cd test && python runtest.py && python array_test.py && python microbenchmarks.py'
-else
-  # Mac OS X test
-  source setup-env.sh
-  pushd test
-    python runtest.py
-    python array_test.py
-    python microbenchmarks.py
-  popd
-fi

test/travis-ci/hyperopt_example.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-# Hyperparameter optimization test
-# Runs only on Docker under Linux
-if [[ $TRAVIS_OS_NAME == 'linux' ]]; then
-  docker run --shm-size=500m amplab/ray:test-examples bash -c 'source setup-env.sh && cd examples/hyperopt && python driver.py'
-fi

test/travis-ci/lbfgs_example.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-# L-BFGS Test
-# Runs only on Docker under Linux
-if [[ $TRAVIS_OS_NAME == 'linux' ]]; then
-  docker run --shm-size=500m amplab/ray:test-examples bash -c 'source setup-env.sh && cd examples/lbfgs && python driver.py'
-fi

test/travis-ci/run_test.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# This script is for use in the Travis CI testing system. When running
+# on Linux this script runs the command in a Docker container.
+# Otherwise it runs the commands natively, which is what we want for
+# the Mac OS X tests.
+#
+# Usage:
+#   
+#   run_test.sh [OPTIONS] [COMMAND]
+#
+# Key options are:
+#
+#  --docker-image=img    Docker image to run the tests
+#  --shm-size=xxx        Shared memory setting for Docker run command
+#  --docker-only         Skip this test unless running on docker
+#
+# Example:
+#
+#   run_test.sh --docker-only --shm-size=500m \
+#     --docker-image=amplab/ray:test-examples \
+#     'source setup-env.sh && cd examples/lbfgs && python driver.py'
+#
+# For further examples see this project's .travis.yml
+#
+
+# Argument parsing adapted from http://stackoverflow.com/a/14203146
+for i in "$@"
+do
+TEST_COMMAND=''
+case $i in
+    --docker-image=*)
+    DOCKER_IMAGE="${i#*=}"
+    shift
+    ;;
+    --shm-size=*)
+    SHM_SIZE="${i#*=}"
+    shift
+    ;;
+    --docker-only)
+    DOCKER_ONLY=YES
+    ;;
+    *)
+    TEST_COMMAND=$i
+    ;;
+esac
+done
+
+if [[ $TRAVIS_OS_NAME == 'linux' ]]; then
+    # Linux test uses Docker
+    SHM_ARG=$([[ -z $SHM_SIZE ]] && echo "" || echo "--shm-size $SHM_SIZE")
+    docker run $SHM_ARG $DOCKER_IMAGE bash -c "$TEST_COMMAND"
+else
+    # Mac OS X test
+    if [[ -z $DOCKER_ONLY || $DOCKER_ONLY -ne YES ]]; then
+        bash -c "$TEST_COMMAND"
+    else
+        echo "not on Docker, skipping test"
+    fi
+fi