[ray-core] Initial addition of performance integration testing files (#4325)

.travis.yml

@@ -176,7 +176,7 @@ script:
   # ray tests
   # Python3.5+ only. Otherwise we will get `SyntaxError` regardless of how we set the tester.
   - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -c 'import sys;exit(sys.version_info>=(3,5))' || python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/test/async_test.py; fi
-  - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=10 --timeout=300 python/ray/tests; fi
+  - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=10 --timeout=300 python/ray/tests --ignore=python/ray/tests/perf_integration_tests; fi
 
 deploy:
   - provider: s3

ci/jenkins_tests/perf_integration_tests/run_perf_integration.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Show explicitly which commands are currently running.
+set -ex
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+pushd "$ROOT_DIR"
+
+python -m pip install pytest-benchmark
+
+pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.7.0.dev3-cp27-cp27mu-manylinux1_x86_64.whl
+python -m pytest --benchmark-autosave --benchmark-min-rounds=10 --benchmark-columns="min, max, mean" $ROOT_DIR/../../../python/ray/tests/perf_integration_tests/test_perf_integration.py
+
+pushd $ROOT_DIR/../../../python
+python -m pip install -e .
+popd
+
+python -m pytest --benchmark-compare --benchmark-min-rounds=10 --benchmark-compare-fail=min:5% --benchmark-columns="min, max, mean" $ROOT_DIR/../../../python/ray/tests/perf_integration_tests/test_perf_integration.py
+
+# This is how Modin stores the values in an S3 bucket
+#sha_tag=`git rev-parse --verify --short HEAD`
+# save the results to S3
+#aws s3 cp .benchmarks/*/*.json s3://modin-jenkins-result/${sha_tag}-perf-${BUCKET_SUFFIX}/ --acl public-read
+#rm -rf .benchmarks

python/ray/tests/perf_integration_tests/test_perf_integration.py

@@ -0,0 +1,43 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import pytest
+
+import ray
+
+num_tasks_submitted = [10**n for n in range(0, 6)]
+num_tasks_ids = ["{}_tasks".format(i) for i in num_tasks_submitted]
+
+
+@ray.remote
+def dummy_task(val):
+    return val
+
+
+def benchmark_task_submission(num_tasks):
+    total_tasks = 100000
+    for _ in range(total_tasks // num_tasks):
+        ray.get([dummy_task.remote(i) for i in range(num_tasks)])
+
+
+def warmup():
+    x = np.zeros(10**6, dtype=np.uint8)
+    for _ in range(5):
+        for _ in range(5):
+            ray.put(x)
+        for _ in range(5):
+            ray.get([dummy_task.remote(0) for _ in range(1000)])
+
+
+@pytest.mark.benchmark
+@pytest.mark.parametrize("num_tasks", num_tasks_submitted, ids=num_tasks_ids)
+def test_task_submission(benchmark, num_tasks):
+    num_cpus = 16
+    ray.init(
+        num_cpus=num_cpus, object_store_memory=10**7, ignore_reinit_error=True)
+    # warm up the plasma store
+    warmup()
+    benchmark(benchmark_task_submission, num_tasks)
+    ray.shutdown()