From edb84659101870d77b761081552da8e96122fc6b Mon Sep 17 00:00:00 2001
From: Devin Petersohn <devin-petersohn@users.noreply.github.com>
Date: Wed, 8 May 2019 13:40:54 -0700
Subject: [PATCH] [ray-core] Initial addition of performance integration
 testing files (#4325)

---
 .travis.yml                                   |  2 +-
 .../run_perf_integration.sh                   | 25 +++++++++++
 .../tests/perf_integration_tests/__init__.py  |  0
 .../test_perf_integration.py                  | 43 +++++++++++++++++++
 4 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100755 ci/jenkins_tests/perf_integration_tests/run_perf_integration.sh
 create mode 100644 python/ray/tests/perf_integration_tests/__init__.py
 create mode 100644 python/ray/tests/perf_integration_tests/test_perf_integration.py

diff --git a/.travis.yml b/.travis.yml
index ce4a7c961..7402e6fb6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -176,7 +176,7 @@ script:
   # ray tests
   # Python3.5+ only. Otherwise we will get `SyntaxError` regardless of how we set the tester.
   - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -c 'import sys;exit(sys.version_info>=(3,5))' || python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/test/async_test.py; fi
-  - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=10 --timeout=300 python/ray/tests; fi
+  - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=10 --timeout=300 python/ray/tests --ignore=python/ray/tests/perf_integration_tests; fi
 
 deploy:
   - provider: s3
diff --git a/ci/jenkins_tests/perf_integration_tests/run_perf_integration.sh b/ci/jenkins_tests/perf_integration_tests/run_perf_integration.sh
new file mode 100755
index 000000000..927f8bf5e
--- /dev/null
+++ b/ci/jenkins_tests/perf_integration_tests/run_perf_integration.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Show explicitly which commands are currently running.
+set -ex
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+pushd "$ROOT_DIR"
+
+python -m pip install pytest-benchmark
+
+pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.7.0.dev3-cp27-cp27mu-manylinux1_x86_64.whl
+python -m pytest --benchmark-autosave --benchmark-min-rounds=10 --benchmark-columns="min, max, mean" $ROOT_DIR/../../../python/ray/tests/perf_integration_tests/test_perf_integration.py
+
+pushd $ROOT_DIR/../../../python
+python -m pip install -e .
+popd
+
+python -m pytest --benchmark-compare --benchmark-min-rounds=10 --benchmark-compare-fail=min:5% --benchmark-columns="min, max, mean" $ROOT_DIR/../../../python/ray/tests/perf_integration_tests/test_perf_integration.py
+
+# This is how Modin stores the values in an S3 bucket
+#sha_tag=`git rev-parse --verify --short HEAD`
+# save the results to S3
+#aws s3 cp .benchmarks/*/*.json s3://modin-jenkins-result/${sha_tag}-perf-${BUCKET_SUFFIX}/ --acl public-read
+#rm -rf .benchmarks
diff --git a/python/ray/tests/perf_integration_tests/__init__.py b/python/ray/tests/perf_integration_tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/python/ray/tests/perf_integration_tests/test_perf_integration.py b/python/ray/tests/perf_integration_tests/test_perf_integration.py
new file mode 100644
index 000000000..2ce2a305a
--- /dev/null
+++ b/python/ray/tests/perf_integration_tests/test_perf_integration.py
@@ -0,0 +1,43 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import pytest
+
+import ray
+
+num_tasks_submitted = [10**n for n in range(0, 6)]
+num_tasks_ids = ["{}_tasks".format(i) for i in num_tasks_submitted]
+
+
+@ray.remote
+def dummy_task(val):
+    return val
+
+
+def benchmark_task_submission(num_tasks):
+    total_tasks = 100000
+    for _ in range(total_tasks // num_tasks):
+        ray.get([dummy_task.remote(i) for i in range(num_tasks)])
+
+
+def warmup():
+    x = np.zeros(10**6, dtype=np.uint8)
+    for _ in range(5):
+        for _ in range(5):
+            ray.put(x)
+        for _ in range(5):
+            ray.get([dummy_task.remote(0) for _ in range(1000)])
+
+
+@pytest.mark.benchmark
+@pytest.mark.parametrize("num_tasks", num_tasks_submitted, ids=num_tasks_ids)
+def test_task_submission(benchmark, num_tasks):
+    num_cpus = 16
+    ray.init(
+        num_cpus=num_cpus, object_store_memory=10**7, ignore_reinit_error=True)
+    # warm up the plasma store
+    warmup()
+    benchmark(benchmark_task_submission, num_tasks)
+    ray.shutdown()