From aaa8147acbb42a81a178b7ee25067ed6b2aa86a0 Mon Sep 17 00:00:00 2001
From: Gekho457 <62982571+Gekho457@users.noreply.github.com>
Date: Mon, 5 Oct 2020 14:20:45 -0400
Subject: [PATCH] [CI] Add mypy to ci (#11117)

* Add mypy check to format.sh

* format_all() runs mypy check only on specified files
---
 ci/travis/format.sh             | 34 ++++++++++++++++++++++++++++++++-
 python/requirements_linters.txt |  1 +
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/ci/travis/format.sh b/ci/travis/format.sh
index 4b3252e10..35add5cbd 100755
--- a/ci/travis/format.sh
+++ b/ci/travis/format.sh
@@ -8,6 +8,7 @@ set -euo pipefail
 FLAKE8_VERSION_REQUIRED="3.7.7"
 YAPF_VERSION_REQUIRED="0.23.0"
 SHELLCHECK_VERSION_REQUIRED="0.7.1"
+MYPY_VERSION_REQUIRED="0.782"
 
 check_command_exist() {
     VERSION=""
@@ -21,6 +22,9 @@ check_command_exist() {
         shellcheck)
             VERSION=$SHELLCHECK_VERSION_REQUIRED
             ;;
+        mypy)
+            VERSION=$MYPY_VERSION_REQUIRED
+            ;;
         *)
             echo "$1 is not a required dependency"
             exit 1
@@ -33,6 +37,7 @@ check_command_exist() {
 
 check_command_exist yapf
 check_command_exist flake8
+check_command_exist mypy
 
 ver=$(yapf --version)
 if ! echo "$ver" | grep -q 0.23.0; then
@@ -46,9 +51,10 @@ builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
 ROOT="$(git rev-parse --show-toplevel)"
 builtin cd "$ROOT" || exit 1
 
-FLAKE8_VERSION=$(flake8 --version | awk '{print $1}')
+FLAKE8_VERSION=$(flake8 --version | head -n 1 | awk '{print $1}')
 YAPF_VERSION=$(yapf --version | awk '{print $2}')
 SHELLCHECK_VERSION=$(shellcheck --version | awk '/^version:/ {print $2}')
+MYPY_VERSION=$(mypy --version | awk '{print $2}')
 
 # params: tool name, tool version, required version
 tool_version_check() {
@@ -60,6 +66,7 @@ tool_version_check() {
 tool_version_check "flake8" "$FLAKE8_VERSION" "$FLAKE8_VERSION_REQUIRED"
 tool_version_check "yapf" "$YAPF_VERSION" "$YAPF_VERSION_REQUIRED"
 tool_version_check "shellcheck" "$SHELLCHECK_VERSION" "$SHELLCHECK_VERSION_REQUIRED"
+tool_version_check "mypy" "$MYPY_VERSION" "$MYPY_VERSION_REQUIRED"
 
 if which clang-format >/dev/null; then
   CLANG_FORMAT_VERSION=$(clang-format --version | awk '{print $3}')
@@ -84,6 +91,16 @@ YAPF_FLAGS=(
     '--parallel'
 )
 
+# TODO(dmitri): When more of the codebase is typed properly, the mypy flags
+# should be set to do a more stringent check. 
+MYPY_FLAGS=(
+    '--follow-imports=skip'
+)
+
+MYPY_FILES=(
+    'python/ray/autoscaler/node_provider.py'
+)
+
 YAPF_EXCLUDES=(
     '--exclude' 'python/ray/cloudpickle/*'
     '--exclude' 'python/build/*'
@@ -106,6 +123,16 @@ shellcheck_scripts() {
   shellcheck "${SHELLCHECK_FLAGS[@]}" "$@"
 }
 
+# Runs mypy on each argument in sequence. This is different than running mypy 
+# once on the list of arguments.
+mypy_on_each() {
+    for file in "$@"; do
+       echo "Running mypy on $file"
+       mypy ${MYPY_FLAGS[@]+"${MYPY_FLAGS[@]}"} "$file"
+    done
+}
+
+
 # Format specified files
 format_files() {
     local shell_files=() python_files=() bazel_files=()
@@ -139,6 +166,8 @@ format_files() {
 
     if [ 0 -lt "${#python_files[@]}" ]; then
       yapf --in-place "${YAPF_FLAGS[@]}" -- "${python_files[@]}"
+      echo "Running mypy on provided python files:"
+      mypy_on_each "${python_files[@]}" 
     fi
 
     if shellcheck --shell=sh --format=diff - < /dev/null; then
@@ -154,6 +183,7 @@ format_files() {
 }
 
 # Format all files, and print the diff to stdout for travis.
+# Mypy is run only on files specified in the array MYPY_FILES.
 format_all() {
     command -v flake8 &> /dev/null;
     HAS_FLAKE8=$?
@@ -161,6 +191,8 @@ format_all() {
     echo "$(date)" "YAPF...."
     git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \
       yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
+    echo "$(date)" "MYPY...."
+    mypy_on_each "${MYPY_FILES[@]}"
     if [ $HAS_FLAKE8 ]; then
       echo "$(date)" "Flake8...."
       git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 5 \
diff --git a/python/requirements_linters.txt b/python/requirements_linters.txt
index 0af7a00b4..41cad4dd6 100644
--- a/python/requirements_linters.txt
+++ b/python/requirements_linters.txt
@@ -1,4 +1,5 @@
 flake8==3.7.7 
 flake8-comprehensions
 flake8-quotes==2.0.0
+mypy==0.782
 yapf==0.23.0