From 48042be8bb47604cbaee87b4df996e07ce817198 Mon Sep 17 00:00:00 2001
From: Richard Liaw <rliaw@berkeley.edu>
Date: Fri, 20 Nov 2020 13:01:20 -0800
Subject: [PATCH] [tune] Avoid dependency on Kubernetes (#12188)

* fix-kubernetes

Signed-off-by: Richard Liaw <rliaw@berkeley.edu>

* kub

Signed-off-by: Richard Liaw <rliaw@berkeley.edu>
---
 python/ray/tune/integration/kubernetes.py | 17 +++++++++++++++--
 python/ray/tune/syncer.py                 | 13 +++++++++++--
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/python/ray/tune/integration/kubernetes.py b/python/ray/tune/integration/kubernetes.py
index 3f776b168..484475508 100644
--- a/python/ray/tune/integration/kubernetes.py
+++ b/python/ray/tune/integration/kubernetes.py
@@ -1,7 +1,5 @@
 import os
 from typing import Any, Optional, Tuple
-
-import kubernetes
 import subprocess
 
 from ray import services, logger
@@ -10,6 +8,17 @@ from ray.tune.syncer import NodeSyncer
 from ray.tune.sync_client import SyncClient
 
 
+def try_import_kubernetes():
+    try:
+        import kubernetes
+    except ImportError:
+        kubernetes = None
+    return kubernetes
+
+
+kubernetes = try_import_kubernetes()
+
+
 def NamespacedKubernetesSyncer(namespace):
     """Wrapper to return a ``KubernetesSyncer`` for a Kubernetes namespace.
 
@@ -53,6 +62,10 @@ class KubernetesSyncer(NodeSyncer):
                  local_dir: str,
                  remote_dir: str,
                  sync_client: Optional[SyncClient] = None):
+        if not kubernetes:
+            raise ImportError(
+                "kubernetes is not installed on this machine/container. "
+                "Try: pip install kubernetes")
         self.local_ip = services.get_node_ip_address()
         self.local_node = self._get_kubernetes_node_by_ip(self.local_ip)
         self.worker_ip = None
diff --git a/python/ray/tune/syncer.py b/python/ray/tune/syncer.py
index 6e5ff23a0..16518edbf 100644
--- a/python/ray/tune/syncer.py
+++ b/python/ray/tune/syncer.py
@@ -455,7 +455,6 @@ def detect_sync_to_driver(
         sync_to_driver: Union[None, bool, Type],
         cluster_config_file: str = "~/ray_bootstrap_config.yaml"):
     from ray.tune.integration.docker import DockerSyncer
-    from ray.tune.integration.kubernetes import NamespacedKubernetesSyncer
 
     if isinstance(sync_to_driver, Type):
         return sync_to_driver
@@ -479,9 +478,19 @@ def detect_sync_to_driver(
         return DockerSyncer
 
     if config.get("provider", {}).get("type", "") == "kubernetes":
+        from ray.tune.integration.kubernetes import (
+            NamespacedKubernetesSyncer, try_import_kubernetes)
+        if not try_import_kubernetes():
+            logger.warning(
+                "Detected Ray autoscaling environment on Kubernetes, "
+                "but Kubernetes Python CLI is not installed. "
+                "Checkpoint syncing may not work properly across "
+                "multiple pods. Be sure to install 'kubernetes' on "
+                "each container.")
+
         namespace = config["provider"].get("namespace", "ray")
         logger.debug(
-            f"Detected Kubernetes autoscaling environment. Using "
+            f"Detected Ray autoscaling environment on Kubernetes. Using "
             f"`NamespacedKubernetesSyncer` with namespace `{namespace}` "
             f"as sync client. If this is not correct or leads to errors, "
             f"please pass a `sync_to_driver` parameter in the `SyncConfig` "