Fix bug in which remote function redefinition doesn't happen. (#6175)

python/ray/import_thread.py

@@ -2,10 +2,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import redis
+from collections import defaultdict
 import threading
 import traceback
 
+import redis
+
 import ray
 from ray import ray_constants
 from ray import cloudpickle as pickle
@@ -30,6 +32,11 @@ class ImportThread(object):
         redis_client: the redis client used to query exports.
         threads_stopped (threading.Event): A threading event used to signal to
             the thread that it should exit.
+        imported_collision_identifiers: This is a dictionary mapping collision
+            identifiers for the exported remote functions and actor classes to
+            the number of times that collision identifier has appeared. This is
+            used to provide good error messages when the same function or class
+            is exported many times.
     """
 
     def __init__(self, worker, mode, threads_stopped):
@@ -37,6 +44,7 @@ class ImportThread(object):
         self.mode = mode
         self.redis_client = worker.redis_client
         self.threads_stopped = threads_stopped
+        self.imported_collision_identifiers = defaultdict(int)
 
     def start(self):
         """Start the import thread."""
@@ -91,6 +99,18 @@ class ImportThread(object):
             # Close the pubsub client to avoid leaking file descriptors.
             import_pubsub_client.close()
 
+    def _get_import_info_for_collision_detection(self, key):
+        """Retrieve the collision identifier, type, and name of the import."""
+        if key.startswith(b"RemoteFunction"):
+            collision_identifier, function_name = (self.redis_client.hmget(
+                key, ["collision_identifier", "function_name"]))
+            return (collision_identifier, ray.utils.decode(function_name),
+                    "remote function")
+        elif key.startswith(b"ActorClass"):
+            collision_identifier, class_name = self.redis_client.hmget(
+                key, ["collision_identifier", "class_name"])
+            return collision_identifier, ray.utils.decode(class_name), "actor"
+
     def _process_key(self, key):
         """Process the given export key from redis."""
         # Handle the driver case first.
@@ -98,6 +118,32 @@ class ImportThread(object):
             if key.startswith(b"FunctionsToRun"):
                 with profiling.profile("fetch_and_run_function"):
                     self.fetch_and_execute_function_to_run(key)
+
+            # If the same remote function or actor definition appears to be
+            # exported many times, then print a warning. We only issue this
+            # warning from the driver so that it is only triggered once instead
+            # of many times. TODO(rkn): We may want to push this to the driver
+            # through Redis so that it can be displayed in the dashboard more
+            # easily.
+            elif (key.startswith(b"RemoteFunction")
+                  or key.startswith(b"ActorClass")):
+                collision_identifier, name, import_type = (
+                    self._get_import_info_for_collision_detection(key))
+                self.imported_collision_identifiers[collision_identifier] += 1
+                if (self.imported_collision_identifiers[collision_identifier]
+                        == ray_constants.DUPLICATE_REMOTE_FUNCTION_THRESHOLD):
+                    logger.warning(
+                        "The %s '%s' has been exported %s times. It's "
+                        "possible that this warning is accidental, but this "
+                        "may indicate that the same remote function is being "
+                        "defined repeatedly from within many tasks and "
+                        "exported to all of the workers. This can be a "
+                        "performance issue and can be resolved by defining "
+                        "the remote function on the driver instead. See "
+                        "https://github.com/ray-project/ray/issues/6240 for "
+                        "more discussion.", import_type, name,
+                        ray_constants.DUPLICATE_REMOTE_FUNCTION_THRESHOLD)
+
             # Return because FunctionsToRun are the only things that
             # the driver should import.
             return