Make ActorHandles pickleable, also make proper ActorHandle and ActorC… (#2007)

* Make ActorHandles pickleable, also make proper ActorHandle and ActorClass classes.

* Fix bug.

* Fix actor test bug.

* Update __ray_terminate__ usage.

* Fix most linting, add documentation, and small cleanups.

* Handle forking and pickling differently for actor handles. Fix linting.

* Fixes for named actors via pickling.

* Generate actor handle IDs deterministically in the pickling case.

python/ray/rllib/a3c/a3c.py

@@ -126,7 +126,7 @@ class A3CAgent(Agent):
     def _stop(self):
         # workaround for https://github.com/ray-project/ray/issues/1516
         for ev in self.remote_evaluators:
-            ev.__ray_terminate__.remote(ev._ray_actor_id.id())
+            ev.__ray_terminate__.remote()
 
     def _save(self, checkpoint_dir):
         checkpoint_path = os.path.join(

python/ray/rllib/ddpg/ddpg.py

@@ -234,7 +234,7 @@ class DDPGAgent(Agent):
     def _stop(self):
         # workaround for https://github.com/ray-project/ray/issues/1516
         for ev in self.remote_evaluators:
-            ev.__ray_terminate__.remote(ev._ray_actor_id.id())
+            ev.__ray_terminate__.remote()
 
     def _save(self, checkpoint_dir):
         checkpoint_path = self.saver.save(

python/ray/rllib/dqn/dqn.py

@@ -232,7 +232,7 @@ class DQNAgent(Agent):
     def _stop(self):
         # workaround for https://github.com/ray-project/ray/issues/1516
         for ev in self.remote_evaluators:
-            ev.__ray_terminate__.remote(ev._ray_actor_id.id())
+            ev.__ray_terminate__.remote()
 
     def _save(self, checkpoint_dir):
         checkpoint_path = self.saver.save(

python/ray/rllib/es/es.py

@@ -311,7 +311,7 @@ class ESAgent(agent.Agent):
     def _stop(self):
         # workaround for https://github.com/ray-project/ray/issues/1516
         for w in self.workers:
-            w.__ray_terminate__.remote(w._ray_actor_id.id())
+            w.__ray_terminate__.remote()
 
     def _save(self, checkpoint_dir):
         checkpoint_path = os.path.join(

python/ray/rllib/ppo/ppo.py

@@ -269,7 +269,7 @@ class PPOAgent(Agent):
     def _stop(self):
         # workaround for https://github.com/ray-project/ray/issues/1516
         for ev in self.remote_evaluators:
-            ev.__ray_terminate__.remote(ev._ray_actor_id.id())
+            ev.__ray_terminate__.remote()
 
     def _save(self, checkpoint_dir):
         checkpoint_path = self.saver.save(

python/ray/rllib/utils/actors.py

@@ -30,7 +30,7 @@ class TaskPool(object):
 def drop_colocated(actors):
     colocated, non_colocated = split_colocated(actors)
     for a in colocated:
-        a.__ray_terminate__.remote(a._ray_actor_id.id())
+        a.__ray_terminate__.remote()
     return non_colocated