diff --git a/python/ray/rllib/agents/trainer_template.py b/python/ray/rllib/agents/trainer_template.py
index ee0b4181c..70bdf254d 100644
--- a/python/ray/rllib/agents/trainer_template.py
+++ b/python/ray/rllib/agents/trainer_template.py
@@ -151,12 +151,12 @@ def build_trainer(name,
 
         def __getstate__(self):
             state = Trainer.__getstate__(self)
-            state.update(self.state)
+            state["trainer_state"] = self.state.copy()
             return state
 
         def __setstate__(self, state):
             Trainer.__setstate__(self, state)
-            self.state = state
+            self.state = state["trainer_state"].copy()
 
     @staticmethod
     def with_updates(**overrides):