[rllib] Ape-X implementation and DQN refactor to handle replay in policy optimizer (#1604)

* minimal apex checkin

* cleanup dqn options

* actor utils

* Sun Feb 25 17:39:54 PST 2018

* update

* compression refactor

* fix

* add test

* fix models

* Sun Feb 25 21:46:27 PST 2018

* Wed Feb 28 10:26:34 PST 2018

* Wed Feb 28 10:28:09 PST 2018

* Wed Feb 28 10:42:59 PST 2018

* refactor

* Wed Feb 28 11:17:19 PST 2018

* Wed Feb 28 11:42:08 PST 2018

* Wed Feb 28 11:42:13 PST 2018

* Wed Feb 28 11:59:02 PST 2018

* Wed Feb 28 11:59:58 PST 2018

* Wed Feb 28 12:00:08 PST 2018

* Wed Feb 28 12:02:19 PST 2018

* Wed Feb 28 13:44:31 PST 2018

* Wed Feb 28 17:01:20 PST 2018

* Sat Mar  3 14:55:59 PST 2018

* make optimizer construction explicit

* Sat Mar  3 18:23:08 PST 2018

* Sat Mar  3 18:24:28 PST 2018

* Sat Mar  3 18:49:28 PST 2018

* Sat Mar  3 18:50:42 PST 2018

* Sat Mar  3 18:56:10 PST 2018

python/ray/rllib/utils/compression.py

@@ -0,0 +1,32 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import base64
+import pyarrow
+
+try:
+    import snappy
+    SNAPPY_ENABLED = True
+except ImportError:
+    print("WARNING: python-snappy not available, disabling sample compression")
+    SNAPPY_ENABLED = False
+
+
+def pack(data):
+    if SNAPPY_ENABLED:
+        data = snappy.compress(
+            pyarrow.serialize(data).to_buffer().to_pybytes())
+        # TODO(ekl) we shouldn't need to base64 encode this data, but this
+        # seems to not survive a transfer through the object store if we don't.
+        return base64.b64encode(data)
+    else:
+        return data
+
+
+def unpack(data):
+    if SNAPPY_ENABLED:
+        data = base64.b64decode(data)
+        return pyarrow.deserialize(snappy.decompress(data))
+    else:
+        return data