mirror of
https://github.com/wassname/ray.git
synced 2026-06-29 21:08:50 +08:00
Eric Liang
ecb811c26e
* minimal apex checkin * cleanup dqn options * actor utils * Sun Feb 25 17:39:54 PST 2018 * update * compression refactor * fix * add test * fix models * Sun Feb 25 21:46:27 PST 2018 * Wed Feb 28 10:26:34 PST 2018 * Wed Feb 28 10:28:09 PST 2018 * Wed Feb 28 10:42:59 PST 2018 * refactor * Wed Feb 28 11:17:19 PST 2018 * Wed Feb 28 11:42:08 PST 2018 * Wed Feb 28 11:42:13 PST 2018 * Wed Feb 28 11:59:02 PST 2018 * Wed Feb 28 11:59:58 PST 2018 * Wed Feb 28 12:00:08 PST 2018 * Wed Feb 28 12:02:19 PST 2018 * Wed Feb 28 13:44:31 PST 2018 * Wed Feb 28 17:01:20 PST 2018 * Sat Mar 3 14:55:59 PST 2018 * make optimizer construction explicit * Sat Mar 3 18:23:08 PST 2018 * Sat Mar 3 18:24:28 PST 2018 * Sat Mar 3 18:49:28 PST 2018 * Sat Mar 3 18:50:42 PST 2018 * Sat Mar 3 18:56:10 PST 2018
33 lines
845 B
Python
33 lines
845 B
Python
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import base64
|
|
import pyarrow
|
|
|
|
try:
|
|
import snappy
|
|
SNAPPY_ENABLED = True
|
|
except ImportError:
|
|
print("WARNING: python-snappy not available, disabling sample compression")
|
|
SNAPPY_ENABLED = False
|
|
|
|
|
|
def pack(data):
|
|
if SNAPPY_ENABLED:
|
|
data = snappy.compress(
|
|
pyarrow.serialize(data).to_buffer().to_pybytes())
|
|
# TODO(ekl) we shouldn't need to base64 encode this data, but this
|
|
# seems to not survive a transfer through the object store if we don't.
|
|
return base64.b64encode(data)
|
|
else:
|
|
return data
|
|
|
|
|
|
def unpack(data):
|
|
if SNAPPY_ENABLED:
|
|
data = base64.b64decode(data)
|
|
return pyarrow.deserialize(snappy.decompress(data))
|
|
else:
|
|
return data
|