wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-07 17:03:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Avoid segfaults in arrow if data is too large (#287)

Browse Source 
* arrow limits

* more logging

* set the right limit

* update

* simplify

* fix

* account for subsequences

* fixes and deactivate arrow limit tests in travis

* fixes

* Minor formatting.

* Add a couple more tests.
This commit is contained in:
Philipp Moritz
2017-02-16 15:16:20 -08:00
committed by Robert Nishihara
parent 88a5b4e77b
commit dd7e8d9105
3 changed files with 81 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/numbuf/python/test/runtest.py
+39
View File
@@ -6,6 +6,7 @@ import unittest
import numbuf
import numpy as np
from numpy.testing import assert_equal
import os
import sys
TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43, "hello world",
@@ -126,5 +127,43 @@ class SerializationTests(unittest.TestCase):
with self.assertRaises(ValueError):
result[0][0] = 1
def testArrowLimits(self):
# Test that objects that are too large for Arrow throw a Python exception.
# These tests give out of memory errors on Travis and need to be run on a
# machine with lots of RAM.
if os.getenv("TRAVIS") is None:
l = 2 ** 29 * [1.0]
with self.assertRaises(numbuf.numbuf_error):
self.roundTripTest(l)
self.roundTripTest([l])
del l
l = 2 ** 29 * ["s"]
with self.assertRaises(numbuf.numbuf_error):
self.roundTripTest(l)
self.roundTripTest([l])
del l
l = 2 ** 29 * [["1"], 2, 3, [{"s": 4}]]
with self.assertRaises(numbuf.numbuf_error):
self.roundTripTest(l)
self.roundTripTest([l])
del l
with self.assertRaises(numbuf.numbuf_error):
l = 2 ** 29 * [{"s": 1}] + 2 ** 29 * [1.0]
self.roundTripTest(l)
self.roundTripTest([l])
del l
with self.assertRaises(numbuf.numbuf_error):
l = np.zeros(2 ** 25)
self.roundTripTest([l])
del l
with self.assertRaises(numbuf.numbuf_error):
l = [np.zeros(2 ** 18) for _ in range(2 ** 7)]
self.roundTripTest(l)
self.roundTripTest([l])
del l
else:
print("Not running testArrowLimits on Travis because of the test's "
"memory requirements.")
if __name__ == "__main__":
unittest.main(verbosity=2)