Fix two types of eviction hangs (#5225)

This commit is contained in:
Eric Liang
2019-07-23 21:20:17 -07:00
committed by GitHub
parent 97c43284a6
commit 5b76238bce
9 changed files with 126 additions and 45 deletions
@@ -0,0 +1,40 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
import unittest
import ray
class TestUnreconstructableErrors(unittest.TestCase):
def setUp(self):
ray.init(object_store_memory=10000000, redis_max_memory=10000000)
def tearDown(self):
ray.shutdown()
def testDriverPutEvictedCannotReconstruct(self):
x_id = ray.put(np.zeros(1 * 1024 * 1024))
ray.get(x_id)
for _ in range(10):
ray.put(np.zeros(1 * 1024 * 1024))
self.assertRaises(ray.exceptions.UnreconstructableError,
lambda: ray.get(x_id))
def testLineageEvictedReconstructionFails(self):
@ray.remote
def f(data):
return 0
x_id = f.remote(None)
ray.get(x_id)
for _ in range(400):
ray.get([f.remote(np.zeros(10000)) for _ in range(50)])
self.assertRaises(ray.exceptions.UnreconstructableError,
lambda: ray.get(x_id))
if __name__ == "__main__":
unittest.main(verbosity=2)