mirror of
https://github.com/wassname/ray.git
synced 2026-07-11 00:51:38 +08:00
Lint Python files with Yapf (#1872)
This commit is contained in:
Philipp Moritz
committed by
Robert Nishihara
Robert Nishihara
parent
a3ddde398c
commit
74162d1492
@@ -5,5 +5,6 @@ from __future__ import print_function
|
||||
from ray.plasma.plasma import (start_plasma_store, start_plasma_manager,
|
||||
DEFAULT_PLASMA_STORE_MEMORY)
|
||||
|
||||
__all__ = ["start_plasma_store", "start_plasma_manager",
|
||||
"DEFAULT_PLASMA_STORE_MEMORY"]
|
||||
__all__ = [
|
||||
"start_plasma_store", "start_plasma_manager", "DEFAULT_PLASMA_STORE_MEMORY"
|
||||
]
|
||||
|
||||
+65
-46
@@ -8,13 +8,13 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
__all__ = [
|
||||
"start_plasma_store", "start_plasma_manager", "DEFAULT_PLASMA_STORE_MEMORY"
|
||||
]
|
||||
|
||||
__all__ = ["start_plasma_store", "start_plasma_manager",
|
||||
"DEFAULT_PLASMA_STORE_MEMORY"]
|
||||
PLASMA_WAIT_TIMEOUT = 2**30
|
||||
|
||||
PLASMA_WAIT_TIMEOUT = 2 ** 30
|
||||
|
||||
DEFAULT_PLASMA_STORE_MEMORY = 10 ** 9
|
||||
DEFAULT_PLASMA_STORE_MEMORY = 10**9
|
||||
|
||||
|
||||
def random_name():
|
||||
@@ -22,9 +22,12 @@ def random_name():
|
||||
|
||||
|
||||
def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
|
||||
use_valgrind=False, use_profiler=False,
|
||||
stdout_file=None, stderr_file=None,
|
||||
plasma_directory=None, huge_pages=False):
|
||||
use_valgrind=False,
|
||||
use_profiler=False,
|
||||
stdout_file=None,
|
||||
stderr_file=None,
|
||||
plasma_directory=None,
|
||||
huge_pages=False):
|
||||
"""Start a plasma store process.
|
||||
|
||||
Args:
|
||||
@@ -48,8 +51,8 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
|
||||
if use_valgrind and use_profiler:
|
||||
raise Exception("Cannot use valgrind and profiler at the same time.")
|
||||
|
||||
if huge_pages and not (sys.platform == "linux" or
|
||||
sys.platform == "linux2"):
|
||||
if huge_pages and not (sys.platform == "linux"
|
||||
or sys.platform == "linux2"):
|
||||
raise Exception("The huge_pages argument is only supported on "
|
||||
"Linux.")
|
||||
|
||||
@@ -57,29 +60,33 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
|
||||
raise Exception("If huge_pages is True, then the "
|
||||
"plasma_directory argument must be provided.")
|
||||
|
||||
plasma_store_executable = os.path.join(os.path.abspath(
|
||||
os.path.dirname(__file__)),
|
||||
plasma_store_executable = os.path.join(
|
||||
os.path.abspath(os.path.dirname(__file__)),
|
||||
"../core/src/plasma/plasma_store")
|
||||
plasma_store_name = "/tmp/plasma_store{}".format(random_name())
|
||||
command = [plasma_store_executable,
|
||||
"-s", plasma_store_name,
|
||||
"-m", str(plasma_store_memory)]
|
||||
command = [
|
||||
plasma_store_executable, "-s", plasma_store_name, "-m",
|
||||
str(plasma_store_memory)
|
||||
]
|
||||
if plasma_directory is not None:
|
||||
command += ["-d", plasma_directory]
|
||||
if huge_pages:
|
||||
command += ["-h"]
|
||||
if use_valgrind:
|
||||
pid = subprocess.Popen(["valgrind",
|
||||
"--track-origins=yes",
|
||||
"--leak-check=full",
|
||||
"--show-leak-kinds=all",
|
||||
"--leak-check-heuristics=stdstring",
|
||||
"--error-exitcode=1"] + command,
|
||||
stdout=stdout_file, stderr=stderr_file)
|
||||
pid = subprocess.Popen(
|
||||
[
|
||||
"valgrind", "--track-origins=yes", "--leak-check=full",
|
||||
"--show-leak-kinds=all", "--leak-check-heuristics=stdstring",
|
||||
"--error-exitcode=1"
|
||||
] + command,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
time.sleep(1.0)
|
||||
elif use_profiler:
|
||||
pid = subprocess.Popen(["valgrind", "--tool=callgrind"] + command,
|
||||
stdout=stdout_file, stderr=stderr_file)
|
||||
pid = subprocess.Popen(
|
||||
["valgrind", "--tool=callgrind"] + command,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
time.sleep(1.0)
|
||||
else:
|
||||
pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
@@ -91,10 +98,14 @@ def new_port():
|
||||
return random.randint(10000, 65535)
|
||||
|
||||
|
||||
def start_plasma_manager(store_name, redis_address,
|
||||
node_ip_address="127.0.0.1", plasma_manager_port=None,
|
||||
num_retries=20, use_valgrind=False,
|
||||
run_profiler=False, stdout_file=None,
|
||||
def start_plasma_manager(store_name,
|
||||
redis_address,
|
||||
node_ip_address="127.0.0.1",
|
||||
plasma_manager_port=None,
|
||||
num_retries=20,
|
||||
use_valgrind=False,
|
||||
run_profiler=False,
|
||||
stdout_file=None,
|
||||
stderr_file=None):
|
||||
"""Start a plasma manager and return the ports it listens on.
|
||||
|
||||
@@ -133,27 +144,35 @@ def start_plasma_manager(store_name, redis_address,
|
||||
while counter < num_retries:
|
||||
if counter > 0:
|
||||
print("Plasma manager failed to start, retrying now.")
|
||||
command = [plasma_manager_executable,
|
||||
"-s", store_name,
|
||||
"-m", plasma_manager_name,
|
||||
"-h", node_ip_address,
|
||||
"-p", str(plasma_manager_port),
|
||||
"-r", redis_address,
|
||||
]
|
||||
command = [
|
||||
plasma_manager_executable,
|
||||
"-s",
|
||||
store_name,
|
||||
"-m",
|
||||
plasma_manager_name,
|
||||
"-h",
|
||||
node_ip_address,
|
||||
"-p",
|
||||
str(plasma_manager_port),
|
||||
"-r",
|
||||
redis_address,
|
||||
]
|
||||
if use_valgrind:
|
||||
process = subprocess.Popen(["valgrind",
|
||||
"--track-origins=yes",
|
||||
"--leak-check=full",
|
||||
"--show-leak-kinds=all",
|
||||
"--error-exitcode=1"] + command,
|
||||
stdout=stdout_file, stderr=stderr_file)
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
"valgrind", "--track-origins=yes", "--leak-check=full",
|
||||
"--show-leak-kinds=all", "--error-exitcode=1"
|
||||
] + command,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
elif run_profiler:
|
||||
process = subprocess.Popen((["valgrind", "--tool=callgrind"] +
|
||||
command),
|
||||
stdout=stdout_file, stderr=stderr_file)
|
||||
process = subprocess.Popen(
|
||||
(["valgrind", "--tool=callgrind"] + command),
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
else:
|
||||
process = subprocess.Popen(command, stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
process = subprocess.Popen(
|
||||
command, stdout=stdout_file, stderr=stderr_file)
|
||||
# This sleep is critical. If the plasma_manager fails to start because
|
||||
# the port is already in use, then we need it to fail within 0.1
|
||||
# seconds.
|
||||
|
||||
+143
-107
@@ -16,8 +16,8 @@ import unittest
|
||||
# The ray import must come before the pyarrow import because ray modifies the
|
||||
# python path so that the right version of pyarrow is found.
|
||||
import ray
|
||||
from ray.plasma.utils import (random_object_id,
|
||||
create_object_with_id, create_object)
|
||||
from ray.plasma.utils import (random_object_id, create_object_with_id,
|
||||
create_object)
|
||||
from ray import services
|
||||
import pyarrow as pa
|
||||
import pyarrow.plasma as plasma
|
||||
@@ -30,8 +30,12 @@ def random_name():
|
||||
return str(random.randint(0, 99999999))
|
||||
|
||||
|
||||
def assert_get_object_equal(unit_test, client1, client2, object_id,
|
||||
memory_buffer=None, metadata=None):
|
||||
def assert_get_object_equal(unit_test,
|
||||
client1,
|
||||
client2,
|
||||
object_id,
|
||||
memory_buffer=None,
|
||||
metadata=None):
|
||||
client1_buff = client1.get_buffers([object_id])[0]
|
||||
client2_buff = client2.get_buffers([object_id])[0]
|
||||
client1_metadata = client1.get_metadata([object_id])[0]
|
||||
@@ -39,27 +43,33 @@ def assert_get_object_equal(unit_test, client1, client2, object_id,
|
||||
unit_test.assertEqual(len(client1_buff), len(client2_buff))
|
||||
unit_test.assertEqual(len(client1_metadata), len(client2_metadata))
|
||||
# Check that the buffers from the two clients are the same.
|
||||
assert_equal(np.frombuffer(client1_buff, dtype="uint8"),
|
||||
np.frombuffer(client2_buff, dtype="uint8"))
|
||||
assert_equal(
|
||||
np.frombuffer(client1_buff, dtype="uint8"),
|
||||
np.frombuffer(client2_buff, dtype="uint8"))
|
||||
# Check that the metadata buffers from the two clients are the same.
|
||||
assert_equal(np.frombuffer(client1_metadata, dtype="uint8"),
|
||||
np.frombuffer(client2_metadata, dtype="uint8"))
|
||||
assert_equal(
|
||||
np.frombuffer(client1_metadata, dtype="uint8"),
|
||||
np.frombuffer(client2_metadata, dtype="uint8"))
|
||||
# If a reference buffer was provided, check that it is the same as well.
|
||||
if memory_buffer is not None:
|
||||
assert_equal(np.frombuffer(memory_buffer, dtype="uint8"),
|
||||
np.frombuffer(client1_buff, dtype="uint8"))
|
||||
assert_equal(
|
||||
np.frombuffer(memory_buffer, dtype="uint8"),
|
||||
np.frombuffer(client1_buff, dtype="uint8"))
|
||||
# If reference metadata was provided, check that it is the same as well.
|
||||
if metadata is not None:
|
||||
assert_equal(np.frombuffer(metadata, dtype="uint8"),
|
||||
np.frombuffer(client1_metadata, dtype="uint8"))
|
||||
assert_equal(
|
||||
np.frombuffer(metadata, dtype="uint8"),
|
||||
np.frombuffer(client1_metadata, dtype="uint8"))
|
||||
|
||||
|
||||
DEFAULT_PLASMA_STORE_MEMORY = 10 ** 9
|
||||
DEFAULT_PLASMA_STORE_MEMORY = 10**9
|
||||
|
||||
|
||||
def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
|
||||
use_valgrind=False, use_profiler=False,
|
||||
stdout_file=None, stderr_file=None):
|
||||
use_valgrind=False,
|
||||
use_profiler=False,
|
||||
stdout_file=None,
|
||||
stderr_file=None):
|
||||
"""Start a plasma store process.
|
||||
Args:
|
||||
use_valgrind (bool): True if the plasma store should be started inside
|
||||
@@ -78,21 +88,25 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
|
||||
raise Exception("Cannot use valgrind and profiler at the same time.")
|
||||
plasma_store_executable = os.path.join(pa.__path__[0], "plasma_store")
|
||||
plasma_store_name = "/tmp/plasma_store{}".format(random_name())
|
||||
command = [plasma_store_executable,
|
||||
"-s", plasma_store_name,
|
||||
"-m", str(plasma_store_memory)]
|
||||
command = [
|
||||
plasma_store_executable, "-s", plasma_store_name, "-m",
|
||||
str(plasma_store_memory)
|
||||
]
|
||||
if use_valgrind:
|
||||
pid = subprocess.Popen(["valgrind",
|
||||
"--track-origins=yes",
|
||||
"--leak-check=full",
|
||||
"--show-leak-kinds=all",
|
||||
"--leak-check-heuristics=stdstring",
|
||||
"--error-exitcode=1"] + command,
|
||||
stdout=stdout_file, stderr=stderr_file)
|
||||
pid = subprocess.Popen(
|
||||
[
|
||||
"valgrind", "--track-origins=yes", "--leak-check=full",
|
||||
"--show-leak-kinds=all", "--leak-check-heuristics=stdstring",
|
||||
"--error-exitcode=1"
|
||||
] + command,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
time.sleep(1.0)
|
||||
elif use_profiler:
|
||||
pid = subprocess.Popen(["valgrind", "--tool=callgrind"] + command,
|
||||
stdout=stdout_file, stderr=stderr_file)
|
||||
pid = subprocess.Popen(
|
||||
["valgrind", "--tool=callgrind"] + command,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file)
|
||||
time.sleep(1.0)
|
||||
else:
|
||||
pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
|
||||
@@ -104,13 +118,10 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
|
||||
|
||||
|
||||
class TestPlasmaManager(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# Start two PlasmaStores.
|
||||
store_name1, self.p2 = start_plasma_store(
|
||||
use_valgrind=USE_VALGRIND)
|
||||
store_name2, self.p3 = start_plasma_store(
|
||||
use_valgrind=USE_VALGRIND)
|
||||
store_name1, self.p2 = start_plasma_store(use_valgrind=USE_VALGRIND)
|
||||
store_name2, self.p3 = start_plasma_store(use_valgrind=USE_VALGRIND)
|
||||
# Start a Redis server.
|
||||
redis_address, _ = services.start_redis("127.0.0.1")
|
||||
# Start two PlasmaManagers.
|
||||
@@ -152,9 +163,8 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
def test_fetch(self):
|
||||
for _ in range(10):
|
||||
# Create an object.
|
||||
object_id1, memory_buffer1, metadata1 = create_object(self.client1,
|
||||
2000,
|
||||
2000)
|
||||
object_id1, memory_buffer1, metadata1 = create_object(
|
||||
self.client1, 2000, 2000)
|
||||
self.client1.fetch([object_id1])
|
||||
self.assertEqual(self.client1.contains(object_id1), True)
|
||||
self.assertEqual(self.client2.contains(object_id1), False)
|
||||
@@ -164,18 +174,20 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
while not self.client2.contains(object_id1):
|
||||
self.client2.fetch([object_id1])
|
||||
# Compare the two buffers.
|
||||
assert_get_object_equal(self, self.client1, self.client2,
|
||||
object_id1,
|
||||
memory_buffer=memory_buffer1,
|
||||
metadata=metadata1)
|
||||
assert_get_object_equal(
|
||||
self,
|
||||
self.client1,
|
||||
self.client2,
|
||||
object_id1,
|
||||
memory_buffer=memory_buffer1,
|
||||
metadata=metadata1)
|
||||
|
||||
# Test that we can call fetch on object IDs that don't exist yet.
|
||||
object_id2 = random_object_id()
|
||||
self.client1.fetch([object_id2])
|
||||
self.assertEqual(self.client1.contains(object_id2), False)
|
||||
memory_buffer2, metadata2 = create_object_with_id(self.client2,
|
||||
object_id2,
|
||||
2000, 2000)
|
||||
memory_buffer2, metadata2 = create_object_with_id(
|
||||
self.client2, object_id2, 2000, 2000)
|
||||
# # Check that the object has been fetched.
|
||||
# self.assertEqual(self.client1.contains(object_id2), True)
|
||||
# Compare the two buffers.
|
||||
@@ -190,68 +202,88 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
for _ in range(10):
|
||||
self.client1.fetch([object_id3])
|
||||
self.client2.fetch([object_id3])
|
||||
memory_buffer3, metadata3 = create_object_with_id(self.client1,
|
||||
object_id3,
|
||||
2000, 2000)
|
||||
memory_buffer3, metadata3 = create_object_with_id(
|
||||
self.client1, object_id3, 2000, 2000)
|
||||
for _ in range(10):
|
||||
self.client1.fetch([object_id3])
|
||||
self.client2.fetch([object_id3])
|
||||
# TODO(rkn): Right now we must wait for the object table to be updated.
|
||||
while not self.client2.contains(object_id3):
|
||||
self.client2.fetch([object_id3])
|
||||
assert_get_object_equal(self, self.client1, self.client2, object_id3,
|
||||
memory_buffer=memory_buffer3,
|
||||
metadata=metadata3)
|
||||
assert_get_object_equal(
|
||||
self,
|
||||
self.client1,
|
||||
self.client2,
|
||||
object_id3,
|
||||
memory_buffer=memory_buffer3,
|
||||
metadata=metadata3)
|
||||
|
||||
def test_fetch_multiple(self):
|
||||
for _ in range(20):
|
||||
# Create two objects and a third fake one that doesn't exist.
|
||||
object_id1, memory_buffer1, metadata1 = create_object(self.client1,
|
||||
2000,
|
||||
2000)
|
||||
object_id1, memory_buffer1, metadata1 = create_object(
|
||||
self.client1, 2000, 2000)
|
||||
missing_object_id = random_object_id()
|
||||
object_id2, memory_buffer2, metadata2 = create_object(self.client1,
|
||||
2000,
|
||||
2000)
|
||||
object_id2, memory_buffer2, metadata2 = create_object(
|
||||
self.client1, 2000, 2000)
|
||||
object_ids = [object_id1, missing_object_id, object_id2]
|
||||
# Fetch the objects from the other plasma store. The second object
|
||||
# ID should timeout since it does not exist.
|
||||
# TODO(rkn): Right now we must wait for the object table to be
|
||||
# updated.
|
||||
while ((not self.client2.contains(object_id1)) or
|
||||
(not self.client2.contains(object_id2))):
|
||||
while ((not self.client2.contains(object_id1))
|
||||
or (not self.client2.contains(object_id2))):
|
||||
self.client2.fetch(object_ids)
|
||||
# Compare the buffers of the objects that do exist.
|
||||
assert_get_object_equal(self, self.client1, self.client2,
|
||||
object_id1, memory_buffer=memory_buffer1,
|
||||
metadata=metadata1)
|
||||
assert_get_object_equal(self, self.client1, self.client2,
|
||||
object_id2, memory_buffer=memory_buffer2,
|
||||
metadata=metadata2)
|
||||
assert_get_object_equal(
|
||||
self,
|
||||
self.client1,
|
||||
self.client2,
|
||||
object_id1,
|
||||
memory_buffer=memory_buffer1,
|
||||
metadata=metadata1)
|
||||
assert_get_object_equal(
|
||||
self,
|
||||
self.client1,
|
||||
self.client2,
|
||||
object_id2,
|
||||
memory_buffer=memory_buffer2,
|
||||
metadata=metadata2)
|
||||
# Fetch in the other direction. The fake object still does not
|
||||
# exist.
|
||||
self.client1.fetch(object_ids)
|
||||
assert_get_object_equal(self, self.client2, self.client1,
|
||||
object_id1, memory_buffer=memory_buffer1,
|
||||
metadata=metadata1)
|
||||
assert_get_object_equal(self, self.client2, self.client1,
|
||||
object_id2, memory_buffer=memory_buffer2,
|
||||
metadata=metadata2)
|
||||
assert_get_object_equal(
|
||||
self,
|
||||
self.client2,
|
||||
self.client1,
|
||||
object_id1,
|
||||
memory_buffer=memory_buffer1,
|
||||
metadata=metadata1)
|
||||
assert_get_object_equal(
|
||||
self,
|
||||
self.client2,
|
||||
self.client1,
|
||||
object_id2,
|
||||
memory_buffer=memory_buffer2,
|
||||
metadata=metadata2)
|
||||
|
||||
# Check that we can call fetch with duplicated object IDs.
|
||||
object_id3 = random_object_id()
|
||||
self.client1.fetch([object_id3, object_id3])
|
||||
object_id4, memory_buffer4, metadata4 = create_object(self.client1,
|
||||
2000,
|
||||
2000)
|
||||
object_id4, memory_buffer4, metadata4 = create_object(
|
||||
self.client1, 2000, 2000)
|
||||
time.sleep(0.1)
|
||||
# TODO(rkn): Right now we must wait for the object table to be updated.
|
||||
while not self.client2.contains(object_id4):
|
||||
self.client2.fetch([object_id3, object_id3, object_id4,
|
||||
object_id4])
|
||||
assert_get_object_equal(self, self.client2, self.client1, object_id4,
|
||||
memory_buffer=memory_buffer4,
|
||||
metadata=metadata4)
|
||||
self.client2.fetch(
|
||||
[object_id3, object_id3, object_id4, object_id4])
|
||||
assert_get_object_equal(
|
||||
self,
|
||||
self.client2,
|
||||
self.client1,
|
||||
object_id4,
|
||||
memory_buffer=memory_buffer4,
|
||||
metadata=metadata4)
|
||||
|
||||
def test_wait(self):
|
||||
# Test timeout.
|
||||
@@ -263,8 +295,8 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
obj_id1 = random_object_id()
|
||||
self.client1.create(obj_id1, 1000)
|
||||
self.client1.seal(obj_id1)
|
||||
ready, waiting = self.client1.wait([obj_id1], timeout=100,
|
||||
num_returns=1)
|
||||
ready, waiting = self.client1.wait(
|
||||
[obj_id1], timeout=100, num_returns=1)
|
||||
self.assertEqual(set(ready), set([obj_id1]))
|
||||
self.assertEqual(waiting, [])
|
||||
|
||||
@@ -273,8 +305,8 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
obj_id2 = random_object_id()
|
||||
self.client1.create(obj_id2, 1000)
|
||||
# Don't seal.
|
||||
ready, waiting = self.client1.wait([obj_id2, obj_id1], timeout=100,
|
||||
num_returns=1)
|
||||
ready, waiting = self.client1.wait(
|
||||
[obj_id2, obj_id1], timeout=100, num_returns=1)
|
||||
self.assertEqual(set(ready), set([obj_id1]))
|
||||
self.assertEqual(set(waiting), set([obj_id2]))
|
||||
|
||||
@@ -287,8 +319,8 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
|
||||
t = threading.Timer(0.1, finish)
|
||||
t.start()
|
||||
ready, waiting = self.client1.wait([obj_id3, obj_id2, obj_id1],
|
||||
timeout=1000, num_returns=2)
|
||||
ready, waiting = self.client1.wait(
|
||||
[obj_id3, obj_id2, obj_id1], timeout=1000, num_returns=2)
|
||||
self.assertEqual(set(ready), set([obj_id1, obj_id3]))
|
||||
self.assertEqual(set(waiting), set([obj_id2]))
|
||||
|
||||
@@ -319,26 +351,26 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
waiting = object_ids
|
||||
retrieved = []
|
||||
for i in range(1, n + 1):
|
||||
ready, waiting = self.client1.wait(waiting, timeout=1000,
|
||||
num_returns=i)
|
||||
ready, waiting = self.client1.wait(
|
||||
waiting, timeout=1000, num_returns=i)
|
||||
self.assertEqual(len(ready), i)
|
||||
retrieved += ready
|
||||
self.assertEqual(set(retrieved), set(object_ids))
|
||||
ready, waiting = self.client1.wait(object_ids, timeout=1000,
|
||||
num_returns=len(object_ids))
|
||||
ready, waiting = self.client1.wait(
|
||||
object_ids, timeout=1000, num_returns=len(object_ids))
|
||||
self.assertEqual(set(ready), set(object_ids))
|
||||
self.assertEqual(waiting, [])
|
||||
# Try waiting for all of the object IDs on the second client.
|
||||
waiting = object_ids
|
||||
retrieved = []
|
||||
for i in range(1, n + 1):
|
||||
ready, waiting = self.client2.wait(waiting, timeout=1000,
|
||||
num_returns=i)
|
||||
ready, waiting = self.client2.wait(
|
||||
waiting, timeout=1000, num_returns=i)
|
||||
self.assertEqual(len(ready), i)
|
||||
retrieved += ready
|
||||
self.assertEqual(set(retrieved), set(object_ids))
|
||||
ready, waiting = self.client2.wait(object_ids, timeout=1000,
|
||||
num_returns=len(object_ids))
|
||||
ready, waiting = self.client2.wait(
|
||||
object_ids, timeout=1000, num_returns=len(object_ids))
|
||||
self.assertEqual(set(ready), set(object_ids))
|
||||
self.assertEqual(waiting, [])
|
||||
|
||||
@@ -363,9 +395,8 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
num_attempts = 100
|
||||
for _ in range(100):
|
||||
# Create an object.
|
||||
object_id1, memory_buffer1, metadata1 = create_object(self.client1,
|
||||
2000,
|
||||
2000)
|
||||
object_id1, memory_buffer1, metadata1 = create_object(
|
||||
self.client1, 2000, 2000)
|
||||
# Transfer the buffer to the the other Plasma store. There is a
|
||||
# race condition on the create and transfer of the object, so keep
|
||||
# trying until the object appears on the second Plasma store.
|
||||
@@ -379,9 +410,13 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
del buff
|
||||
|
||||
# Compare the two buffers.
|
||||
assert_get_object_equal(self, self.client1, self.client2,
|
||||
object_id1, memory_buffer=memory_buffer1,
|
||||
metadata=metadata1)
|
||||
assert_get_object_equal(
|
||||
self,
|
||||
self.client1,
|
||||
self.client2,
|
||||
object_id1,
|
||||
memory_buffer=memory_buffer1,
|
||||
metadata=metadata1)
|
||||
# # Transfer the buffer again.
|
||||
# self.client1.transfer("127.0.0.1", self.port2, object_id1)
|
||||
# # Compare the two buffers.
|
||||
@@ -391,8 +426,8 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
# metadata=metadata1)
|
||||
|
||||
# Create an object.
|
||||
object_id2, memory_buffer2, metadata2 = create_object(self.client2,
|
||||
20000, 20000)
|
||||
object_id2, memory_buffer2, metadata2 = create_object(
|
||||
self.client2, 20000, 20000)
|
||||
# Transfer the buffer to the the other Plasma store. There is a
|
||||
# race condition on the create and transfer of the object, so keep
|
||||
# trying until the object appears on the second Plasma store.
|
||||
@@ -406,9 +441,13 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
del buff
|
||||
|
||||
# Compare the two buffers.
|
||||
assert_get_object_equal(self, self.client1, self.client2,
|
||||
object_id2, memory_buffer=memory_buffer2,
|
||||
metadata=metadata2)
|
||||
assert_get_object_equal(
|
||||
self,
|
||||
self.client1,
|
||||
self.client2,
|
||||
object_id2,
|
||||
memory_buffer=memory_buffer2,
|
||||
metadata=metadata2)
|
||||
|
||||
def test_illegal_functionality(self):
|
||||
# Create an object id string.
|
||||
@@ -437,7 +476,6 @@ class TestPlasmaManager(unittest.TestCase):
|
||||
|
||||
|
||||
class TestPlasmaManagerRecovery(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# Start a Plasma store.
|
||||
self.store_name, self.p2 = start_plasma_store(
|
||||
@@ -446,9 +484,7 @@ class TestPlasmaManagerRecovery(unittest.TestCase):
|
||||
self.redis_address, _ = services.start_redis("127.0.0.1")
|
||||
# Start a PlasmaManagers.
|
||||
manager_name, self.p3, self.port1 = ray.plasma.start_plasma_manager(
|
||||
self.store_name,
|
||||
self.redis_address,
|
||||
use_valgrind=USE_VALGRIND)
|
||||
self.store_name, self.redis_address, use_valgrind=USE_VALGRIND)
|
||||
# Connect a PlasmaClient.
|
||||
self.client = plasma.connect(self.store_name, manager_name, 64)
|
||||
|
||||
@@ -501,8 +537,8 @@ class TestPlasmaManagerRecovery(unittest.TestCase):
|
||||
client2 = plasma.connect(self.store_name, manager_name, 64)
|
||||
ready, waiting = [], object_ids
|
||||
while True:
|
||||
ready, waiting = client2.wait(object_ids, num_returns=num_objects,
|
||||
timeout=0)
|
||||
ready, waiting = client2.wait(
|
||||
object_ids, num_returns=num_objects, timeout=0)
|
||||
if len(ready) == len(object_ids):
|
||||
break
|
||||
|
||||
|
||||
@@ -18,8 +18,8 @@ def generate_metadata(length):
|
||||
metadata_buffer[0] = random.randint(0, 255)
|
||||
metadata_buffer[-1] = random.randint(0, 255)
|
||||
for _ in range(100):
|
||||
metadata_buffer[random.randint(0, length - 1)] = (
|
||||
random.randint(0, 255))
|
||||
metadata_buffer[random.randint(0, length - 1)] = (random.randint(
|
||||
0, 255))
|
||||
return metadata_buffer
|
||||
|
||||
|
||||
@@ -32,7 +32,10 @@ def write_to_data_buffer(buff, length):
|
||||
array[random.randint(0, length - 1)] = random.randint(0, 255)
|
||||
|
||||
|
||||
def create_object_with_id(client, object_id, data_size, metadata_size,
|
||||
def create_object_with_id(client,
|
||||
object_id,
|
||||
data_size,
|
||||
metadata_size,
|
||||
seal=True):
|
||||
metadata = generate_metadata(metadata_size)
|
||||
memory_buffer = client.create(object_id, data_size, metadata)
|
||||
@@ -44,7 +47,6 @@ def create_object_with_id(client, object_id, data_size, metadata_size,
|
||||
|
||||
def create_object(client, data_size, metadata_size, seal=True):
|
||||
object_id = random_object_id()
|
||||
memory_buffer, metadata = create_object_with_id(client, object_id,
|
||||
data_size, metadata_size,
|
||||
seal=seal)
|
||||
memory_buffer, metadata = create_object_with_id(
|
||||
client, object_id, data_size, metadata_size, seal=seal)
|
||||
return object_id, memory_buffer, metadata
|
||||
|
||||
Reference in New Issue
Block a user