mirror of
https://github.com/wassname/ray.git
synced 2026-06-29 14:06:42 +08:00
Philipp Moritz
c3b39b4d86
* Rebase Ray on top of Plasma in Apache Arrow * add thirdparty building scripts * use rebased arrow * fix * fix build * fix python visibility * comment out C tests for now * fix multithreading * fix * reduce logging * fix plasma manager multithreading * make sure old and new object IDs can coexist peacefully * more rebasing * update * fixes * fix * install pyarrow * install cython * fix * install newer cmake * fix * rebase on top of latest arrow * getting runtest.py run locally (needed to comment out a test for that to work) * work on plasma tests * more fixes * fix local scheduler tests * fix global scheduler test * more fixes * fix python 3 bytes vs string * fix manager tests valgrind * fix documentation building * fix linting * fix c++ linting * fix linting * add tests back in * Install without sudo. * Set PKG_CONFIG_PATH in build.sh so that Ray can find plasma. * Install pkg-config * Link -lpthread, note that find_package(Threads) doesn't seem to work reliably. * Comment in testGPUIDs in runtest.py. * Set PKG_CONFIG_PATH when building pyarrow. * Pull apache/arrow and not pcmoritz/arrow. * Fix installation in docker image. * adapt to changes of the plasma api * Fix installation of pyarrow module. * Fix linting. * Use correct python executable to build pyarrow.
51 lines
1.6 KiB
Python
51 lines
1.6 KiB
Python
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import numpy as np
|
|
import random
|
|
|
|
import pyarrow.plasma as plasma
|
|
|
|
|
|
def random_object_id():
|
|
return plasma.ObjectID(np.random.bytes(20))
|
|
|
|
|
|
def generate_metadata(length):
|
|
metadata_buffer = bytearray(length)
|
|
if length > 0:
|
|
metadata_buffer[0] = random.randint(0, 255)
|
|
metadata_buffer[-1] = random.randint(0, 255)
|
|
for _ in range(100):
|
|
metadata_buffer[random.randint(0, length - 1)] = (
|
|
random.randint(0, 255))
|
|
return metadata_buffer
|
|
|
|
|
|
def write_to_data_buffer(buff, length):
|
|
array = np.frombuffer(buff, dtype="uint8")
|
|
if length > 0:
|
|
array[0] = random.randint(0, 255)
|
|
array[-1] = random.randint(0, 255)
|
|
for _ in range(100):
|
|
array[random.randint(0, length - 1)] = random.randint(0, 255)
|
|
|
|
|
|
def create_object_with_id(client, object_id, data_size, metadata_size,
|
|
seal=True):
|
|
metadata = generate_metadata(metadata_size)
|
|
memory_buffer = client.create(object_id, data_size, metadata)
|
|
write_to_data_buffer(memory_buffer, data_size)
|
|
if seal:
|
|
client.seal(object_id)
|
|
return memory_buffer, metadata
|
|
|
|
|
|
def create_object(client, data_size, metadata_size, seal=True):
|
|
object_id = random_object_id()
|
|
memory_buffer, metadata = create_object_with_id(client, object_id,
|
|
data_size, metadata_size,
|
|
seal=seal)
|
|
return object_id, memory_buffer, metadata
|