wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-12 12:51:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Make the Plasma store ready for Arrow integration (#579)

Browse Source 
* port plasma to arrow

* fixes

* refactor plasma client

* more modernization

* fix plasma manager tests

* everything compiles

* fix plasma client tests

* update plasma serialization tests

* fix plasma manager tests

* fix bug

* updates

* fix bug

* fix tests

* fix rebase

* address comments

* fix travis valgrind build

* fix linting

* fix include order again

* fix linting

* address comments
This commit is contained in:
Philipp Moritz
2017-05-31 16:24:23 -07:00
committed by Robert Nishihara
parent 609b5c1a4c
commit b94b4a35e0
40 changed files with 4097 additions and 2070 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plasma/test/client_tests.cc
+111 -91
View File
@@ -4,6 +4,7 @@
#include <unistd.h>
#include <sys/time.h>
#include "plasma_common.h"
#include "plasma.h"
#include "plasma_protocol.h"
#include "plasma_client.h"
@@ -11,14 +12,17 @@
SUITE(plasma_client_tests);
TEST plasma_status_tests(void) {
PlasmaConnection *plasma_conn1 = plasma_connect(
"/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY);
PlasmaConnection *plasma_conn2 = plasma_connect(
"/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY);
ObjectID oid1 = globally_unique_id();
PlasmaClient client1;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
PLASMA_DEFAULT_RELEASE_DELAY));
PlasmaClient client2;
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
PLASMA_DEFAULT_RELEASE_DELAY));
ObjectID oid1 = ObjectID::from_random();
/* Test for object non-existence. */
int status = plasma_status(plasma_conn1, oid1);
int status;
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT(status == ObjectStatus_Nonexistent);
/* Test for the object being in local Plasma store. */
@@ -27,36 +31,39 @@ TEST plasma_status_tests(void) {
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
uint8_t *data;
plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data);
plasma_seal(plasma_conn1, oid1);
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
ARROW_CHECK_OK(client1.Seal(oid1));
/* Sleep to avoid race condition of Plasma Manager waiting for notification.
*/
sleep(1);
status = plasma_status(plasma_conn1, oid1);
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT(status == ObjectStatus_Local);
/* Test for object being remote. */
status = plasma_status(plasma_conn2, oid1);
ARROW_CHECK_OK(client2.Info(oid1, &status));
ASSERT(status == ObjectStatus_Remote);
plasma_disconnect(plasma_conn1);
plasma_disconnect(plasma_conn2);
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
TEST plasma_fetch_tests(void) {
PlasmaConnection *plasma_conn1 = plasma_connect(
"/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY);
PlasmaConnection *plasma_conn2 = plasma_connect(
"/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY);
ObjectID oid1 = globally_unique_id();
PlasmaClient client1;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
PLASMA_DEFAULT_RELEASE_DELAY));
PlasmaClient client2;
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
PLASMA_DEFAULT_RELEASE_DELAY));
ObjectID oid1 = ObjectID::from_random();
/* Test for object non-existence. */
int status;
/* No object in the system */
status = plasma_status(plasma_conn1, oid1);
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT(status == ObjectStatus_Nonexistent);
/* Test for the object being in local Plasma store. */
@@ -65,37 +72,38 @@ TEST plasma_fetch_tests(void) {
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
uint8_t *data;
plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data);
plasma_seal(plasma_conn1, oid1);
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
ARROW_CHECK_OK(client1.Seal(oid1));
/* Object with ID oid1 has been just inserted. On the next fetch we might
* either find the object or not, depending on whether the Plasma Manager has
* received the notification from the Plasma Store or not. */
ObjectID oid_array1[1] = {oid1};
plasma_fetch(plasma_conn1, 1, oid_array1);
status = plasma_status(plasma_conn1, oid1);
ARROW_CHECK_OK(client1.Fetch(1, oid_array1));
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT((status == ObjectStatus_Local) ||
(status == ObjectStatus_Nonexistent));
/* Sleep to make sure Plasma Manager got the notification. */
sleep(1);
status = plasma_status(plasma_conn1, oid1);
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT(status == ObjectStatus_Local);
/* Test for object being remote. */
status = plasma_status(plasma_conn2, oid1);
ARROW_CHECK_OK(client2.Info(oid1, &status));
ASSERT(status == ObjectStatus_Remote);
/* Sleep to make sure the object has been fetched and it is now stored in the
* local Plasma Store. */
plasma_fetch(plasma_conn2, 1, oid_array1);
ARROW_CHECK_OK(client2.Fetch(1, oid_array1));
sleep(1);
status = plasma_status(plasma_conn2, oid1);
ARROW_CHECK_OK(client2.Info(oid1, &status));
ASSERT(status == ObjectStatus_Local);
sleep(1);
plasma_disconnect(plasma_conn1);
plasma_disconnect(plasma_conn2);
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
@@ -116,14 +124,15 @@ bool is_equal_data_123(uint8_t *data1, uint8_t *data2, uint64_t size) {
}
TEST plasma_nonblocking_get_tests(void) {
PlasmaConnection *plasma_conn = plasma_connect("/tmp/store1", "/tmp/manager1",
PLASMA_DEFAULT_RELEASE_DELAY);
ObjectID oid = globally_unique_id();
PlasmaClient client;
ARROW_CHECK_OK(client.Connect("/tmp/store1", "/tmp/manager1",
PLASMA_DEFAULT_RELEASE_DELAY));
ObjectID oid = ObjectID::from_random();
ObjectID oid_array[1] = {oid};
ObjectBuffer obj_buffer;
/* Test for object non-existence. */
plasma_get(plasma_conn, oid_array, 1, 0, &obj_buffer);
ARROW_CHECK_OK(client.Get(oid_array, 1, 0, &obj_buffer));
ASSERT(obj_buffer.data_size == -1);
/* Test for the object being in local Plasma store. */
@@ -132,27 +141,29 @@ TEST plasma_nonblocking_get_tests(void) {
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
uint8_t *data;
plasma_create(plasma_conn, oid, data_size, metadata, metadata_size, &data);
ARROW_CHECK_OK(client.Create(oid, data_size, metadata, metadata_size, &data));
init_data_123(data, data_size, 0);
plasma_seal(plasma_conn, oid);
ARROW_CHECK_OK(client.Seal(oid));
sleep(1);
plasma_get(plasma_conn, oid_array, 1, 0, &obj_buffer);
ARROW_CHECK_OK(client.Get(oid_array, 1, 0, &obj_buffer));
ASSERT(is_equal_data_123(data, obj_buffer.data, data_size) == true);
sleep(1);
plasma_disconnect(plasma_conn);
ARROW_CHECK_OK(client.Disconnect());
PASS();
}
TEST plasma_wait_for_objects_tests(void) {
PlasmaConnection *plasma_conn1 = plasma_connect(
"/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY);
PlasmaConnection *plasma_conn2 = plasma_connect(
"/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY);
ObjectID oid1 = globally_unique_id();
ObjectID oid2 = globally_unique_id();
PlasmaClient client1;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
PLASMA_DEFAULT_RELEASE_DELAY));
PlasmaClient client2;
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
PLASMA_DEFAULT_RELEASE_DELAY));
ObjectID oid1 = ObjectID::from_random();
ObjectID oid2 = ObjectID::from_random();
#define NUM_OBJ_REQUEST 2
#define WAIT_TIMEOUT_MS 1000
ObjectRequest obj_requests[NUM_OBJ_REQUEST];
@@ -164,8 +175,9 @@ TEST plasma_wait_for_objects_tests(void) {
struct timeval start, end;
gettimeofday(&start, NULL);
int n = plasma_wait(plasma_conn1, NUM_OBJ_REQUEST, obj_requests,
NUM_OBJ_REQUEST, WAIT_TIMEOUT_MS);
int n;
ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, n));
ASSERT(n == 0);
gettimeofday(&end, NULL);
float diff_ms = (end.tv_sec - start.tv_sec);
@@ -178,48 +190,51 @@ TEST plasma_wait_for_objects_tests(void) {
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
uint8_t *data;
plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data);
plasma_seal(plasma_conn1, oid1);
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
ARROW_CHECK_OK(client1.Seal(oid1));
n = plasma_wait(plasma_conn1, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS);
ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, n));
ASSERT(n == 1);
/* Create and insert an object in plasma_conn2. */
plasma_create(plasma_conn2, oid2, data_size, metadata, metadata_size, &data);
plasma_seal(plasma_conn2, oid2);
/* Create and insert an object in client2. */
ARROW_CHECK_OK(
client2.Create(oid2, data_size, metadata, metadata_size, &data));
ARROW_CHECK_OK(client2.Seal(oid2));
n = plasma_wait(plasma_conn1, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS);
ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, n));
ASSERT(n == 2);
n = plasma_wait(plasma_conn2, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS);
ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, n));
ASSERT(n == 2);
obj_requests[0].type = PLASMA_QUERY_LOCAL;
obj_requests[1].type = PLASMA_QUERY_LOCAL;
n = plasma_wait(plasma_conn1, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS);
ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, n));
ASSERT(n == 1);
n = plasma_wait(plasma_conn2, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS);
ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, n));
ASSERT(n == 1);
plasma_disconnect(plasma_conn1);
plasma_disconnect(plasma_conn2);
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
TEST plasma_get_tests(void) {
PlasmaConnection *plasma_conn1 = plasma_connect(
"/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY);
PlasmaConnection *plasma_conn2 = plasma_connect(
"/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY);
ObjectID oid1 = globally_unique_id();
ObjectID oid2 = globally_unique_id();
PlasmaClient client1, client2;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
PLASMA_DEFAULT_RELEASE_DELAY));
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
PLASMA_DEFAULT_RELEASE_DELAY));
ObjectID oid1 = ObjectID::from_random();
ObjectID oid2 = ObjectID::from_random();
ObjectBuffer obj_buffer;
ObjectID oid_array1[1] = {oid1};
@@ -229,35 +244,38 @@ TEST plasma_get_tests(void) {
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
uint8_t *data;
plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data);
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
init_data_123(data, data_size, 1);
plasma_seal(plasma_conn1, oid1);
ARROW_CHECK_OK(client1.Seal(oid1));
plasma_get(plasma_conn1, oid_array1, 1, -1, &obj_buffer);
ARROW_CHECK_OK(client1.Get(oid_array1, 1, -1, &obj_buffer));
ASSERT(data[0] == obj_buffer.data[0]);
plasma_create(plasma_conn2, oid2, data_size, metadata, metadata_size, &data);
ARROW_CHECK_OK(
client2.Create(oid2, data_size, metadata, metadata_size, &data));
init_data_123(data, data_size, 2);
plasma_seal(plasma_conn2, oid2);
ARROW_CHECK_OK(client2.Seal(oid2));
plasma_fetch(plasma_conn1, 1, oid_array2);
plasma_get(plasma_conn1, oid_array2, 1, -1, &obj_buffer);
ARROW_CHECK_OK(client1.Fetch(1, oid_array2));
ARROW_CHECK_OK(client1.Get(oid_array2, 1, -1, &obj_buffer));
ASSERT(data[0] == obj_buffer.data[0]);
sleep(1);
plasma_disconnect(plasma_conn1);
plasma_disconnect(plasma_conn2);
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
TEST plasma_get_multiple_tests(void) {
PlasmaConnection *plasma_conn1 = plasma_connect(
"/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY);
PlasmaConnection *plasma_conn2 = plasma_connect(
"/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY);
ObjectID oid1 = globally_unique_id();
ObjectID oid2 = globally_unique_id();
PlasmaClient client1, client2;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
PLASMA_DEFAULT_RELEASE_DELAY));
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
PLASMA_DEFAULT_RELEASE_DELAY));
ObjectID oid1 = ObjectID::from_random();
ObjectID oid2 = ObjectID::from_random();
ObjectID obj_ids[NUM_OBJ_REQUEST];
ObjectBuffer obj_buffer[NUM_OBJ_REQUEST];
int obj1_first = 1, obj2_first = 2;
@@ -269,26 +287,28 @@ TEST plasma_get_multiple_tests(void) {
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
uint8_t *data;
plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data);
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
init_data_123(data, data_size, obj1_first);
plasma_seal(plasma_conn1, oid1);
ARROW_CHECK_OK(client1.Seal(oid1));
/* This only waits for oid1. */
plasma_get(plasma_conn1, obj_ids, 1, -1, obj_buffer);
ARROW_CHECK_OK(client1.Get(obj_ids, 1, -1, obj_buffer));
ASSERT(data[0] == obj_buffer[0].data[0]);
plasma_create(plasma_conn2, oid2, data_size, metadata, metadata_size, &data);
ARROW_CHECK_OK(
client2.Create(oid2, data_size, metadata, metadata_size, &data));
init_data_123(data, data_size, obj2_first);
plasma_seal(plasma_conn2, oid2);
ARROW_CHECK_OK(client2.Seal(oid2));
plasma_fetch(plasma_conn1, 2, obj_ids);
plasma_get(plasma_conn1, obj_ids, 2, -1, obj_buffer);
ARROW_CHECK_OK(client1.Fetch(2, obj_ids));
ARROW_CHECK_OK(client1.Get(obj_ids, 2, -1, obj_buffer));
ASSERT(obj1_first == obj_buffer[0].data[0]);
ASSERT(obj2_first == obj_buffer[1].data[0]);
sleep(1);
plasma_disconnect(plasma_conn1);
plasma_disconnect(plasma_conn2);
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
src/plasma/test/manager_tests.cc
+16 -12
View File
@@ -53,7 +53,7 @@ typedef struct {
ClientConnection *read_conn;
/* Connect a new client to the local plasma manager and mock a request to an
* object. */
PlasmaConnection *plasma_conn;
PlasmaClient *plasma_client;
ClientConnection *client_conn;
} plasma_mock;
@@ -85,8 +85,9 @@ plasma_mock *init_plasma_mock(plasma_mock *remote_mock) {
}
/* Connect a new client to the local plasma manager and mock a request to an
* object. */
mock->plasma_conn = plasma_connect(plasma_store_socket_name,
utstring_body(manager_socket_name), 0);
mock->plasma_client = new PlasmaClient();
ARROW_CHECK_OK(mock->plasma_client->Connect(
plasma_store_socket_name, utstring_body(manager_socket_name), 0));
wait_for_pollin(mock->manager_local_fd);
mock->client_conn = ClientConnection_listen(
mock->loop, mock->manager_local_fd, mock->state, 0);
@@ -96,7 +97,8 @@ plasma_mock *init_plasma_mock(plasma_mock *remote_mock) {
void destroy_plasma_mock(plasma_mock *mock) {
PlasmaManagerState_free(mock->state);
plasma_disconnect(mock->plasma_conn);
ARROW_CHECK_OK(mock->plasma_client->Disconnect());
delete mock->plasma_client;
close(mock->local_store);
close(mock->manager_local_fd);
close(mock->manager_remote_fd);
@@ -127,17 +129,18 @@ TEST request_transfer_test(void) {
local_mock->state);
event_loop_run(local_mock->loop);
int read_fd = get_client_sock(remote_mock->read_conn);
uint8_t *request_data =
plasma_receive(read_fd, MessageType_PlasmaDataRequest);
std::vector<uint8_t> request_data;
ARROW_CHECK_OK(
PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, request_data));
ObjectID object_id2;
char *address;
int port;
plasma_read_DataRequest(request_data, &object_id2, &address, &port);
ARROW_CHECK_OK(
ReadDataRequest(request_data.data(), &object_id2, &address, &port));
ASSERT(ObjectID_equal(object_id, object_id2));
free(address);
/* Clean up. */
utstring_free(addr);
free(request_data);
destroy_plasma_mock(remote_mock);
destroy_plasma_mock(local_mock);
PASS();
@@ -180,18 +183,19 @@ TEST request_transfer_retry_test(void) {
event_loop_run(local_mock->loop);
int read_fd = get_client_sock(remote_mock2->read_conn);
uint8_t *request_data =
plasma_receive(read_fd, MessageType_PlasmaDataRequest);
std::vector<uint8_t> request_data;
ARROW_CHECK_OK(
PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, request_data));
ObjectID object_id2;
char *address;
int port;
plasma_read_DataRequest(request_data, &object_id2, &address, &port);
ARROW_CHECK_OK(
ReadDataRequest(request_data.data(), &object_id2, &address, &port));
free(address);
ASSERT(ObjectID_equal(object_id, object_id2));
/* Clean up. */
utstring_free(addr0);
utstring_free(addr1);
free(request_data);
destroy_plasma_mock(remote_mock2);
destroy_plasma_mock(remote_mock1);
destroy_plasma_mock(local_mock);
src/plasma/test/serialization_tests.cc
+149 -162
View File
@@ -3,16 +3,13 @@
#include <sys/types.h>
#include <unistd.h>
#include "plasma_common.h"
#include "plasma.h"
#include "plasma_io.h"
#include "plasma_protocol.h"
#include "common.h"
#include "io.h"
#include "../plasma.h"
SUITE(plasma_serialization_tests);
protocol_builder *g_B;
/**
* Create a temporary file. Needs to be closed by the caller.
*
@@ -34,14 +31,13 @@ int create_temp_file(void) {
* @return Pointer to the content of the message. Needs to be freed by the
* caller.
*/
uint8_t *read_message_from_file(int fd, int message_type) {
std::vector<uint8_t> read_message_from_file(int fd, int message_type) {
/* Go to the beginning of the file. */
lseek(fd, 0, SEEK_SET);
int64_t type;
int64_t length;
uint8_t *data;
read_message(fd, &type, &length, &data);
CHECK(type == message_type);
std::vector<uint8_t> data;
ARROW_CHECK_OK(ReadMessage(fd, &type, data));
ARROW_CHECK(type == message_type);
return data;
}
@@ -60,70 +56,68 @@ PlasmaObject random_plasma_object(void) {
TEST plasma_create_request_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
ObjectID object_id1 = ObjectID::from_random();
int64_t data_size1 = 42;
int64_t metadata_size1 = 11;
plasma_send_CreateRequest(fd, g_B, object_id1, data_size1, metadata_size1);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaCreateRequest);
ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaCreateRequest);
ObjectID object_id2;
int64_t data_size2;
int64_t metadata_size2;
plasma_read_CreateRequest(data, &object_id2, &data_size2, &metadata_size2);
ARROW_CHECK_OK(ReadCreateRequest(data.data(), &object_id2, &data_size2,
&metadata_size2));
ASSERT_EQ(data_size1, data_size2);
ASSERT_EQ(metadata_size1, metadata_size2);
ASSERT(ObjectID_equal(object_id1, object_id2));
free(data);
ASSERT(object_id1 == object_id2);
close(fd);
PASS();
}
TEST plasma_create_reply_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
ObjectID object_id1 = ObjectID::from_random();
PlasmaObject object1 = random_plasma_object();
plasma_send_CreateReply(fd, g_B, object_id1, &object1, 0);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaCreateReply);
ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaCreateReply);
ObjectID object_id2;
PlasmaObject object2;
memset(&object2, 0, sizeof(object2));
int error_code;
plasma_read_CreateReply(data, &object_id2, &object2, &error_code);
ASSERT(ObjectID_equal(object_id1, object_id2));
ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2));
ASSERT(object_id1 == object_id2);
ASSERT(memcmp(&object1, &object2, sizeof(object1)) == 0);
free(data);
close(fd);
PASS();
}
TEST plasma_seal_request_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
unsigned char digest1[DIGEST_SIZE];
memset(&digest1[0], 7, DIGEST_SIZE);
plasma_send_SealRequest(fd, g_B, object_id1, &digest1[0]);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaSealRequest);
ObjectID object_id1 = ObjectID::from_random();
unsigned char digest1[kDigestSize];
memset(&digest1[0], 7, kDigestSize);
ARROW_CHECK_OK(SendSealRequest(fd, object_id1, &digest1[0]));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaSealRequest);
ObjectID object_id2;
unsigned char digest2[DIGEST_SIZE];
plasma_read_SealRequest(data, &object_id2, &digest2[0]);
ASSERT(ObjectID_equal(object_id1, object_id2));
ASSERT(memcmp(&digest1[0], &digest2[0], DIGEST_SIZE) == 0);
free(data);
unsigned char digest2[kDigestSize];
ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0]));
ASSERT(object_id1 == object_id2);
ASSERT(memcmp(&digest1[0], &digest2[0], kDigestSize) == 0);
close(fd);
PASS();
}
TEST plasma_seal_reply_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
int error1 = 5;
plasma_send_SealReply(fd, g_B, object_id1, error1);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaSealReply);
ObjectID object_id1 = ObjectID::from_random();
ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaSealReply);
ObjectID object_id2;
int error2;
plasma_read_SealReply(data, &object_id2, &error2);
ASSERT(ObjectID_equal(object_id1, object_id2));
ASSERT(error1 == error2);
free(data);
Status s = ReadSealReply(data.data(), &object_id2);
ASSERT(object_id1 == object_id2);
ASSERT(s.IsPlasmaObjectExists());
close(fd);
PASS();
}
@@ -131,18 +125,19 @@ TEST plasma_seal_reply_test(void) {
TEST plasma_get_request_test(void) {
int fd = create_temp_file();
ObjectID object_ids[2];
object_ids[0] = globally_unique_id();
object_ids[1] = globally_unique_id();
object_ids[0] = ObjectID::from_random();
object_ids[1] = ObjectID::from_random();
int64_t timeout_ms = 1234;
plasma_send_GetRequest(fd, g_B, object_ids, 2, timeout_ms);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaGetRequest);
ObjectID object_ids_return[2];
ARROW_CHECK_OK(SendGetRequest(fd, object_ids, 2, timeout_ms));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaGetRequest);
std::vector<ObjectID> object_ids_return;
int64_t timeout_ms_return;
plasma_read_GetRequest(data, &object_ids_return[0], &timeout_ms_return, 2);
ASSERT(ObjectID_equal(object_ids[0], object_ids_return[0]));
ASSERT(ObjectID_equal(object_ids[1], object_ids_return[1]));
ARROW_CHECK_OK(
ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return));
ASSERT(object_ids[0] == object_ids_return[0]);
ASSERT(object_ids[1] == object_ids_return[1]);
ASSERT(timeout_ms == timeout_ms_return);
free(data);
close(fd);
PASS();
}
@@ -150,101 +145,97 @@ TEST plasma_get_request_test(void) {
TEST plasma_get_reply_test(void) {
int fd = create_temp_file();
ObjectID object_ids[2];
object_ids[0] = globally_unique_id();
object_ids[1] = globally_unique_id();
object_ids[0] = ObjectID::from_random();
object_ids[1] = ObjectID::from_random();
std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> plasma_objects;
plasma_objects[object_ids[0]] = random_plasma_object();
plasma_objects[object_ids[1]] = random_plasma_object();
plasma_send_GetReply(fd, g_B, object_ids, plasma_objects, 2);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaGetReply);
int64_t num_objects = plasma_read_GetRequest_num_objects(data);
ObjectID object_ids_return[num_objects];
ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaGetReply);
ObjectID object_ids_return[2];
PlasmaObject plasma_objects_return[2];
memset(&plasma_objects_return, 0, sizeof(plasma_objects_return));
plasma_read_GetReply(data, object_ids_return, &plasma_objects_return[0],
num_objects);
ASSERT(ObjectID_equal(object_ids[0], object_ids_return[0]));
ASSERT(ObjectID_equal(object_ids[1], object_ids_return[1]));
ARROW_CHECK_OK(ReadGetReply(data.data(), object_ids_return,
&plasma_objects_return[0], 2));
ASSERT(object_ids[0] == object_ids_return[0]);
ASSERT(object_ids[1] == object_ids_return[1]);
ASSERT(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
sizeof(PlasmaObject)) == 0);
ASSERT(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1],
sizeof(PlasmaObject)) == 0);
free(data);
close(fd);
PASS();
}
TEST plasma_release_request_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
plasma_send_ReleaseRequest(fd, g_B, object_id1);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaReleaseRequest);
ObjectID object_id1 = ObjectID::from_random();
ARROW_CHECK_OK(SendReleaseRequest(fd, object_id1));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaReleaseRequest);
ObjectID object_id2;
plasma_read_ReleaseRequest(data, &object_id2);
ASSERT(ObjectID_equal(object_id1, object_id2));
free(data);
ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2));
ASSERT(object_id1 == object_id2);
close(fd);
PASS();
}
TEST plasma_release_reply_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
int error1 = 5;
plasma_send_ReleaseReply(fd, g_B, object_id1, error1);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaReleaseReply);
ObjectID object_id1 = ObjectID::from_random();
ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaReleaseReply);
ObjectID object_id2;
int error2;
plasma_read_ReleaseReply(data, &object_id2, &error2);
ASSERT(ObjectID_equal(object_id1, object_id2));
ASSERT(error1 == error2);
free(data);
Status s = ReadReleaseReply(data.data(), &object_id2);
ASSERT(object_id1 == object_id2);
ASSERT(s.IsPlasmaObjectExists());
close(fd);
PASS();
}
TEST plasma_delete_request_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
plasma_send_DeleteRequest(fd, g_B, object_id1);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaDeleteRequest);
ObjectID object_id1 = ObjectID::from_random();
ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaDeleteRequest);
ObjectID object_id2;
plasma_read_DeleteRequest(data, &object_id2);
ASSERT(ObjectID_equal(object_id1, object_id2));
free(data);
ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2));
ASSERT(object_id1 == object_id2);
close(fd);
PASS();
}
TEST plasma_delete_reply_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
int error1 = 5;
plasma_send_DeleteReply(fd, g_B, object_id1, error1);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaDeleteReply);
ObjectID object_id1 = ObjectID::from_random();
int error1 = PlasmaError_ObjectExists;
ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaDeleteReply);
ObjectID object_id2;
int error2;
plasma_read_DeleteReply(data, &object_id2, &error2);
ASSERT(ObjectID_equal(object_id1, object_id2));
ASSERT(error1 == error2);
free(data);
Status s = ReadDeleteReply(data.data(), &object_id2);
ASSERT(object_id1 == object_id2);
ASSERT(s.IsPlasmaObjectExists());
close(fd);
PASS();
}
TEST plasma_status_request_test(void) {
int fd = create_temp_file();
ObjectID object_ids[2];
object_ids[0] = globally_unique_id();
object_ids[1] = globally_unique_id();
plasma_send_StatusRequest(fd, g_B, object_ids, 2);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaStatusRequest);
int64_t num_objects = plasma_read_StatusRequest_num_objects(data);
int64_t num_objects = 2;
ObjectID object_ids[num_objects];
object_ids[0] = ObjectID::from_random();
object_ids[1] = ObjectID::from_random();
ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaStatusRequest);
ObjectID object_ids_read[num_objects];
plasma_read_StatusRequest(data, object_ids_read, num_objects);
ASSERT(ObjectID_equal(object_ids[0], object_ids_read[0]));
ASSERT(ObjectID_equal(object_ids[1], object_ids_read[1]));
free(data);
ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects));
ASSERT(object_ids[0] == object_ids_read[0]);
ASSERT(object_ids[1] == object_ids_read[1]);
close(fd);
PASS();
}
@@ -252,21 +243,21 @@ TEST plasma_status_request_test(void) {
TEST plasma_status_reply_test(void) {
int fd = create_temp_file();
ObjectID object_ids[2];
object_ids[0] = globally_unique_id();
object_ids[1] = globally_unique_id();
object_ids[0] = ObjectID::from_random();
object_ids[1] = ObjectID::from_random();
int object_statuses[2] = {42, 43};
plasma_send_StatusReply(fd, g_B, object_ids, object_statuses, 2);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaStatusReply);
int64_t num_objects = plasma_read_StatusReply_num_objects(data);
ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaStatusReply);
int64_t num_objects = ReadStatusReply_num_objects(data.data());
ObjectID object_ids_read[num_objects];
int object_statuses_read[num_objects];
plasma_read_StatusReply(data, object_ids_read, object_statuses_read,
num_objects);
ASSERT(ObjectID_equal(object_ids[0], object_ids_read[0]));
ASSERT(ObjectID_equal(object_ids[1], object_ids_read[1]));
ARROW_CHECK_OK(ReadStatusReply(data.data(), object_ids_read,
object_statuses_read, num_objects));
ASSERT(object_ids[0] == object_ids_read[0]);
ASSERT(object_ids[1] == object_ids_read[1]);
ASSERT_EQ(object_statuses[0], object_statuses_read[0]);
ASSERT_EQ(object_statuses[1], object_statuses_read[1]);
free(data);
close(fd);
PASS();
}
@@ -274,12 +265,12 @@ TEST plasma_status_reply_test(void) {
TEST plasma_evict_request_test(void) {
int fd = create_temp_file();
int64_t num_bytes = 111;
plasma_send_EvictRequest(fd, g_B, num_bytes);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaEvictRequest);
ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaEvictRequest);
int64_t num_bytes_received;
plasma_read_EvictRequest(data, &num_bytes_received);
ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received));
ASSERT_EQ(num_bytes, num_bytes_received);
free(data);
close(fd);
PASS();
}
@@ -287,12 +278,12 @@ TEST plasma_evict_request_test(void) {
TEST plasma_evict_reply_test(void) {
int fd = create_temp_file();
int64_t num_bytes = 111;
plasma_send_EvictReply(fd, g_B, num_bytes);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaEvictReply);
ARROW_CHECK_OK(SendEvictReply(fd, num_bytes));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaEvictReply);
int64_t num_bytes_received;
plasma_read_EvictReply(data, &num_bytes_received);
ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received));
ASSERT_EQ(num_bytes, num_bytes_received);
free(data);
close(fd);
PASS();
}
@@ -300,15 +291,15 @@ TEST plasma_evict_reply_test(void) {
TEST plasma_fetch_request_test(void) {
int fd = create_temp_file();
ObjectID object_ids[2];
object_ids[0] = globally_unique_id();
object_ids[1] = globally_unique_id();
plasma_send_FetchRequest(fd, g_B, object_ids, 2);
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaFetchRequest);
ObjectID object_ids_read[2];
plasma_read_FetchRequest(data, &object_ids_read[0], 2);
ASSERT(ObjectID_equal(object_ids[0], object_ids_read[0]));
ASSERT(ObjectID_equal(object_ids[1], object_ids_read[1]));
free(data);
object_ids[0] = ObjectID::from_random();
object_ids[1] = ObjectID::from_random();
ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2));
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaFetchRequest);
std::vector<ObjectID> object_ids_read;
ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read));
ASSERT(object_ids[0] == object_ids_read[0]);
ASSERT(object_ids[1] == object_ids_read[1]);
close(fd);
PASS();
}
@@ -317,22 +308,21 @@ TEST plasma_wait_request_test(void) {
int fd = create_temp_file();
const int num_objects_in = 2;
ObjectRequest object_requests_in[num_objects_in] = {
ObjectRequest({globally_unique_id(), PLASMA_QUERY_ANYWHERE, 0}),
ObjectRequest({globally_unique_id(), PLASMA_QUERY_LOCAL, 0})};
ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_ANYWHERE, 0}),
ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_LOCAL, 0})};
const int num_ready_objects_in = 1;
int64_t timeout_ms = 1000;
plasma_send_WaitRequest(fd, g_B, &object_requests_in[0], num_objects_in,
num_ready_objects_in, timeout_ms);
ARROW_CHECK_OK(SendWaitRequest(fd, &object_requests_in[0], num_objects_in,
num_ready_objects_in, timeout_ms));
/* Read message back. */
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaWaitRequest);
int num_object_ids_out = plasma_read_WaitRequest_num_object_ids(data);
ASSERT_EQ(num_object_ids_out, num_objects_in);
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaWaitRequest);
int num_ready_objects_out;
int64_t timeout_ms_read;
ObjectRequestMap object_requests_out;
plasma_read_WaitRequest(data, object_requests_out, num_object_ids_out,
&timeout_ms_read, &num_ready_objects_out);
ARROW_CHECK_OK(ReadWaitRequest(data.data(), object_requests_out,
&timeout_ms_read, &num_ready_objects_out));
ASSERT_EQ(num_objects_in, object_requests_out.size());
ASSERT_EQ(num_ready_objects_out, num_ready_objects_in);
for (int i = 0; i < num_objects_in; i++) {
@@ -340,11 +330,9 @@ TEST plasma_wait_request_test(void) {
ASSERT_EQ(1, object_requests_out.count(object_id));
const auto &entry = object_requests_out.find(object_id);
ASSERT(entry != object_requests_out.end());
ASSERT(ObjectID_equal(entry->second.object_id,
object_requests_in[i].object_id));
ASSERT(entry->second.object_id == object_requests_in[i].object_id);
ASSERT_EQ(entry->second.type, object_requests_in[i].type);
}
free(data);
close(fd);
PASS();
}
@@ -354,68 +342,69 @@ TEST plasma_wait_reply_test(void) {
const int num_objects_in = 2;
/* Create a map with two ObjectRequests in it. */
ObjectRequestMap objects_in(num_objects_in);
ObjectID id1 = globally_unique_id();
ObjectID id1 = ObjectID::from_random();
objects_in[id1] = ObjectRequest({id1, 0, ObjectStatus_Local});
ObjectID id2 = globally_unique_id();
ObjectID id2 = ObjectID::from_random();
objects_in[id2] = ObjectRequest({id2, 0, ObjectStatus_Nonexistent});
plasma_send_WaitReply(fd, g_B, objects_in, num_objects_in);
ARROW_CHECK_OK(SendWaitReply(fd, objects_in, num_objects_in));
/* Read message back. */
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaWaitReply);
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaWaitReply);
ObjectRequest objects_out[2];
int num_objects_out;
plasma_read_WaitReply(data, &objects_out[0], &num_objects_out);
ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out));
ASSERT(num_objects_in == num_objects_out);
for (int i = 0; i < num_objects_out; i++) {
/* Each object request must appear exactly once. */
ASSERT(1 == objects_in.count(objects_out[i].object_id));
const auto &entry = objects_in.find(objects_out[i].object_id);
ASSERT(entry != objects_in.end());
ASSERT(ObjectID_equal(entry->second.object_id, objects_out[i].object_id));
ASSERT(entry->second.object_id == objects_out[i].object_id);
ASSERT(entry->second.status == objects_out[i].status);
}
free(data);
close(fd);
PASS();
}
TEST plasma_data_request_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
ObjectID object_id1 = ObjectID::from_random();
const char *address1 = "address1";
int port1 = 12345;
plasma_send_DataRequest(fd, g_B, object_id1, address1, port1);
ARROW_CHECK_OK(SendDataRequest(fd, object_id1, address1, port1));
/* Reading message back. */
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaDataRequest);
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaDataRequest);
ObjectID object_id2;
char *address2;
int port2;
plasma_read_DataRequest(data, &object_id2, &address2, &port2);
ASSERT(ObjectID_equal(object_id1, object_id2));
ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2));
ASSERT(object_id1 == object_id2);
ASSERT(strcmp(address1, address2) == 0);
ASSERT(port1 == port2);
free(address2);
free(data);
close(fd);
PASS();
}
TEST plasma_data_reply_test(void) {
int fd = create_temp_file();
ObjectID object_id1 = globally_unique_id();
ObjectID object_id1 = ObjectID::from_random();
int64_t object_size1 = 146;
int64_t metadata_size1 = 198;
plasma_send_DataReply(fd, g_B, object_id1, object_size1, metadata_size1);
ARROW_CHECK_OK(SendDataReply(fd, object_id1, object_size1, metadata_size1));
/* Reading message back. */
uint8_t *data = read_message_from_file(fd, MessageType_PlasmaDataReply);
std::vector<uint8_t> data =
read_message_from_file(fd, MessageType_PlasmaDataReply);
ObjectID object_id2;
int64_t object_size2;
int64_t metadata_size2;
plasma_read_DataReply(data, &object_id2, &object_size2, &metadata_size2);
ASSERT(ObjectID_equal(object_id1, object_id2));
ARROW_CHECK_OK(
ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2));
ASSERT(object_id1 == object_id2);
ASSERT(object_size1 == object_size2);
ASSERT(metadata_size1 == metadata_size2);
free(data);
PASS();
}
@@ -444,9 +433,7 @@ SUITE(plasma_serialization_tests) {
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
g_B = make_protocol_builder();
GREATEST_MAIN_BEGIN();
RUN_SUITE(plasma_serialization_tests);
GREATEST_MAIN_END();
free_protocol_builder(g_B);
}