wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-13 12:30:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Object hashes (#104)

Browse Source 
* factoring out object_info for general use by several Ray components

* addressing comments

* Replace SHA256 task hash with MD5

Add object hash to object table (always overwrites)

Support for table operations that span multiple asynchronous Redis
commands

Add a new object location in a transaction, using Redis's optimistic
concurrency

Use Redis GETSET instead of transactions and Python frontend code for object hashing

Remove spurious log message

Fix for object_table_add

Revert "Replace SHA256 task hash with MD5"

This reverts commit e599de473c8dad9189ccb0600429534b469b76a2.

Revert to sha256

Test case for illegal puts

Use SETNX to set object hashes

Initialize digest with zeros

Initialize plasma_request with zeros

* Fixes

* replace SHA256 with a faster hash in the object store

* Fix valgrind

* Address Robert's comments

* Check that plasma_compute_object_hash succeeds.

* Don't run test_illegal_put test with valgrind because it causes an intentional crash which causes valgrind to complain.

* Debugging after rebase.

* handling Robert's comments

* Fix bugs after rebase.

* final fixes for Stephanie's PR

* fix
This commit is contained in:
Stephanie Wang
2016-12-08 20:57:08 -08:00
committed by Philipp Moritz
parent 4a62a3c5d7
commit 61904c4c3e
22 changed files with 1713 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/test/object_table_tests.c
+67 -14
View File
@@ -146,7 +146,7 @@ int lookup_failed = 0;
void lookup_done_callback(object_id object_id,
int manager_count,
OWNER const char *manager_vector[],
const char *manager_vector[],
void *context) {
/* The done callback should not be called. */
CHECK(0);
@@ -202,8 +202,8 @@ TEST add_timeout_test(void) {
retry_info retry = {
.num_retries = 5, .timeout = 100, .fail_callback = add_fail_callback,
};
object_table_add(db, NIL_ID, &retry, add_done_callback,
(void *) add_timeout_context);
object_table_add(db, NIL_ID, (unsigned char *) NIL_DIGEST, &retry,
add_done_callback, (void *) add_timeout_context);
/* Disconnect the database to see if the lookup times out. */
close(db->context->c.fd);
event_loop_run(g_loop);
@@ -272,6 +272,7 @@ int64_t reconnect_context_callback(event_loop *loop,
db->sync_context = redisConnect("127.0.0.1", 6379);
/* Re-attach the database to the event loop (the file descriptor changed). */
db_attach(db, loop, true);
LOG_DEBUG("Reconnected to Redis");
return EVENT_LOOP_TIMER_DONE;
}
@@ -289,7 +290,7 @@ int lookup_retry_succeeded = 0;
void lookup_retry_done_callback(object_id object_id,
int manager_count,
OWNER const char *manager_vector[],
const char *manager_vector[],
void *context) {
CHECK(context == (void *) lookup_retry_context);
lookup_retry_succeeded = 1;
@@ -304,6 +305,7 @@ void lookup_retry_fail_callback(unique_id id,
TEST lookup_retry_test(void) {
g_loop = event_loop_create();
lookup_retry_succeeded = 0;
db_handle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 11235);
db_attach(db, g_loop, false);
@@ -345,7 +347,7 @@ void add_retry_fail_callback(unique_id id,
void *user_context,
void *user_data) {
/* The fail callback should not be called. */
CHECK(0);
LOG_FATAL("add_retry_succeded value was %d", add_retry_succeeded);
}
TEST add_retry_test(void) {
@@ -358,8 +360,8 @@ TEST add_retry_test(void) {
.timeout = 100,
.fail_callback = add_retry_fail_callback,
};
object_table_add(db, NIL_ID, &retry, add_retry_done_callback,
(void *) add_retry_context);
object_table_add(db, NIL_ID, (unsigned char *) NIL_DIGEST, &retry,
add_retry_done_callback, (void *) add_retry_context);
/* Disconnect the database to let the add time out the first time. */
close(db->context->c.fd);
/* Install handler for reconnecting the database. */
@@ -377,6 +379,54 @@ TEST add_retry_test(void) {
PASS();
}
/* === Test add then lookup retry === */
void add_lookup_done_callback(object_id object_id,
int manager_count,
const char *manager_vector[],
void *context) {
CHECK(context == (void *) lookup_retry_context);
CHECK(manager_count == 1);
CHECK(strcmp(manager_vector[0], "127.0.0.1:11235") == 0);
lookup_retry_succeeded = 1;
}
void add_lookup_callback(object_id object_id, void *user_context) {
db_handle *db = user_context;
retry_info retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = lookup_retry_fail_callback,
};
object_table_lookup(db, NIL_ID, &retry, add_lookup_done_callback,
(void *) lookup_retry_context);
}
TEST add_lookup_test(void) {
g_loop = event_loop_create();
lookup_retry_succeeded = 0;
db_handle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 11235);
db_attach(db, g_loop, true);
retry_info retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = lookup_retry_fail_callback,
};
object_table_add(db, NIL_ID, (unsigned char *) NIL_DIGEST, &retry,
add_lookup_callback, (void *) db);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(lookup_retry_succeeded);
PASS();
}
/* === Test subscribe retry === */
const char *subscribe_retry_context = "subscribe_retry";
@@ -462,7 +512,7 @@ void lookup_late_fail_callback(unique_id id,
void lookup_late_done_callback(object_id object_id,
int manager_count,
OWNER const char *manager_vector[],
const char *manager_vector[],
void *context) {
/* This function should never be called. */
CHECK(0);
@@ -518,8 +568,8 @@ TEST add_late_test(void) {
retry_info retry = {
.num_retries = 0, .timeout = 0, .fail_callback = add_late_fail_callback,
};
object_table_add(db, NIL_ID, &retry, add_late_done_callback,
(void *) add_late_context);
object_table_add(db, NIL_ID, (unsigned char *) NIL_DIGEST, &retry,
add_late_done_callback, (void *) add_late_context);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
@@ -601,9 +651,10 @@ void subscribe_success_done_callback(object_id object_id,
const char *manager_vector[],
void *user_context) {
retry_info retry = {
.num_retries = 0, .timeout = 0, .fail_callback = NULL,
.num_retries = 0, .timeout = 750, .fail_callback = NULL,
};
object_table_add((db_handle *) user_context, object_id, &retry, NULL, NULL);
object_table_add((db_handle *) user_context, object_id,
(unsigned char *) NIL_DIGEST, &retry, NULL, NULL);
subscribe_success_done = 1;
}
@@ -670,7 +721,7 @@ TEST subscribe_object_present_test(void) {
retry_info retry = {
.num_retries = 0, .timeout = 100, .fail_callback = NULL,
};
object_table_add(db, id, &retry, NULL, NULL);
object_table_add(db, id, (unsigned char *) NIL_DIGEST, &retry, NULL, NULL);
object_table_subscribe(
db, id, subscribe_object_present_object_available_callback,
(void *) subscribe_object_present_context, &retry, NULL, (void *) db);
@@ -751,7 +802,8 @@ int64_t add_object_callback(event_loop *loop, int64_t timer_id, void *context) {
retry_info retry = {
.num_retries = 0, .timeout = 100, .fail_callback = NULL,
};
object_table_add(db, NIL_ID, &retry, NULL, NULL);
object_table_add(db, NIL_ID, (unsigned char *) NIL_DIGEST, &retry, NULL,
NULL);
/* Reset the timer to this large value, so it doesn't trigger again. */
return 10000;
}
@@ -794,6 +846,7 @@ SUITE(object_table_tests) {
RUN_REDIS_TEST(subscribe_timeout_test);
RUN_REDIS_TEST(lookup_retry_test);
RUN_REDIS_TEST(add_retry_test);
RUN_REDIS_TEST(add_lookup_test);
RUN_REDIS_TEST(subscribe_retry_test);
RUN_REDIS_TEST(lookup_late_test);
RUN_REDIS_TEST(add_late_test);