Rearrange files to prepare to merge into Ray.

This commit is contained in:
Robert Nishihara
2016-10-25 13:59:47 -07:00
parent db6375701e
commit 1915539c5f
53 changed files with 0 additions and 816 deletions
+24
View File
@@ -0,0 +1,24 @@
#include "greatest.h"
#include "common.h"
SUITE(common_tests);
TEST sha1_test(void) {
static char hex[2 * UNIQUE_ID_SIZE + 1];
unique_id uid = globally_unique_id();
sha1_to_hex(&uid.id[0], &hex[0]);
PASS();
}
SUITE(common_tests) {
RUN_TEST(sha1_test);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(common_tests);
GREATEST_MAIN_END();
}
+175
View File
@@ -0,0 +1,175 @@
#include "greatest.h"
#include <assert.h>
#include <unistd.h>
#include <sys/wait.h>
#include "event_loop.h"
#include "test/example_task.h"
#include "state/db.h"
#include "state/object_table.h"
#include "state/task_log.h"
#include "state/redis.h"
#include "task.h"
SUITE(db_tests);
const char *manager_addr = "127.0.0.1";
int manager_port1 = 12345;
int manager_port2 = 12346;
char received_addr1[16] = {0};
char received_port1[6] = {0};
char received_addr2[16] = {0};
char received_port2[6] = {0};
/* Test if entries have been written to the database. */
void test_callback(object_id object_id,
int manager_count,
const char *manager_vector[],
void *context) {
CHECK(manager_count == 2);
if (!manager_vector[0] ||
sscanf(manager_vector[0], "%15[0-9.]:%5[0-9]", received_addr1,
received_port1) != 2) {
CHECK(0);
}
if (!manager_vector[1] ||
sscanf(manager_vector[1], "%15[0-9.]:%5[0-9]", received_addr2,
received_port2) != 2) {
CHECK(0);
}
free(manager_vector);
}
int timeout_handler(event_loop *loop, timer_id timer_id, void *context) {
event_loop_stop(loop);
return EVENT_LOOP_TIMER_DONE;
}
TEST object_table_lookup_test(void) {
event_loop *loop = event_loop_create();
db_handle *db1 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
manager_port1);
db_handle *db2 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
manager_port2);
db_attach(db1, loop);
db_attach(db2, loop);
unique_id id = globally_unique_id();
object_table_add(db1, id);
object_table_add(db2, id);
event_loop_add_timer(loop, 100, timeout_handler, NULL);
event_loop_run(loop);
object_table_lookup(db1, id, test_callback, NULL);
event_loop_add_timer(loop, 100, timeout_handler, NULL);
event_loop_run(loop);
int port1 = atoi(received_port1);
int port2 = atoi(received_port2);
ASSERT_STR_EQ(&received_addr1[0], manager_addr);
ASSERT((port1 == manager_port1 && port2 == manager_port2) ||
(port2 == manager_port1 && port1 == manager_port2));
db_disconnect(db1);
db_disconnect(db2);
event_loop_destroy(loop);
PASS();
}
void task_log_test_callback(task_instance *instance, void *userdata) {
task_instance *other = userdata;
CHECK(*task_instance_state(instance) == TASK_STATUS_SCHEDULED);
CHECK(task_instance_size(instance) == task_instance_size(other));
CHECK(memcmp(instance, other, task_instance_size(instance)) == 0);
}
TEST task_log_test(void) {
event_loop *loop = event_loop_create();
db_handle *db = db_connect("127.0.0.1", 6379, "local_scheduler", "", -1);
db_attach(db, loop);
node_id node = globally_unique_id();
task_spec *task = example_task();
task_instance *instance = make_task_instance(globally_unique_id(), task,
TASK_STATUS_SCHEDULED, node);
task_log_register_callback(db, task_log_test_callback, node,
TASK_STATUS_SCHEDULED, instance);
task_log_add_task(db, instance);
event_loop_add_timer(loop, 100, timeout_handler, NULL);
event_loop_run(loop);
task_instance_free(instance);
free_task_spec(task);
db_disconnect(db);
event_loop_destroy(loop);
PASS();
}
int num_test_callback_called = 0;
void task_log_all_test_callback(task_instance *instance, void *userdata) {
num_test_callback_called += 1;
}
TEST task_log_all_test(void) {
event_loop *loop = event_loop_create();
db_handle *db = db_connect("127.0.0.1", 6379, "local_scheduler", "", -1);
db_attach(db, loop);
task_spec *task = example_task();
/* Schedule two tasks on different nodes. */
task_instance *instance1 = make_task_instance(
globally_unique_id(), task, TASK_STATUS_SCHEDULED, globally_unique_id());
task_instance *instance2 = make_task_instance(
globally_unique_id(), task, TASK_STATUS_SCHEDULED, globally_unique_id());
task_log_register_callback(db, task_log_all_test_callback, NIL_ID,
TASK_STATUS_SCHEDULED, NULL);
task_log_add_task(db, instance1);
task_log_add_task(db, instance2);
event_loop_add_timer(loop, 100, timeout_handler, NULL);
event_loop_run(loop);
task_instance_free(instance2);
task_instance_free(instance1);
free_task_spec(task);
db_disconnect(db);
event_loop_destroy(loop);
ASSERT(num_test_callback_called == 2);
PASS();
}
TEST unique_client_id_test(void) {
const int num_conns = 50;
db_handle *db;
pid_t pid = fork();
for (int i = 0; i < num_conns; ++i) {
db = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
manager_port1);
db_disconnect(db);
}
if (pid == 0) {
exit(0);
} else {
wait(NULL);
}
db = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
manager_port1);
ASSERT_EQ(get_client_id(db), num_conns * 2);
db_disconnect(db);
PASS();
}
SUITE(db_tests) {
redisContext *context = redisConnect("127.0.0.1", 6379);
freeReplyObject(redisCommand(context, "FLUSHALL"));
RUN_REDIS_TEST(context, object_table_lookup_test);
RUN_REDIS_TEST(context, task_log_test);
RUN_REDIS_TEST(context, task_log_all_test);
RUN_REDIS_TEST(context, unique_client_id_test);
redisFree(context);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(db_tests);
GREATEST_MAIN_END();
}
+14
View File
@@ -0,0 +1,14 @@
#ifndef EXAMPLE_TASK_H
#define EXAMPLE_TASK_H
#include "task.h"
task_spec *example_task(void) {
function_id func_id = globally_unique_id();
task_spec *task = alloc_task_spec(func_id, 2, 1, 0);
task_args_add_ref(task, globally_unique_id());
task_args_add_ref(task, globally_unique_id());
return task;
}
#endif
+106
View File
@@ -0,0 +1,106 @@
#include "greatest.h"
#include <assert.h>
#include <unistd.h>
#include <inttypes.h>
#include "io.h"
#include "utstring.h"
SUITE(io_tests);
TEST ipc_socket_test(void) {
const char *socket_pathname = "test-socket";
int socket_fd = bind_ipc_sock(socket_pathname);
ASSERT(socket_fd >= 0);
char *test_string = "hello world";
char *test_bytes = "another string";
pid_t pid = fork();
if (pid == 0) {
close(socket_fd);
socket_fd = connect_ipc_sock(socket_pathname);
ASSERT(socket_fd >= 0);
write_log_message(socket_fd, test_string);
write_message(socket_fd, LOG_MESSAGE, strlen(test_bytes),
(uint8_t *) test_bytes);
close(socket_fd);
exit(0);
} else {
int client_fd = accept_client(socket_fd);
ASSERT(client_fd >= 0);
char *message = read_log_message(client_fd);
ASSERT(message != NULL);
ASSERT_STR_EQ(test_string, message);
free(message);
int64_t type;
int64_t len;
uint8_t *bytes;
read_message(client_fd, &type, &len, &bytes);
ASSERT(type == LOG_MESSAGE);
ASSERT(memcmp(test_bytes, bytes, len) == 0);
free(bytes);
close(client_fd);
close(socket_fd);
unlink(socket_pathname);
}
PASS();
}
TEST long_ipc_socket_test(void) {
const char *socket_pathname = "long-test-socket";
int socket_fd = bind_ipc_sock(socket_pathname);
ASSERT(socket_fd >= 0);
UT_string *test_string;
utstring_new(test_string);
for (int i = 0; i < 10000; i++) {
utstring_printf(test_string, "hello world ");
}
char *test_bytes = "another string";
pid_t pid = fork();
if (pid == 0) {
close(socket_fd);
socket_fd = connect_ipc_sock(socket_pathname);
ASSERT(socket_fd >= 0);
write_log_message(socket_fd, utstring_body(test_string));
write_message(socket_fd, LOG_MESSAGE, strlen(test_bytes),
(uint8_t *) test_bytes);
close(socket_fd);
exit(0);
} else {
int client_fd = accept_client(socket_fd);
ASSERT(client_fd >= 0);
char *message = read_log_message(client_fd);
ASSERT(message != NULL);
ASSERT_STR_EQ(utstring_body(test_string), message);
free(message);
int64_t type;
int64_t len;
uint8_t *bytes;
read_message(client_fd, &type, &len, &bytes);
ASSERT(type == LOG_MESSAGE);
ASSERT(memcmp(test_bytes, bytes, len) == 0);
free(bytes);
close(client_fd);
close(socket_fd);
unlink(socket_pathname);
}
utstring_free(test_string);
PASS();
}
SUITE(io_tests) {
RUN_TEST(ipc_socket_test);
RUN_TEST(long_ipc_socket_test);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(io_tests);
GREATEST_MAIN_END();
}
+225
View File
@@ -0,0 +1,225 @@
#include "greatest.h"
#include <assert.h>
#include <unistd.h>
#include "utarray.h"
#include "event_loop.h"
#include "state/db.h"
#include "state/redis.h"
#include "io.h"
#include "logging.h"
SUITE(redis_tests);
const char *test_set_format = "SET %s %s";
const char *test_get_format = "GET %s";
const char *test_key = "foo";
const char *test_value = "bar";
UT_array *connections = NULL;
int async_redis_socket_test_callback_called = 0;
void async_redis_socket_test_callback(redisAsyncContext *ac,
void *r,
void *privdata) {
async_redis_socket_test_callback_called = 1;
redisContext *context = redisConnect("127.0.0.1", 6379);
redisReply *reply = redisCommand(context, test_get_format, test_key);
redisFree(context);
CHECK(reply != NULL);
if (strcmp(reply->str, test_value)) {
freeReplyObject(reply);
CHECK(0);
}
freeReplyObject(reply);
}
TEST redis_socket_test(void) {
const char *socket_pathname = "redis-test-socket";
redisContext *context = redisConnect("127.0.0.1", 6379);
ASSERT(context != NULL);
int socket_fd = bind_ipc_sock(socket_pathname);
ASSERT(socket_fd >= 0);
int client_fd = connect_ipc_sock(socket_pathname);
ASSERT(client_fd >= 0);
write_formatted_log_message(client_fd, test_set_format, test_key, test_value);
int server_fd = accept_client(socket_fd);
char *cmd = read_log_message(server_fd);
close(client_fd);
close(server_fd);
close(socket_fd);
unlink(socket_pathname);
redisReply *reply;
reply = redisCommand(context, cmd, 0, 0);
freeReplyObject(reply);
reply = redisCommand(context, "GET %s", test_key);
ASSERT(reply != NULL);
ASSERT_STR_EQ(reply->str, test_value);
freeReplyObject(reply);
free(cmd);
redisFree(context);
PASS();
}
void redis_read_callback(event_loop *loop, int fd, void *context, int events) {
db_handle *db = context;
char *cmd = read_log_message(fd);
redisAsyncCommand(db->context, async_redis_socket_test_callback, NULL, cmd,
db->client_id, 0);
free(cmd);
}
void redis_accept_callback(event_loop *loop,
int socket_fd,
void *context,
int events) {
int accept_fd = accept_client(socket_fd);
CHECK(accept_fd >= 0);
utarray_push_back(connections, &accept_fd);
event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, redis_read_callback,
context);
}
int timeout_handler(event_loop *loop, timer_id timer_id, void *context) {
event_loop_stop(loop);
return EVENT_LOOP_TIMER_DONE;
}
TEST async_redis_socket_test(void) {
utarray_new(connections, &ut_int_icd);
event_loop *loop = event_loop_create();
/* Start IPC channel. */
const char *socket_pathname = "async-redis-test-socket";
int socket_fd = bind_ipc_sock(socket_pathname);
ASSERT(socket_fd >= 0);
utarray_push_back(connections, &socket_fd);
/* Start connection to Redis. */
db_handle *db = db_connect("127.0.0.1", 6379, "", "", 0);
db_attach(db, loop);
/* Send a command to the Redis process. */
int client_fd = connect_ipc_sock(socket_pathname);
ASSERT(client_fd >= 0);
utarray_push_back(connections, &client_fd);
write_formatted_log_message(client_fd, test_set_format, test_key, test_value);
event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, redis_read_callback,
db);
event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, redis_accept_callback,
db);
event_loop_add_timer(loop, 100, timeout_handler, NULL);
event_loop_run(loop);
CHECK(async_redis_socket_test_callback_called);
db_disconnect(db);
event_loop_destroy(loop);
for (int *p = (int *) utarray_front(connections); p != NULL;
p = (int *) utarray_next(connections, p)) {
close(*p);
}
unlink(socket_pathname);
utarray_free(connections);
PASS();
}
int logging_test_callback_called = 0;
void logging_test_callback(redisAsyncContext *ac, void *r, void *privdata) {
logging_test_callback_called = 1;
redisContext *context = redisConnect("127.0.0.1", 6379);
redisReply *reply = redisCommand(context, "KEYS %s", "log:*");
redisFree(context);
CHECK(reply != NULL);
CHECK(reply->elements > 0);
freeReplyObject(reply);
}
void logging_read_callback(event_loop *loop,
int fd,
void *context,
int events) {
db_handle *conn = context;
char *cmd = read_log_message(fd);
redisAsyncCommand(conn->context, logging_test_callback, NULL, cmd,
conn->client_id, 0);
free(cmd);
}
void logging_accept_callback(event_loop *loop,
int socket_fd,
void *context,
int events) {
int accept_fd = accept_client(socket_fd);
CHECK(accept_fd >= 0);
utarray_push_back(connections, &accept_fd);
event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, logging_read_callback,
context);
}
TEST logging_test(void) {
utarray_new(connections, &ut_int_icd);
event_loop *loop = event_loop_create();
/* Start IPC channel. */
const char *socket_pathname = "logging-test-socket";
int socket_fd = bind_ipc_sock(socket_pathname);
ASSERT(socket_fd >= 0);
utarray_push_back(connections, &socket_fd);
/* Start connection to Redis. */
db_handle *conn = db_connect("127.0.0.1", 6379, "", "", 0);
db_attach(conn, loop);
/* Send a command to the Redis process. */
int client_fd = connect_ipc_sock(socket_pathname);
ASSERT(client_fd >= 0);
utarray_push_back(connections, &client_fd);
ray_logger *logger = init_ray_logger("worker", RAY_INFO, 0, &client_fd);
ray_log(logger, RAY_INFO, "TEST", "Message");
event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, logging_accept_callback,
conn);
event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, logging_read_callback,
conn);
event_loop_add_timer(loop, 100, timeout_handler, NULL);
event_loop_run(loop);
CHECK(logging_test_callback_called);
free_ray_logger(logger);
db_disconnect(conn);
event_loop_destroy(loop);
for (int *p = (int *) utarray_front(connections); p != NULL;
p = (int *) utarray_next(connections, p)) {
close(*p);
}
unlink(socket_pathname);
utarray_free(connections);
PASS();
}
SUITE(redis_tests) {
redisContext *context = redisConnect("127.0.0.1", 6379);
freeReplyObject(redisCommand(context, "FLUSHALL"));
RUN_REDIS_TEST(context, redis_socket_test);
RUN_REDIS_TEST(context, async_redis_socket_test);
RUN_REDIS_TEST(context, logging_test);
redisFree(context);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(redis_tests);
GREATEST_MAIN_END();
}
+77
View File
@@ -0,0 +1,77 @@
#include "greatest.h"
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include "common.h"
#include "test/example_task.h"
#include "task.h"
#include "io.h"
SUITE(task_tests);
TEST task_test(void) {
function_id func_id = globally_unique_id();
task_spec *task = alloc_task_spec(func_id, 4, 2, 10);
ASSERT(task_num_args(task) == 4);
ASSERT(task_num_returns(task) == 2);
unique_id arg1 = globally_unique_id();
ASSERT(task_args_add_ref(task, arg1) == 0);
ASSERT(task_args_add_val(task, (uint8_t *) "hello", 5) == 1);
unique_id arg2 = globally_unique_id();
ASSERT(task_args_add_ref(task, arg2) == 2);
ASSERT(task_args_add_val(task, (uint8_t *) "world", 5) == 3);
unique_id ret0 = globally_unique_id();
unique_id ret1 = globally_unique_id();
memcpy(task_return(task, 0), &ret0, sizeof(ret0));
memcpy(task_return(task, 1), &ret1, sizeof(ret1));
ASSERT(memcmp(task_arg_id(task, 0), &arg1, sizeof(arg1)) == 0);
ASSERT(memcmp(task_arg_val(task, 1), (uint8_t *) "hello",
task_arg_length(task, 1)) == 0);
ASSERT(memcmp(task_arg_id(task, 2), &arg2, sizeof(arg2)) == 0);
ASSERT(memcmp(task_arg_val(task, 3), (uint8_t *) "world",
task_arg_length(task, 3)) == 0);
ASSERT(memcmp(task_return(task, 0), &ret0, sizeof(unique_id)) == 0);
ASSERT(memcmp(task_return(task, 1), &ret1, sizeof(unique_id)) == 0);
free_task_spec(task);
PASS();
}
TEST send_task(void) {
function_id func_id = globally_unique_id();
task_spec *task = alloc_task_spec(func_id, 4, 2, 10);
*task_return(task, 1) = globally_unique_id();
int fd[2];
socketpair(AF_UNIX, SOCK_STREAM, 0, fd);
write_message(fd[0], SUBMIT_TASK, task_size(task), (uint8_t *) task);
int64_t type;
int64_t length;
uint8_t *message;
read_message(fd[1], &type, &length, &message);
task_spec *result = (task_spec *) message;
ASSERT(type == SUBMIT_TASK);
ASSERT(memcmp(task, result, task_size(task)) == 0);
ASSERT(memcmp(task, result, task_size(result)) == 0);
free(task);
free(result);
PASS();
}
SUITE(task_tests) {
RUN_TEST(task_test);
RUN_TEST(send_task);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(task_tests);
GREATEST_MAIN_END();
}
+119
View File
@@ -0,0 +1,119 @@
from __future__ import print_function
import pickle
import unittest
import common
BASE_SIMPLE_OBJECTS = [
0, 1, 100000, 0L, 1L, 100000L, 1L << 100, 0.0, 0.5, 0.9, 100000.1, (), [], {},
"", 990 * "h", u"", 990 * u"h"
]
LIST_SIMPLE_OBJECTS = [[obj] for obj in BASE_SIMPLE_OBJECTS]
TUPLE_SIMPLE_OBJECTS = [(obj,) for obj in BASE_SIMPLE_OBJECTS]
DICT_SIMPLE_OBJECTS = [{(): obj} for obj in BASE_SIMPLE_OBJECTS]
SIMPLE_OBJECTS = (BASE_SIMPLE_OBJECTS +
LIST_SIMPLE_OBJECTS +
TUPLE_SIMPLE_OBJECTS +
DICT_SIMPLE_OBJECTS)
# Create some complex objects that cannot be serialized by value in tasks.
l = []
l.append(l)
class Foo(object):
def __init__(self):
pass
BASE_COMPLEX_OBJECTS = [999 * "h", 999 * u"h", l, Foo(), 10 * [10 * [10 * [1]]]]
LIST_COMPLEX_OBJECTS = [[obj] for obj in BASE_COMPLEX_OBJECTS]
TUPLE_COMPLEX_OBJECTS = [(obj,) for obj in BASE_COMPLEX_OBJECTS]
DICT_COMPLEX_OBJECTS = [{(): obj} for obj in BASE_COMPLEX_OBJECTS]
COMPLEX_OBJECTS = (BASE_COMPLEX_OBJECTS +
LIST_COMPLEX_OBJECTS +
TUPLE_COMPLEX_OBJECTS +
DICT_COMPLEX_OBJECTS)
class TestSerialization(unittest.TestCase):
def test_serialize_by_value(self):
for val in SIMPLE_OBJECTS:
self.assertTrue(common.check_simple_value(val))
for val in COMPLEX_OBJECTS:
self.assertFalse(common.check_simple_value(val))
class TestObjectID(unittest.TestCase):
def test_create_object_id(self):
object_id = common.ObjectID(20 * "a")
def test_cannot_pickle_object_ids(self):
object_ids = [common.ObjectID(20 * chr(i)) for i in range(256)]
def f():
return object_ids
def g(val=object_ids):
return 1
def h():
x = object_ids[0]
return 1
# Make sure that object IDs cannot be pickled (including functions that
# close over object IDs).
self.assertRaises(Exception, lambda : pickling.dumps(object_ids[0]))
self.assertRaises(Exception, lambda : pickling.dumps(object_ids))
self.assertRaises(Exception, lambda : pickling.dumps(f))
self.assertRaises(Exception, lambda : pickling.dumps(g))
self.assertRaises(Exception, lambda : pickling.dumps(h))
class TestTask(unittest.TestCase):
def test_create_task(self):
# TODO(rkn): The function ID should be a FunctionID object, not an ObjectID.
function_id = common.ObjectID(20 * "a")
object_ids = [common.ObjectID(20 * chr(i)) for i in range(256)]
args_list = [
[],
1 * [1],
10 * [1],
100 * [1],
1000 * [1],
1 * ["a"],
10 * ["a"],
100 * ["a"],
1000 * ["a"],
[1, 1.3, 2L, 1L << 100, "hi", u"hi", [1, 2]],
object_ids[:1],
object_ids[:2],
object_ids[:3],
object_ids[:4],
object_ids[:5],
object_ids[:10],
object_ids[:100],
object_ids[:256],
[1, object_ids[0]],
[object_ids[0], "a"],
[1, object_ids[0], "a"],
[object_ids[0], 1, object_ids[1], "a"],
object_ids[:3] + [1, "hi", 2.3] + object_ids[:5],
object_ids + 100 * ["a"] + object_ids
]
for args in args_list:
for num_return_vals in [0, 1, 2, 3, 5, 10, 100]:
task = common.Task(function_id, args, num_return_vals)
self.assertEqual(function_id.id(), task.function_id().id())
retrieved_args = task.arguments()
self.assertEqual(num_return_vals, len(task.returns()))
self.assertEqual(len(args), len(retrieved_args))
for i in range(len(retrieved_args)):
if isinstance(retrieved_args[i], common.ObjectID):
self.assertEqual(retrieved_args[i].id(), args[i].id())
else:
self.assertEqual(retrieved_args[i], args[i])
if __name__ == "__main__":
unittest.main(verbosity=2)