wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-13 12:30:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Start and clean up workers from the local scheduler. (#250)

Browse Source 
* Start and clean up workers from the local scheduler

Ability to kill workers in photon scheduler

Test for old method of starting workers

Common codepath for killing workers

Common codepath for killing workers

Photon test case for starting and killing workers

fix build

Fix component failure test

Register a worker's pid as part of initial connection

Address comments and revert photon_connect

Set PATH during travis install

Fix

* Fix photon test case to accept clients on plasma manager fd
This commit is contained in:
Stephanie Wang
2017-02-10 12:46:23 -08:00
committed by Robert Nishihara
parent ec175b7dfb
commit 2b8e6485e3
12 changed files with 556 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/photon/test/photon_tests.c
+142 -38
View File
@@ -33,25 +33,33 @@ int64_t timeout_handler(event_loop *loop, int64_t id, void *context) {
}
typedef struct {
/** A socket to mock the Plasma store. */
int plasma_fd;
/** A socket to mock the Plasma manager. Clients (such as workers) that
* connect to this file descriptor must be accepted. */
int plasma_manager_fd;
/** A socket to communicate with the Plasma store. */
int plasma_store_fd;
/** Photon's socket for IPC requests. */
int photon_fd;
/** Photon's local scheduler state. */
local_scheduler_state *photon_state;
/** Photon's event loop. */
event_loop *loop;
/** A Photon client connection. */
photon_conn *conn;
/** Number of Photon client connections, or mock workers. */
int num_photon_conns;
/** Photon client connections. */
photon_conn **conns;
} photon_mock;
photon_mock *init_photon_mock(bool connect_to_redis) {
photon_mock *init_photon_mock(bool connect_to_redis,
int num_workers,
int num_mock_workers) {
const char *node_ip_address = "127.0.0.1";
const char *redis_addr = NULL;
int redis_port = -1;
const double static_resource_conf[MAX_RESOURCE_INDEX] = {DEFAULT_NUM_CPUS,
DEFAULT_NUM_GPUS};
if (connect_to_redis) {
redis_addr = "127.0.0.1";
redis_addr = node_ip_address;
redis_port = 6379;
}
@@ -59,34 +67,60 @@ photon_mock *init_photon_mock(bool connect_to_redis) {
memset(mock, 0, sizeof(photon_mock));
mock->loop = event_loop_create();
/* Bind to the Photon port and initialize the Photon scheduler. */
/* TODO(rkn): Why are we reusing mock->plasma_fd for both the store and the
* manager? */
UT_string *plasma_manager_socket_name =
bind_ipc_sock_retry(plasma_manager_socket_name_format, &mock->plasma_fd);
mock->plasma_fd = socket_connect_retry(plasma_store_socket_name, 5, 100);
UT_string *plasma_manager_socket_name = bind_ipc_sock_retry(
plasma_manager_socket_name_format, &mock->plasma_manager_fd);
mock->plasma_store_fd =
socket_connect_retry(plasma_store_socket_name, 5, 100);
UT_string *photon_socket_name =
bind_ipc_sock_retry(photon_socket_name_format, &mock->photon_fd);
CHECK(mock->plasma_fd >= 0 && mock->photon_fd >= 0);
CHECK(mock->plasma_store_fd >= 0 && mock->photon_fd >= 0);
UT_string *worker_command;
utstring_new(worker_command);
utstring_printf(worker_command,
"python ../../python/ray/workers/default_worker.py "
"--node-ip-address=%s --object-store-name=%s "
"--object-store-manager-name=%s --local-scheduler-name=%s "
"--redis-address=%s:%d",
node_ip_address, plasma_store_socket_name,
utstring_body(plasma_manager_socket_name),
utstring_body(photon_socket_name), redis_addr, redis_port);
mock->photon_state = init_local_scheduler(
"127.0.0.1", mock->loop, redis_addr, redis_port,
utstring_body(photon_socket_name), plasma_store_socket_name,
utstring_body(plasma_manager_socket_name), NULL, false, NULL,
static_resource_conf);
utstring_body(plasma_manager_socket_name), NULL, false,
static_resource_conf, utstring_body(worker_command), num_workers);
/* Accept the workers as clients to the plasma manager. */
for (int i = 0; i < num_workers; ++i) {
accept_client(mock->plasma_manager_fd);
}
/* Connect a Photon client. */
mock->conn = photon_connect(utstring_body(photon_socket_name));
new_client_connection(mock->loop, mock->photon_fd,
(void *) mock->photon_state, 0);
mock->num_photon_conns = num_mock_workers;
mock->conns = malloc(sizeof(photon_conn *) * num_mock_workers);
for (int i = 0; i < num_mock_workers; ++i) {
mock->conns[i] = photon_connect(utstring_body(photon_socket_name));
new_client_connection(mock->loop, mock->photon_fd,
(void *) mock->photon_state, 0);
}
utstring_free(worker_command);
utstring_free(plasma_manager_socket_name);
utstring_free(photon_socket_name);
return mock;
}
void destroy_photon_mock(photon_mock *mock) {
photon_disconnect(mock->conn);
close(mock->photon_fd);
close(mock->plasma_fd);
for (int i = 0; i < mock->num_photon_conns; ++i) {
photon_disconnect(mock->conns[i]);
}
free(mock->conns);
/* This also frees mock->loop. */
free_local_scheduler(mock->photon_state);
close(mock->plasma_store_fd);
close(mock->plasma_manager_fd);
free(mock);
}
@@ -103,7 +137,9 @@ void reset_worker(photon_mock *mock, local_scheduler_client *worker) {
* value, the task should get assigned to a worker again.
*/
TEST object_reconstruction_test(void) {
photon_mock *photon = init_photon_mock(true);
photon_mock *photon = init_photon_mock(true, 0, 1);
photon_conn *worker = photon->conns[0];
/* Create a task with zero dependencies and one return value. */
task_spec *spec = example_task_spec(0, 1);
object_id return_id = task_return(spec, 0);
@@ -125,10 +161,10 @@ TEST object_reconstruction_test(void) {
if (pid == 0) {
/* Make sure we receive the task twice. First from the initial submission,
* and second from the reconstruct request. */
photon_submit(photon->conn, spec);
task_spec *task_assigned = photon_get_task(photon->conn);
photon_submit(worker, spec);
task_spec *task_assigned = photon_get_task(worker);
ASSERT_EQ(memcmp(task_assigned, spec, task_spec_size(spec)), 0);
task_spec *reconstruct_task = photon_get_task(photon->conn);
task_spec *reconstruct_task = photon_get_task(worker);
ASSERT_EQ(memcmp(reconstruct_task, spec, task_spec_size(spec)), 0);
/* Clean up. */
free_task_spec(reconstruct_task);
@@ -150,7 +186,7 @@ TEST object_reconstruction_test(void) {
(retry_info *) &photon_retry, NULL, NULL);
/* Trigger reconstruction, and run the event loop again. */
object_id return_id = task_return(spec, 0);
photon_reconstruct_object(photon->conn, return_id);
photon_reconstruct_object(worker, return_id);
event_loop_add_timer(photon->loop, 500,
(event_loop_timer_handler) timeout_handler, NULL);
event_loop_run(photon->loop);
@@ -171,7 +207,8 @@ TEST object_reconstruction_test(void) {
* should trigger reconstruction of all previous tasks in the lineage.
*/
TEST object_reconstruction_recursive_test(void) {
photon_mock *photon = init_photon_mock(true);
photon_mock *photon = init_photon_mock(true, 0, 1);
photon_conn *worker = photon->conns[0];
/* Create a chain of tasks, each one dependent on the one before it. Mark
* each object as available so that tasks will run immediately. */
const int NUM_TASKS = 10;
@@ -204,11 +241,11 @@ TEST object_reconstruction_recursive_test(void) {
if (pid == 0) {
/* Submit the tasks, and make sure each one gets assigned to a worker. */
for (int i = 0; i < NUM_TASKS; ++i) {
photon_submit(photon->conn, specs[i]);
photon_submit(worker, specs[i]);
}
/* Make sure we receive each task from the initial submission. */
for (int i = 0; i < NUM_TASKS; ++i) {
task_spec *task_assigned = photon_get_task(photon->conn);
task_spec *task_assigned = photon_get_task(worker);
ASSERT_EQ(memcmp(task_assigned, specs[i], task_spec_size(task_assigned)),
0);
free_task_spec(task_assigned);
@@ -216,7 +253,7 @@ TEST object_reconstruction_recursive_test(void) {
/* Check that the workers receive all tasks in the final return object's
* lineage during reconstruction. */
for (int i = 0; i < NUM_TASKS; ++i) {
task_spec *task_assigned = photon_get_task(photon->conn);
task_spec *task_assigned = photon_get_task(worker);
bool found = false;
for (int j = 0; j < NUM_TASKS; ++j) {
if (specs[j] == NULL) {
@@ -249,7 +286,7 @@ TEST object_reconstruction_recursive_test(void) {
/* Trigger reconstruction for the last object, and run the event loop
* again. */
object_id return_id = task_return(specs[NUM_TASKS - 1], 0);
photon_reconstruct_object(photon->conn, return_id);
photon_reconstruct_object(worker, return_id);
event_loop_add_timer(photon->loop, 500,
(event_loop_timer_handler) timeout_handler, NULL);
event_loop_run(photon->loop);
@@ -275,25 +312,27 @@ task_spec *object_reconstruction_suppression_spec;
void object_reconstruction_suppression_callback(object_id object_id,
void *user_context) {
/* Submit the task after adding the object to the object table. */
photon_mock *photon = user_context;
photon_submit(photon->conn, object_reconstruction_suppression_spec);
photon_conn *worker = user_context;
photon_submit(worker, object_reconstruction_suppression_spec);
}
TEST object_reconstruction_suppression_test(void) {
photon_mock *photon = init_photon_mock(true);
photon_mock *photon = init_photon_mock(true, 0, 1);
photon_conn *worker = photon->conns[0];
object_reconstruction_suppression_spec = example_task_spec(0, 1);
object_id return_id = task_return(object_reconstruction_suppression_spec, 0);
pid_t pid = fork();
if (pid == 0) {
/* Make sure we receive the task once. This will block until the
* object_table_add callback completes. */
task_spec *task_assigned = photon_get_task(photon->conn);
task_spec *task_assigned = photon_get_task(worker);
ASSERT_EQ(memcmp(task_assigned, object_reconstruction_suppression_spec,
task_spec_size(object_reconstruction_suppression_spec)),
0);
/* Trigger a reconstruction. We will check that no tasks get queued as a
* result of this line in the event loop process. */
photon_reconstruct_object(photon->conn, return_id);
photon_reconstruct_object(worker, return_id);
/* Clean up. */
free_task_spec(task_assigned);
free_task_spec(object_reconstruction_suppression_spec);
@@ -309,7 +348,7 @@ TEST object_reconstruction_suppression_test(void) {
object_table_add(db, return_id, 1, (unsigned char *) NIL_DIGEST,
(retry_info *) &photon_retry,
object_reconstruction_suppression_callback,
(void *) photon);
(void *) worker);
/* Run the event loop. NOTE: OSX appears to require the parent process to
* listen for events on the open file descriptors. */
event_loop_add_timer(photon->loop, 1000,
@@ -328,7 +367,7 @@ TEST object_reconstruction_suppression_test(void) {
}
TEST task_dependency_test(void) {
photon_mock *photon = init_photon_mock(false);
photon_mock *photon = init_photon_mock(false, 0, 1);
local_scheduler_state *state = photon->photon_state;
scheduling_algorithm_state *algorithm_state = state->algorithm_state;
/* Get the first worker. */
@@ -403,7 +442,7 @@ TEST task_dependency_test(void) {
}
TEST task_multi_dependency_test(void) {
photon_mock *photon = init_photon_mock(false);
photon_mock *photon = init_photon_mock(false, 0, 1);
local_scheduler_state *state = photon->photon_state;
scheduling_algorithm_state *algorithm_state = state->algorithm_state;
/* Get the first worker. */
@@ -476,12 +515,77 @@ TEST task_multi_dependency_test(void) {
PASS();
}
TEST start_kill_workers_test(void) {
/* Start some workers. */
int num_workers = 4;
photon_mock *photon = init_photon_mock(true, num_workers, 0);
/* We start off with num_workers children processes, but no workers
* registered yet. */
ASSERT_EQ(utarray_len(photon->photon_state->child_pids), num_workers);
ASSERT_EQ(utarray_len(photon->photon_state->workers), 0);
/* Make sure that each worker connects to the photon scheduler. This for loop
* will hang if one of the workers does not connect. */
for (int i = 0; i < num_workers; ++i) {
new_client_connection(photon->loop, photon->photon_fd,
(void *) photon->photon_state, 0);
}
/* After handling each worker's initial connection, we should now have all
* workers accounted for, but we haven't yet matched up process IDs with our
* children processes. */
ASSERT_EQ(utarray_len(photon->photon_state->child_pids), num_workers);
ASSERT_EQ(utarray_len(photon->photon_state->workers), num_workers);
/* Each worker should register its process ID. */
for (int i = 0; i < utarray_len(photon->photon_state->workers); ++i) {
local_scheduler_client *worker =
*(local_scheduler_client **) utarray_eltptr(
photon->photon_state->workers, i);
process_message(photon->photon_state->loop, worker->sock, worker, 0);
}
ASSERT_EQ(utarray_len(photon->photon_state->child_pids), 0);
ASSERT_EQ(utarray_len(photon->photon_state->workers), num_workers);
/* After killing a worker, its state is cleaned up. */
local_scheduler_client *worker = *(local_scheduler_client **) utarray_eltptr(
photon->photon_state->workers, 0);
kill_worker(worker, true);
ASSERT_EQ(utarray_len(photon->photon_state->child_pids), 0);
ASSERT_EQ(utarray_len(photon->photon_state->workers), num_workers - 1);
/* Start a worker after the local scheduler has been initialized. */
start_worker(photon->photon_state);
/* Accept the workers as clients to the plasma manager. */
int new_worker_fd = accept_client(photon->plasma_manager_fd);
/* The new worker should register its process ID. */
ASSERT_EQ(utarray_len(photon->photon_state->child_pids), 1);
ASSERT_EQ(utarray_len(photon->photon_state->workers), num_workers - 1);
/* Make sure the new worker connects to the photon scheduler. */
new_client_connection(photon->loop, photon->photon_fd,
(void *) photon->photon_state, 0);
ASSERT_EQ(utarray_len(photon->photon_state->child_pids), 1);
ASSERT_EQ(utarray_len(photon->photon_state->workers), num_workers);
/* Make sure that the new worker registers its process ID. */
worker = *(local_scheduler_client **) utarray_eltptr(
photon->photon_state->workers, num_workers - 1);
process_message(photon->photon_state->loop, worker->sock, worker, 0);
ASSERT_EQ(utarray_len(photon->photon_state->child_pids), 0);
ASSERT_EQ(utarray_len(photon->photon_state->workers), num_workers);
/* Clean up. */
close(new_worker_fd);
destroy_photon_mock(photon);
PASS();
}
SUITE(photon_tests) {
RUN_REDIS_TEST(object_reconstruction_test);
RUN_REDIS_TEST(object_reconstruction_recursive_test);
RUN_REDIS_TEST(object_reconstruction_suppression_test);
RUN_TEST(task_dependency_test);
RUN_TEST(task_multi_dependency_test);
RUN_TEST(start_kill_workers_test);
}
GREATEST_MAIN_DEFS();