Various performance improvements (#24)

* switch from array to linked list for photon queue

* performance optimizations

* fix tests

* various fixes

src/photon/photon_algorithm.c

@@ -2,11 +2,18 @@
 
 #include <stdbool.h>
 #include "utarray.h"
+#include "utlist.h"
 
 #include "state/task_log.h"
 #include "photon.h"
 #include "photon_scheduler.h"
 
+typedef struct task_queue_entry {
+  task_instance *task;
+  struct task_queue_entry *prev;
+  struct task_queue_entry *next;
+} task_queue_entry;
+
 typedef struct {
   /* Object id of this object. */
   object_id object_id;
@@ -17,7 +24,7 @@ typedef struct {
 /** Part of the photon state that is maintained by the scheduling algorithm. */
 struct scheduler_state {
   /** An array of pointers to tasks that are waiting to be scheduled. */
-  UT_array *task_queue;
+  task_queue_entry *task_queue;
   /** An array of worker indices corresponding to clients that are
    *  waiting for tasks. */
   UT_array *available_workers;
@@ -31,21 +38,21 @@ scheduler_state *make_scheduler_state(void) {
   /* Initialize an empty hash map for the cache of local available objects. */
   state->local_objects = NULL;
   /* Initialize the local data structures used for queuing tasks and workers. */
-  utarray_new(state->task_queue, &task_ptr_icd);
+  state->task_queue = NULL;
   utarray_new(state->available_workers, &ut_int_icd);
   return state;
 }
 
 void free_scheduler_state(scheduler_state *s) {
-  for (int i = 0; i < utarray_len(s->task_queue); ++i) {
-    task_instance **instance =
-        (task_instance **) utarray_eltptr(s->task_queue, i);
-    free(*instance);
+  task_queue_entry *elt, *tmp1;
+  DL_FOREACH_SAFE(s->task_queue, elt, tmp1) {
+    DL_DELETE(s->task_queue, elt);
+    free(elt->task);
+    free(elt);
   }
-  utarray_free(s->task_queue);
   utarray_free(s->available_workers);
-  available_object *available_obj, *tmp;
-  HASH_ITER(handle, s->local_objects, available_obj, tmp) {
+  available_object *available_obj, *tmp2;
+  HASH_ITER(handle, s->local_objects, available_obj, tmp2) {
     HASH_DELETE(handle, s->local_objects, available_obj);
     free(available_obj);
   }
@@ -90,14 +97,12 @@ bool can_run(scheduler_state *s, task_spec *task) {
 int find_and_schedule_task_if_possible(scheduler_info *info,
                                        scheduler_state *state,
                                        int worker_index) {
+  task_queue_entry *elt, *tmp;
+  task_spec *spec;
   int found_task_to_schedule = 0;
   /* Find the first task whose dependencies are available locally. */
-  task_spec *spec;
-  task_instance **task;
-  int i = 0;
-  for (; i < utarray_len(state->task_queue); ++i) {
-    task = (task_instance **) utarray_eltptr(state->task_queue, i);
-    spec = task_instance_task_spec(*task);
+  DL_FOREACH_SAFE(state->task_queue, elt, tmp) {
+    spec = task_instance_task_spec(elt->task);
     if (can_run(state, spec)) {
       found_task_to_schedule = 1;
       break;
@@ -108,8 +113,9 @@ int find_and_schedule_task_if_possible(scheduler_info *info,
      * worker. */
     assign_task_to_worker(info, spec, worker_index);
     /* Update the task queue data structure and free the task. */
-    free(*task);
-    utarray_erase(state->task_queue, i, 1);
+    DL_DELETE(state->task_queue, elt);
+    free(elt->task);
+    free(elt);
   }
   return found_task_to_schedule;
 }
@@ -138,7 +144,9 @@ void handle_task_submitted(scheduler_info *info,
   } else {
     /* Add the task to the task queue. This passes ownership of the task queue.
      * And the task will be freed when it is assigned to a worker. */
-    utarray_push_back(s->task_queue, &instance);
+    task_queue_entry *elt = malloc(sizeof(task_queue_entry));
+    elt->task = instance;
+    DL_APPEND(s->task_queue, elt);
   }
   /* Submit the task to redis. */
   /* TODO(swang): We should set these values in a config file somewhere. */
@@ -164,7 +172,7 @@ void handle_worker_available(scheduler_info *info,
   if (!scheduled_task) {
     for (int *p = (int *) utarray_front(state->available_workers); p != NULL;
          p = (int *) utarray_next(state->available_workers, p)) {
-      CHECK(*p != worker_index);
+      DCHECK(*p != worker_index);
     }
     /* Add client_sock to a list of available workers. This struct will be freed
      * when a task is assigned to this worker. */

src/photon/photon_scheduler.c

@@ -32,18 +32,23 @@ typedef struct {
 } worker_index;
 
 struct local_scheduler_state {
-  /* The local scheduler event loop. */
+  /** The local scheduler event loop. */
   event_loop *loop;
-  /* The Plasma client. */
+  /** The Plasma client. */
   plasma_connection *plasma_conn;
-  /* Association between client socket and worker index. */
+  /** Association between client socket and worker index. */
   worker_index *worker_index;
-  /* Info that is exposed to the scheduling algorithm. */
+  /** Info that is exposed to the scheduling algorithm. */
   scheduler_info *scheduler_info;
-  /* State for the scheduling algorithm. */
+  /** State for the scheduling algorithm. */
   scheduler_state *scheduler_state;
+  /** Input buffer, used for reading input in process_message to avoid
+   *  allocation for each call to process_message. */
+  UT_array *input_buffer;
 };
 
+UT_icd byte_icd = {sizeof(uint8_t), NULL, NULL, NULL};
+
 local_scheduler_state *init_local_scheduler(event_loop *loop,
                                             const char *redis_addr,
                                             int redis_port,
@@ -68,6 +73,7 @@ local_scheduler_state *init_local_scheduler(event_loop *loop,
   db_attach(state->scheduler_info->db, loop);
   /* Add scheduler state. */
   state->scheduler_state = make_scheduler_state();
+  utarray_new(state->input_buffer, &byte_icd);
   return state;
 };
 
@@ -82,6 +88,7 @@ void free_local_scheduler(local_scheduler_state *s) {
   utarray_free(s->scheduler_info->workers);
   free(s->scheduler_info);
   free_scheduler_state(s->scheduler_state);
+  utarray_free(s->input_buffer);
   event_loop_destroy(s->loop);
   free(s);
 }
@@ -109,17 +116,14 @@ void process_message(event_loop *loop, int client_sock, void *context,
                      int events) {
   local_scheduler_state *s = context;
 
-  uint8_t *message;
   int64_t type;
-  int64_t length;
-  read_message(client_sock, &type, &length, &message);
+  read_buffer(client_sock, &type, s->input_buffer);
 
   LOG_DEBUG("New event of type %" PRId64, type);
 
   switch (type) {
   case SUBMIT_TASK: {
-    task_spec *spec = (task_spec *) message;
-    CHECK(task_size(spec) == length);
+    task_spec *spec = (task_spec *) utarray_front(s->input_buffer);
     handle_task_submitted(s->scheduler_info, s->scheduler_state, spec);
   } break;
   case TASK_DONE: {
@@ -140,7 +144,6 @@ void process_message(event_loop *loop, int client_sock, void *context,
     /* This code should be unreachable. */
     CHECK(0);
   }
-  free(message);
 }
 
 void new_client_connection(event_loop *loop, int listener_sock, void *context,