Merge task table and task log into a single table (#30)

* Merge task table and task log

* Fix test in db tests

* Address Robert's comments and some better error checking

* Add a LOG_FATAL that exits the program

src/photon/photon_algorithm.c

@@ -4,12 +4,12 @@
 #include "utarray.h"
 #include "utlist.h"
 
-#include "state/task_log.h"
+#include "state/task_table.h"
 #include "photon.h"
 #include "photon_scheduler.h"
 
 typedef struct task_queue_entry {
-  task_instance *task;
+  task *task;
   struct task_queue_entry *prev;
   struct task_queue_entry *next;
 } task_queue_entry;
@@ -102,7 +102,7 @@ int find_and_schedule_task_if_possible(scheduler_info *info,
   int found_task_to_schedule = 0;
   /* Find the first task whose dependencies are available locally. */
   DL_FOREACH_SAFE(state->task_queue, elt, tmp) {
-    spec = task_instance_task_spec(elt->task);
+    spec = task_task_spec(elt->task);
     if (can_run(state, spec)) {
       found_task_to_schedule = 1;
       break;
@@ -122,43 +122,43 @@ int find_and_schedule_task_if_possible(scheduler_info *info,
 
 void handle_task_submitted(scheduler_info *info,
                            scheduler_state *s,
-                           task_spec *task) {
+                           task_spec *spec) {
   /* Create a unique task instance ID. This is different from the task ID and
    * is used to distinguish between potentially multiple executions of the
    * task. */
-  task_iid task_iid = globally_unique_id();
-  task_instance *instance =
-      make_task_instance(task_iid, task, TASK_STATUS_WAITING, NIL_ID);
+  task *task = alloc_task(spec, TASK_STATUS_WAITING, NIL_ID);
   /* If this task's dependencies are available locally, and if there is an
    * available worker, then assign this task to an available worker. Otherwise,
    * add this task to the local task queue. */
   int schedule_locally =
-      (utarray_len(s->available_workers) > 0) && can_run(s, task);
+      (utarray_len(s->available_workers) > 0) && can_run(s, spec);
   if (schedule_locally) {
     /* Get the last available worker in the available worker queue. */
     int *worker_index = (int *) utarray_back(s->available_workers);
     /* Tell the available worker to execute the task. */
-    assign_task_to_worker(info, task, *worker_index);
+    assign_task_to_worker(info, spec, *worker_index);
     /* Remove the available worker from the queue and free the struct. */
     utarray_pop_back(s->available_workers);
   } else {
     /* Add the task to the task queue. This passes ownership of the task queue.
      * And the task will be freed when it is assigned to a worker. */
     task_queue_entry *elt = malloc(sizeof(task_queue_entry));
-    elt->task = instance;
+    elt->task = task;
     DL_APPEND(s->task_queue, elt);
   }
   /* Submit the task to redis. */
-  /* TODO(swang): We should set these values in a config file somewhere. */
+  /* TODO(swang): We should set retry values in a config file somewhere. */
   retry_info retry = {
       .num_retries = 0, .timeout = 0, .fail_callback = NULL,
   };
-  task_log_publish(info->db, instance, &retry, NULL, NULL);
+  /* TODO(swang): This should be task_table_update if the task is already in the
+   * log. */
+  task_table_add_task(info->db, task, &retry, NULL, NULL);
   if (schedule_locally) {
     /* If the task was scheduled locally, we need to free it. Otherwise,
      * ownership of the task is passed to the task_queue, and it will be freed
      * when it is assigned to a worker. */
-    free(instance);
+    free_task(task);
   }
 }
 

src/photon/photon_algorithm.h

@@ -42,7 +42,7 @@ void free_scheduler_state(scheduler_state *state);
  */
 void handle_task_submitted(scheduler_info *info,
                            scheduler_state *state,
-                           task_spec *task);
+                           task_spec *spec);
 
 /**
  * This function will be called when a task is assigned by the global scheduler

src/photon/photon_client.c

@@ -11,7 +11,8 @@ photon_conn *photon_connect(const char *photon_socket) {
 }
 
 void photon_submit(photon_conn *conn, task_spec *task) {
-  write_message(conn->conn, SUBMIT_TASK, task_size(task), (uint8_t *)task);
+  write_message(conn->conn, SUBMIT_TASK, task_spec_size(task),
+                (uint8_t *) task);
 }
 
 task_spec *photon_get_task(photon_conn *conn) {
@@ -24,7 +25,7 @@ task_spec *photon_get_task(photon_conn *conn) {
   read_message(conn->conn, &type, &length, &message);
   CHECK(type == EXECUTE_TASK);
   task_spec *task = (task_spec *)message;
-  CHECK(length == task_size(task));
+  CHECK(length == task_spec_size(task));
   return task;
 }
 

src/photon/photon_scheduler.c

@@ -14,11 +14,12 @@
 #include "photon_scheduler.h"
 #include "plasma_client.h"
 #include "state/db.h"
-#include "state/task_log.h"
+#include "state/task_table.h"
+#include "state/object_table.h"
 #include "utarray.h"
 #include "uthash.h"
 
-UT_icd task_ptr_icd = {sizeof(task_instance *), NULL, NULL, NULL};
+UT_icd task_ptr_icd = {sizeof(task *), NULL, NULL, NULL};
 UT_icd worker_icd = {sizeof(worker), NULL, NULL, NULL};
 
 /** Association between the socket fd of a worker and its worker_index. */
@@ -95,11 +96,11 @@ void free_local_scheduler(local_scheduler_state *s) {
 }
 
 void assign_task_to_worker(scheduler_info *info,
-                           task_spec *task,
+                           task_spec *spec,
                            int worker_index) {
   CHECK(worker_index < utarray_len(info->workers));
   worker *w = (worker *) utarray_eltptr(info->workers, worker_index);
-  write_message(w->sock, EXECUTE_TASK, task_size(task), (uint8_t *) task);
+  write_message(w->sock, EXECUTE_TASK, task_spec_size(spec), (uint8_t *) spec);
 }
 
 void process_plasma_notification(event_loop *loop,
@@ -212,17 +213,14 @@ int main(int argc, char *argv[]) {
       plasma_socket_name = optarg;
       break;
     default:
-      LOG_ERR("unknown option %c", c);
-      exit(-1);
+      LOG_FATAL("unknown option %c", c);
     }
   }
   if (!scheduler_socket_name) {
-    LOG_ERR("please specify socket for incoming connections with -s switch");
-    exit(-1);
+    LOG_FATAL("please specify socket for incoming connections with -s switch");
   }
   if (!plasma_socket_name) {
-    LOG_ERR("please specify socket for connecting to Plasma with -p switch");
-    exit(-1);
+    LOG_FATAL("please specify socket for connecting to Plasma with -p switch");
   }
   /* Parse the Redis address into an IP address and a port. */
   char redis_addr[16] = {0};
@@ -230,8 +228,8 @@ int main(int argc, char *argv[]) {
   if (!redis_addr_port ||
       sscanf(redis_addr_port, "%15[0-9.]:%5[0-9]", redis_addr, redis_port) !=
           2) {
-    LOG_ERR("need to specify redis address like 127.0.0.1:6379 with -r switch");
-    exit(-1);
+    LOG_FATAL(
+        "need to specify redis address like 127.0.0.1:6379 with -r switch");
   }
   start_server(scheduler_socket_name, &redis_addr[0], atoi(redis_port),
                plasma_socket_name);