getting the object store working

lib/orchlib/orchlib.cc

@@ -1,86 +0,0 @@
-#include <iostream>
-#include <memory>
-#include <string>
-#include <thread>
-
-#include <grpc++/grpc++.h>
-
-using grpc::Server;
-using grpc::ServerBuilder;
-using grpc::ServerContext;
-using grpc::Status;
-
-#include "orchestra.grpc.pb.h"
-#include "orchlib.h"
-
-using grpc::Channel;
-using grpc::ClientContext;
-using grpc::Status;
-
-class Client {
- public:
-  Client(std::shared_ptr<Channel> channel)
-      : stub_(Orchestra::NewStub(channel)) {}
-
-  size_t RemoteCall(const std::string& name) {
-    RemoteCallRequest request;
-    request.set_name(name);
-
-    RemoteCallReply reply;
-    ClientContext context;
-
-    Status status = stub_->RemoteCall(&context, request, &reply);
-
-    return reply.result();
-  }
-
-  void RegisterWorker() {
-    RegisterWorkerRequest request;
-    RegisterWorkerReply reply;
-    ClientContext context;
-    Status status = stub_->RegisterWorker(&context, request, &reply);
-    return;
-  }
-
- private:
-  std::unique_ptr<Orchestra::Stub> stub_;
-};
-
-class WorkerServiceImpl final : public Worker::Service {
-  Status InvokeCall(ServerContext* context, const InvokeCallRequest* request,
-    InvokeCallReply* reply) override {
-      std::cout << "invoke call request" << std::endl;
-      return Status::OK;
-  }
-};
-
-void start_server() {
-  std::string server_address("0.0.0.0:50053");
-  WorkerServiceImpl service;
-  ServerBuilder builder;
-  builder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
-  builder.RegisterService(&service);
-  std::unique_ptr<Server> server(builder.BuildAndStart());
-  std::cout << "Server listening on " << server_address << std::endl;
-  server->Wait();
-}
-
-void* orch_create_context(const char* server_addr) {
-  Client* client = new Client(grpc::CreateChannel(server_addr, grpc::InsecureChannelCredentials()));
-  client->RegisterWorker();
-  return client;
-}
-
-size_t orch_remote_call(void* context, const char* name, void* args) {
-  Client* client = (Client*)context;
-  return client->RemoteCall(std::string(name));
-}
-
-int main(int argc, char** argv) {
-  Client greeter(
-      grpc::CreateChannel("localhost:50052", grpc::InsecureChannelCredentials()));
-  std::string user("world");
-  greeter.RemoteCall(user);
-
-  return 0;
-}

lib/orchlib/orchlib.h

@@ -1,11 +0,0 @@
-extern "C" {
-
-void* orch_create_context(const char* server_addr);
-size_t orch_remote_call(void* context, const char* name, void* args);
-
-void* orch_arglist_create();
-void orch_arglist_add_ref(void* arglist, size_t ref);
-void orch_arglist_add_string(void* arglist, const char* str);
-void orch_arglist_destroy(void* arglist);
-
-}

lib/orchpy/orchpy/services.py

@@ -0,0 +1,30 @@
+import subprocess32 as subprocess
+import os
+import atexit
+import time
+
+_services_path = os.path.dirname(os.path.abspath(__file__))
+
+all_processes = []
+
+def cleanup():
+  timeout_sec = 5
+  for p in all_processes:
+    p_sec = 0
+    for second in range(timeout_sec):
+      if p.poll() == None:
+        time.sleep(1)
+        p_sec += 1
+        if p_sec >= timeout_sec:
+          p.kill() # supported from python 2.6
+          print 'helper processes shut down!'
+
+atexit.register(cleanup)
+
+def start_scheduler(scheduler_address):
+  p = subprocess.Popen([os.path.join(_services_path, "scheduler_server"), str(scheduler_address)])
+  all_processes.append(p)
+
+def start_objstore(objstore_address):
+  p = subprocess.Popen([os.path.join(_services_path, "objstore"), str(objstore_address)])
+  all_processes.append(p)

lib/orchpy/orchpy/unison.pyx

@@ -1,8 +1,15 @@
-from libc.stdint cimport uint64_t, int64_t
+# Will be rewritten in C++ for easier deployment once the API is stabilized
+
+from libc.stdint cimport uint64_t, int64_t, uintptr_t
 from libcpp cimport bool
 from libcpp.string cimport string
 import numpy as np
 
+try:
+  import cPickle as pickle
+except:
+  import pickle
+
 cdef extern from "types.pb.h":
   ctypedef enum DataType:
     INT32
@@ -23,10 +30,11 @@ cdef extern from "types.pb.h":
     Value* add_value()
     Value* mutable_value(int index)
 
+
   cdef cppclass String:
-      String()
-      void set_data(const char* val)
-      string* mutable_data()
+    String()
+    void set_data(const char* val)
+    string* mutable_data()
 
   cdef cppclass Int:
     Int()
@@ -38,28 +46,48 @@ cdef extern from "types.pb.h":
     void set_data(double val)
     double data()
 
+  cdef cppclass PyObj:
+    PyObj()
+    void set_data(const char* val, size_t len)
+    string* mutable_data()
+
   cdef cppclass Obj:
     Obj()
     String* mutable_string_data()
     Int* mutable_int_data()
     Double* mutable_double_data()
+    PyObj* mutable_pyobj_data()
     bool has_string_data()
     bool has_int_data()
     bool has_double_data()
+    bool ParseFromString(const string& data)
 
-cdef class PyValues:
+cdef class PyValues: # TODO: unify with the below
   cdef Values *thisptr
   def __cinit__(self):
     self.thisptr = new Values()
   def __dealloc__(self):
     del self.thisptr
+  def get_value(self):
+    return <uintptr_t>self.thisptr
 
-cdef class PyValue:
+cdef class PyValue: # TODO: unify with the below
   cdef Value *thisptr
   def __cinit__(self):
     self.thisptr = new Value()
   def __dealloc__(self):
     del self.thisptr
+  def get_value(self):
+    return <uintptr_t>self.thisptr
+
+cdef class ObjWrapper: # TODO: unify with the above
+  cdef Obj *thisptr
+  def __cinit__(self):
+    self.thisptr = new Obj()
+  # def __dealloc__(self):
+  #   del self.thisptr
+  def get_value(self):
+    return <uintptr_t>self.thisptr
 
 cdef class ObjRef:
   cdef size_t _id
@@ -80,32 +108,70 @@ cdef class ObjRef:
   cpdef get_id(self):
     return self._id
 
-cpdef serialize_args(args):
-  cdef Values* vals
-  cdef Value* val
-  cdef Obj* obj
+cpdef serialize_into(val, objptr):
+  cdef uintptr_t ptr = <uintptr_t>objptr
+  cdef Obj* obj = <Obj*>ptr
   cdef String* string_data
   cdef Int* int_data
   cdef Double* double_data
-  result = PyValues()
-  vals = result.thisptr
+  if type(val) == str:
+    string_data = obj[0].mutable_string_data()
+    string_data[0].set_data(val)
+  elif type(val) == int or type(val) == long:
+    int_data = obj[0].mutable_int_data()
+    int_data[0].set_data(val)
+  elif type(val) == float:
+    double_data = obj[0].mutable_double_data()
+    double_data[0].set_data(val)
+  else:
+    data = pickle.dumps(val, pickle.HIGHEST_PROTOCOL)
+    pyobj_data = obj[0].mutable_pyobj_data()
+    pyobj_data[0].set_data(data, len(data))
+
+cpdef serialize(val):
+  result = ObjWrapper()
+  serialize_into(val, result.get_value())
+  return result
+
+cpdef serialize_args_into(args, valsptr):
+  cdef uintptr_t ptr = <uintptr_t>valsptr
+  cdef Values* vals = <Values*>ptr
+  cdef Value* val
+  cdef Obj* obj
   for arg in args:
     val = vals[0].add_value()
     if type(arg) == ObjRef:
       val[0].set_ref(arg.get_id())
     else:
       obj = val[0].mutable_obj()
-      if type(arg) == str:
-        string_data = obj[0].mutable_string_data()
-        string_data[0].set_data(arg)
-      elif type(arg) == int or type(arg) == long:
-        int_data = obj[0].mutable_int_data()
-        int_data[0].set_data(arg)
-      elif type(arg) == float:
-        double_data = obj[0].mutable_double_data()
-        double_data[0].set_data(arg)
+      serialize_into(arg, <uintptr_t>obj)
+
+cpdef serialize_args(args):
+  result = PyValues()
+  serialize_args_into(args, result.get_value())
   return result
 
+cdef deserialize_from(Obj* obj):
+  if obj[0].has_string_data():
+    return obj[0].mutable_string_data()[0].mutable_data()[0]
+  elif obj[0].has_int_data():
+    return obj[0].mutable_int_data()[0].data()
+  elif obj[0].has_double_data():
+    return obj[0].mutable_double_data()[0].data()
+  else:
+    data = obj[0].mutable_pyobj_data()[0].mutable_data()[0]
+    return pickle.loads(data)
+
+cpdef deserialize_from_string(str):
+  cdef string s = str
+  cdef Obj* obj = new Obj() # TODO: memory leak
+  obj[0].ParseFromString(s)
+  return deserialize_from(obj)
+
+# cpdef deserialize(str):
+#     cdef string s = string(str)
+#     return deserialize_from(obj.get_value())
+
 cpdef deserialize_args(PyValues args):
   cdef Values* vals = args.thisptr
   cdef Value* val
@@ -123,6 +189,9 @@ cpdef deserialize_args(PyValues args):
         result.append(obj[0].mutable_int_data()[0].data())
       elif obj[0].has_double_data():
         result.append(obj[0].mutable_double_data()[0].data())
+      else:
+        data = obj[0].mutable_pyobj_data()[0].mutable_data()[0]
+        result.append(pickle.loads(data))
   return result
 
 cdef int numpy_dtype_to_proto(dtype):

lib/orchpy/orchpy/worker.pyx

@@ -1,5 +1,135 @@
-cdef extern void* orch_create_context(const char* server_addr);
-cdef extern size_t orch_remote_call(void* context, const char* name, void* args);
+from libc.stdint cimport uintptr_t
+import orchpy.unison as unison
+
+from libc.stdint cimport uint64_t, int64_t, uintptr_t
+from libcpp cimport bool
+from libcpp.string cimport string
+
+cdef struct Slice:
+  char* ptr
+  size_t size
+
+cdef extern void* orch_create_context(const char* server_addr, const char* worker_addr, const char* objstore_addr);
+cdef extern void orch_register_function(void* worker, const char* name, size_t num_return_vals)
+cdef extern size_t orch_remote_call(void* context, void* request);
+cdef extern size_t orch_push(void* context, void* value);
+cdef extern void orch_main_loop(void* context);
+cdef extern Slice orch_get_serialized_obj(void* context, size_t objref);
+
+cdef extern from "Python.h":
+    Py_ssize_t PyByteArray_GET_SIZE(object array)
+    object PyUnicode_FromStringAndSize(char *buff, Py_ssize_t len)
+    object PyBytes_FromStringAndSize(char *buff, Py_ssize_t len)
+    object PyString_FromStringAndSize(char *buff, Py_ssize_t len)
+    int PyByteArray_Resize(object self, Py_ssize_t size) except -1
+    char* PyByteArray_AS_STRING(object bytearray)
+
+cdef extern from "types.pb.h":
+  cdef cppclass Values
+
+cdef extern from "orchestra.pb.h":
+  cdef cppclass RemoteCallRequest:
+    RemoteCallRequest()
+    void set_name(const char* value)
+    Values* mutable_arg()
+
+cdef extern from "types.pb.h":
+  ctypedef enum DataType:
+    INT32
+    INT64
+    FLOAT32
+    FLOAT64
+
+  cdef cppclass Value:
+    Value()
+    void set_ref(uint64_t value)
+    uint64_t ref()
+    bool has_obj()
+    Obj* mutable_obj()
+
+  cdef cppclass Values:
+    Values()
+    int value_size()
+    Value* add_value()
+    Value* mutable_value(int index)
+
+
+  cdef cppclass String:
+    String()
+    void set_data(const char* val)
+    string* mutable_data()
+
+  cdef cppclass Int:
+    Int()
+    void set_data(int64_t val)
+    int64_t data()
+
+  cdef cppclass Double:
+    Double()
+    void set_data(double val)
+    double data()
+
+  cdef cppclass PyObj:
+    PyObj()
+    void set_data(const char* val, size_t len)
+    string* mutable_data()
+
+  cdef cppclass Obj:
+    Obj()
+    String* mutable_string_data()
+    Int* mutable_int_data()
+    Double* mutable_double_data()
+    PyObj* mutable_pyobj_data()
+    bool has_string_data()
+    bool has_int_data()
+    bool has_double_data()
+
+cdef serialize_into(val, Obj* obj):
+  cdef String* string_data
+  cdef Int* int_data
+  cdef Double* double_data
+  if type(val) == str:
+    string_data = obj[0].mutable_string_data()
+    string_data[0].set_data(val)
+  elif type(val) == int or type(val) == long:
+    int_data = obj[0].mutable_int_data()
+    int_data[0].set_data(val)
+  elif type(val) == float:
+    double_data = obj[0].mutable_double_data()
+    double_data[0].set_data(val)
+  # else:
+  #   data = pickle.dumps(val, pickle.HIGHEST_PROTOCOL)
+  #   pyobj_data = obj[0].mutable_pyobj_data()
+  #   pyobj_data[0].set_data(data, len(data))
+
+cdef class ObjWrapper: # TODO: unify with the above
+  cdef Obj *thisptr
+  def __cinit__(self):
+    self.thisptr = new Obj()
+  # def __dealloc__(self):
+  #   del self.thisptr
+  def get_value(self):
+    return <uintptr_t>self.thisptr
+
+cpdef serialize_into_2(val, objptr):
+  cdef uintptr_t ptr = <uintptr_t>objptr
+  cdef Obj* obj = <Obj*>ptr
+  cdef String* string_data
+  cdef Int* int_data
+  cdef Double* double_data
+  if type(val) == str:
+    string_data = obj[0].mutable_string_data()
+    string_data[0].set_data(val)
+  elif type(val) == int or type(val) == long:
+    int_data = obj[0].mutable_int_data()
+    int_data[0].set_data(val)
+  elif type(val) == float:
+    double_data = obj[0].mutable_double_data()
+    double_data[0].set_data(val)
+  # else:
+  #   data = pickle.dumps(val, pickle.HIGHEST_PROTOCOL)
+  #   pyobj_data = obj[0].mutable_pyobj_data()
+  #   pyobj_data[0].set_data(data, len(data))
 
 cdef class Worker:
   cdef void* context
@@ -7,11 +137,48 @@ cdef class Worker:
   def __cinit__(self):
     self.context = NULL
 
-  def connect(self, server_addr):
-    self.context = orch_create_context(server_addr)
+  def connect(self, server_addr, worker_addr, objstore_addr):
+    self.context = orch_create_context(server_addr, worker_addr, objstore_addr)
 
-  def call(self, name):
-    return orch_remote_call(self.context, name, <void*>0)
+#   cpdef call(self, name, args):
+#     cdef RemoteCallRequest* result = new RemoteCallRequest()
+#     result[0].set_name(name)
+#     unison.serialize_args_into(args, <uintptr_t>result[0].mutable_arg())
+#     for i in range(10):
+#       orch_remote_call(self.context, result)
+#     # return <uintptr_t>result
+
+  cpdef do_call(self, ptr):
+    return orch_remote_call(self.context, <void*>ptr)
+
+  cpdef do_push(self, val):
+    print("before serialization")
+    result = unison.serialize(val)
+    print("before push")
+    # ptr = result.get_value()
+    # print "pointer is", ptr
+    # cdef Obj* obj = new Obj()
+    o = ObjWrapper()
+    # serialize_into_2(0, <uintptr_t>obj)
+    # cdef Obj* ptr = new Obj() # o.get_value()
+    ## ptr = <uintptr_t>o.get_value()
+    ptr = <uintptr_t>result.get_value()
+    serialize_into_2(0, ptr)
+    return orch_push(self.context, <void*>ptr)
+
+  cpdef get_serialized(self, objref):
+    cdef Slice slice = orch_get_serialized_obj(self.context, objref)
+    data = PyBytes_FromStringAndSize(slice.ptr, slice.size)
+    return data
+
+  cpdef pull(self, objref):
+    cdef Slice slice = orch_get_serialized_obj(self.context, objref)
+
+  cpdef register_function(self, func_name, num_args):
+    orch_register_function(self.context, func_name, num_args)
+
+  cpdef main_loop(self):
+    orch_main_loop(self.context)
 
 global_worker = Worker()
 

lib/orchpy/setup.py

@@ -8,8 +8,10 @@ setup(
   version = "0.1.dev0",
   ext_modules = cythonize([
     Extension("orchpy/worker",
+      include_dirs = ["../../src"],
       sources = ["orchpy/worker.pyx"],
-      extra_link_args=["-Iorchpy -lorchlib"]),
+      extra_link_args=["-Iorchpy -lorchlib"],
+      language = "c++"),
     Extension("orchpy/unison",
       include_dirs = ["../../src/"],
       sources = ["orchpy/unison.pyx"],
@@ -19,7 +21,7 @@ setup(
   use_2to3=True,
   packages=find_packages(),
   package_data = {
-    'orchpy': ['liborchlib.so']
+    'orchpy': ['liborchlib.so', 'scheduler_server', 'objstore']
   },
   zip_safe=False
 )