Basic C++ worker implementation (#6125)

2026-07-07 15:28:52 +08:00 · 2020-03-27 23:01:08 +08:00
parent 93b5c38b7d
commit c195dc8f88
53 changed files with 2464 additions and 1 deletions
@@ -0,0 +1,231 @@
+
+#pragma once
+
+#include <memory>
+
+#include <ray/api/generated/actor_funcs.generated.h>
+#include <ray/api/generated/create_funcs.generated.h>
+#include <ray/api/generated/funcs.generated.h>
+#include <ray/api/ray_runtime.h>
+#include <msgpack.hpp>
+#include "ray/core.h"
+namespace ray {
+namespace api {
+
+template <typename T>
+class RayObject;
+template <typename T>
+class RayActor;
+
+class WaitResult;
+
+class Ray {
+ public:
+  /// Initialize Ray runtime.
+  static void Init();
+
+  /// Store an object in the object store.
+  ///
+  /// \param[in] obj The object which should be stored.
+  /// \return RayObject A reference to the object in the object store.
+  template <typename T>
+  static RayObject<T> Put(const T &obj);
+
+  /// Get a list of objects from the object store.
+  /// This method will be blocked until all the objects are ready.
+  ///
+  /// \param[in] ids The object id array which should be got.
+  /// \return shared pointer array of the result.
+  template <typename T>
+  static std::vector<std::shared_ptr<T>> Get(const std::vector<ObjectID> &ids);
+
+  /// Get a list of objects from the object store.
+  /// This method will be blocked until all the objects are ready.
+  ///
+  /// \param[in] objects The object array which should be got.
+  /// \return shared pointer array of the result.
+  template <typename T>
+  static std::vector<std::shared_ptr<T>> Get(const std::vector<RayObject<T>> &ids);
+
+  /// Wait for a list of RayObjects to be locally available,
+  /// until specified number of objects are ready, or specified timeout has passed.
+  ///
+  /// \param[in] ids The object id array which should be waited.
+  /// \param[in] num_objects The minimum number of objects to wait.
+  /// \param[in] timeout_ms The maximum wait time in milliseconds.
+  /// \return Two arrays, one containing locally available objects, one containing the
+  /// rest.
+  static WaitResult Wait(const std::vector<ObjectID> &ids, int num_objects,
+                         int timeout_ms);
+
+/// Include the `Call` methods for calling remote functions.
+#include "api/generated/call_funcs.generated.h"
+
+/// Include the `CreateActor` methods for creating actors.
+#include "api/generated/create_actors.generated.h"
+
+ private:
+  static RayRuntime *runtime_;
+
+  static std::once_flag is_inited_;
+
+  /// Used by RayObject to implement .Get()
+  template <typename T>
+  static std::shared_ptr<T> Get(const RayObject<T> &object);
+
+  template <typename ReturnType, typename FuncType, typename ExecFuncType,
+            typename... ArgTypes>
+  static RayObject<ReturnType> CallInternal(FuncType &func, ExecFuncType &exec_func,
+                                            ArgTypes &... args);
+
+  template <typename ReturnType, typename FuncType, typename ExecFuncType,
+            typename... ArgTypes>
+  static RayActor<ReturnType> CreateActorInternal(FuncType &func, ExecFuncType &exec_func,
+                                                  ArgTypes &... args);
+
+  template <typename ReturnType, typename ActorType, typename FuncType,
+            typename ExecFuncType, typename... ArgTypes>
+  static RayObject<ReturnType> CallActorInternal(FuncType &actor_func,
+                                                 ExecFuncType &exec_func,
+                                                 RayActor<ActorType> &actor,
+                                                 ArgTypes &... args);
+
+/// Include the `Call` methods for calling actor methods.
+/// Used by RayActor to implement .Call()
+#include "api/generated/call_actors.generated.h"
+
+  template <typename T>
+  friend class RayObject;
+
+  template <typename ActorType>
+  friend class RayActor;
+};
+
+}  // namespace api
+}  // namespace ray
+
+// --------- inline implementation ------------
+#include <ray/api/arguments.h>
+#include <ray/api/ray_actor.h>
+#include <ray/api/ray_object.h>
+#include <ray/api/serializer.h>
+#include <ray/api/wait_result.h>
+
+namespace ray {
+namespace api {
+
+template <typename T>
+inline static std::vector<ObjectID> RayObjectsToObjectIDs(
+    const std::vector<RayObject<T>> &ray_objects) {
+  std::vector<ObjectID> object_ids;
+  for (auto it = ray_objects.begin(); it != ray_objects.end(); it++) {
+    object_ids.push_back(it->ID());
+  }
+  return object_ids;
+}
+
+template <typename T>
+inline RayObject<T> Ray::Put(const T &obj) {
+  std::shared_ptr<msgpack::sbuffer> buffer(new msgpack::sbuffer());
+  msgpack::packer<msgpack::sbuffer> packer(buffer.get());
+  Serializer::Serialize(packer, obj);
+  auto id = runtime_->Put(buffer);
+  return RayObject<T>(id);
+}
+
+template <typename T>
+inline std::shared_ptr<T> Ray::Get(const RayObject<T> &object) {
+  auto packed_object = runtime_->Get(object.ID());
+  msgpack::unpacker unpacker;
+  unpacker.reserve_buffer(packed_object->size());
+  memcpy(unpacker.buffer(), packed_object->data(), packed_object->size());
+  unpacker.buffer_consumed(packed_object->size());
+  std::shared_ptr<T> return_object(new T);
+  Serializer::Deserialize(unpacker, return_object.get());
+  return return_object;
+}
+
+template <typename T>
+inline std::vector<std::shared_ptr<T>> Ray::Get(const std::vector<ObjectID> &ids) {
+  auto result = runtime_->Get(ids);
+  std::vector<std::shared_ptr<T>> return_objects;
+  return_objects.reserve(result.size());
+  for (auto it = result.begin(); it != result.end(); it++) {
+    msgpack::unpacker unpacker;
+    unpacker.reserve_buffer((*it)->size());
+    memcpy(unpacker.buffer(), (*it)->data(), (*it)->size());
+    unpacker.buffer_consumed((*it)->size());
+    std::shared_ptr<T> obj(new T);
+    Serializer::Deserialize(unpacker, obj.get());
+    return_objects.push_back(obj);
+  }
+  return return_objects;
+}
+
+template <typename T>
+inline std::vector<std::shared_ptr<T>> Ray::Get(const std::vector<RayObject<T>> &ids) {
+  auto object_ids = RayObjectsToObjectIDs<T>(ids);
+  return Get<T>(object_ids);
+}
+
+inline WaitResult Ray::Wait(const std::vector<ObjectID> &ids, int num_objects,
+                            int timeout_ms) {
+  return runtime_->Wait(ids, num_objects, timeout_ms);
+}
+
+template <typename ReturnType, typename FuncType, typename ExecFuncType,
+          typename... ArgTypes>
+inline RayObject<ReturnType> Ray::CallInternal(FuncType &func, ExecFuncType &exec_func,
+                                               ArgTypes &... args) {
+  std::shared_ptr<msgpack::sbuffer> buffer(new msgpack::sbuffer());
+  msgpack::packer<msgpack::sbuffer> packer(buffer.get());
+  Arguments::WrapArgs(packer, args...);
+  RemoteFunctionPtrHolder ptr;
+  ptr.function_pointer = reinterpret_cast<uintptr_t>(func);
+  ptr.exec_function_pointer = reinterpret_cast<uintptr_t>(exec_func);
+  auto returned_object_id = runtime_->Call(ptr, buffer);
+  return RayObject<ReturnType>(returned_object_id);
+}
+
+template <typename ReturnType, typename FuncType, typename ExecFuncType,
+          typename... ArgTypes>
+inline RayActor<ReturnType> Ray::CreateActorInternal(FuncType &create_func,
+                                                     ExecFuncType &exec_func,
+                                                     ArgTypes &... args) {
+  std::shared_ptr<msgpack::sbuffer> buffer(new msgpack::sbuffer());
+  msgpack::packer<msgpack::sbuffer> packer(buffer.get());
+  Arguments::WrapArgs(packer, args...);
+  RemoteFunctionPtrHolder ptr;
+  ptr.function_pointer = reinterpret_cast<uintptr_t>(create_func);
+  ptr.exec_function_pointer = reinterpret_cast<uintptr_t>(exec_func);
+  auto returned_actor_id = runtime_->CreateActor(ptr, buffer);
+  return RayActor<ReturnType>(returned_actor_id);
+}
+
+template <typename ReturnType, typename ActorType, typename FuncType,
+          typename ExecFuncType, typename... ArgTypes>
+inline RayObject<ReturnType> Ray::CallActorInternal(FuncType &actor_func,
+                                                    ExecFuncType &exec_func,
+                                                    RayActor<ActorType> &actor,
+                                                    ArgTypes &... args) {
+  std::shared_ptr<msgpack::sbuffer> buffer(new msgpack::sbuffer());
+  msgpack::packer<msgpack::sbuffer> packer(buffer.get());
+  Arguments::WrapArgs(packer, args...);
+  RemoteFunctionPtrHolder ptr;
+  MemberFunctionPtrHolder holder = *(MemberFunctionPtrHolder *)(&actor_func);
+  ptr.function_pointer = reinterpret_cast<uintptr_t>(holder.value[0]);
+  ptr.exec_function_pointer = reinterpret_cast<uintptr_t>(exec_func);
+  auto returned_object_id = runtime_->CallActor(ptr, actor.ID(), buffer);
+  return RayObject<ReturnType>(returned_object_id);
+}
+
+#include <ray/api/generated/exec_funcs.generated.h>
+
+#include <ray/api/generated/call_funcs_impl.generated.h>
+
+#include <ray/api/generated/create_actors_impl.generated.h>
+
+#include <ray/api/generated/call_actors_impl.generated.h>
+
+}  // namespace api
+}  // namespace ray
@@ -0,0 +1,84 @@
+
+#pragma once
+
+#include <ray/api/serializer.h>
+#include <msgpack.hpp>
+
+namespace ray {
+namespace api {
+
+class Arguments {
+ public:
+  static void WrapArgs(msgpack::packer<msgpack::sbuffer> &packer);
+
+  template <typename Arg1Type>
+  static void WrapArgs(msgpack::packer<msgpack::sbuffer> &packer, Arg1Type &arg1);
+
+  template <typename Arg1Type, typename... OtherArgTypes>
+  static void WrapArgs(msgpack::packer<msgpack::sbuffer> &packer, Arg1Type &arg1,
+                       OtherArgTypes &... args);
+
+  static void UnwrapArgs(msgpack::unpacker &unpacker);
+
+  template <typename Arg1Type>
+  static void UnwrapArgs(msgpack::unpacker &unpacker, std::shared_ptr<Arg1Type> *arg1);
+
+  template <typename Arg1Type, typename... OtherArgTypes>
+  static void UnwrapArgs(msgpack::unpacker &unpacker, std::shared_ptr<Arg1Type> *arg1,
+                         std::shared_ptr<OtherArgTypes> *... args);
+};
+
+// --------- inline implementation ------------
+#include <typeinfo>
+
+inline void Arguments::WrapArgs(msgpack::packer<msgpack::sbuffer> &packer) {}
+
+template <typename Arg1Type>
+inline void Arguments::WrapArgs(msgpack::packer<msgpack::sbuffer> &packer,
+                                Arg1Type &arg1) {
+  /// Notice RayObjectClassPrefix should be modified by RayObject class name or namespace.
+  static const std::string RayObjectClassPrefix = "N3ray3api9RayObject";
+  std::string type_name = typeid(arg1).name();
+  if (type_name.rfind(RayObjectClassPrefix, 0) == 0) {
+    /// Pass by reference.
+    Serializer::Serialize(packer, true);
+  } else {
+    /// Pass by value.
+    Serializer::Serialize(packer, false);
+  }
+  Serializer::Serialize(packer, arg1);
+}
+
+template <typename Arg1Type, typename... OtherArgTypes>
+inline void Arguments::WrapArgs(msgpack::packer<msgpack::sbuffer> &packer, Arg1Type &arg1,
+                                OtherArgTypes &... args) {
+  WrapArgs(packer, arg1);
+  WrapArgs(packer, args...);
+}
+
+inline void Arguments::UnwrapArgs(msgpack::unpacker &unpacker) {}
+
+template <typename Arg1Type>
+inline void Arguments::UnwrapArgs(msgpack::unpacker &unpacker,
+                                  std::shared_ptr<Arg1Type> *arg1) {
+  bool is_ray_object;
+  Serializer::Deserialize(unpacker, &is_ray_object);
+  if (is_ray_object) {
+    RayObject<Arg1Type> ray_object;
+    Serializer::Deserialize(unpacker, &ray_object);
+    *arg1 = ray_object.Get();
+  } else {
+    Serializer::Deserialize(unpacker, arg1);
+  }
+}
+
+template <typename Arg1Type, typename... OtherArgTypes>
+inline void Arguments::UnwrapArgs(msgpack::unpacker &unpacker,
+                                  std::shared_ptr<Arg1Type> *arg1,
+                                  std::shared_ptr<OtherArgTypes> *... args) {
+  UnwrapArgs(unpacker, arg1);
+  UnwrapArgs(unpacker, args...);
+}
+
+}  // namespace api
+}  // namespace ray
@@ -0,0 +1,41 @@
+/// This file is auto-generated. DO NOT EDIT.
+/// The following `Call` methods are used to call remote functions of actors.
+/// Their arguments and return types are as following:
+/// \param[in] actor_func The function pointer to be remote execution.
+/// \param[in] arg1...argn The function arguments passed by a value or RayObject.
+/// \return RayObject.
+
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ReturnType>
+RayObject<ReturnType> Call(ActorFunc0<ActorType, ReturnType> actor_func);
+// 1 arg
+template <typename ReturnType, typename Arg1Type>
+RayObject<ReturnType> Call(ActorFunc1<ActorType, ReturnType, Arg1Type> actor_func,
+                           Arg1Type arg1);
+
+template <typename ReturnType, typename Arg1Type>
+RayObject<ReturnType> Call(ActorFunc1<ActorType, ReturnType, Arg1Type> actor_func,
+                           RayObject<Arg1Type> &arg1);
+
+// 2 args
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func, Arg1Type arg1,
+    Arg2Type arg2);
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayObject<Arg1Type> &arg1, Arg2Type arg2);
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func, Arg1Type arg1,
+    RayObject<Arg2Type> &arg2);
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayObject<Arg1Type> &arg1, RayObject<Arg2Type> &arg2);
@@ -0,0 +1,57 @@
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ActorType>
+template <typename ReturnType>
+RayObject<ReturnType> RayActor<ActorType>::Call(
+    ActorFunc0<ActorType, ReturnType> actor_func) {
+  return Ray::Call(actor_func, *this);
+}
+
+// 1 arg
+template <typename ActorType>
+template <typename ReturnType, typename Arg1Type>
+RayObject<ReturnType> RayActor<ActorType>::Call(
+    ActorFunc1<ActorType, ReturnType, Arg1Type> actor_func, Arg1Type arg1) {
+  return Ray::Call(actor_func, *this, arg1);
+}
+
+template <typename ActorType>
+template <typename ReturnType, typename Arg1Type>
+RayObject<ReturnType> RayActor<ActorType>::Call(
+    ActorFunc1<ActorType, ReturnType, Arg1Type> actor_func, RayObject<Arg1Type> &arg1) {
+  return Ray::Call(actor_func, *this, arg1);
+}
+
+// 2 args
+template <typename ActorType>
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> RayActor<ActorType>::Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func, Arg1Type arg1,
+    Arg2Type arg2) {
+  return Ray::Call(actor_func, *this, arg1, arg2);
+}
+
+template <typename ActorType>
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> RayActor<ActorType>::Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayObject<Arg1Type> &arg1, Arg2Type arg2) {
+  return Ray::Call(actor_func, *this, arg1, arg2);
+}
+
+template <typename ActorType>
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> RayActor<ActorType>::Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func, Arg1Type arg1,
+    RayObject<Arg2Type> &arg2) {
+  return Ray::Call(actor_func, *this, arg1, arg2);
+}
+
+template <typename ActorType>
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> RayActor<ActorType>::Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayObject<Arg1Type> &arg1, RayObject<Arg2Type> &arg2) {
+  return Ray::Call(actor_func, *this, arg1, arg2);
+}
@@ -0,0 +1,15 @@
+
+
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ActorType, typename ReturnType>
+using ActorFunc0 = ReturnType (ActorType::*)();
+
+// 1 arg
+template <typename ActorType, typename ReturnType, typename Arg1Type>
+using ActorFunc1 = ReturnType (ActorType::*)(Arg1Type);
+
+// 2 args
+template <typename ActorType, typename ReturnType, typename Arg1Type, typename Arg2Type>
+using ActorFunc2 = ReturnType (ActorType::*)(Arg1Type, Arg2Type);
@@ -0,0 +1,38 @@
+
+
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ReturnType, typename ActorType>
+static RayObject<ReturnType> Call(ActorFunc0<ActorType, ReturnType> actor_func,
+                                  RayActor<ActorType> &actor);
+
+// 1 arg
+template <typename ReturnType, typename ActorType, typename Arg1Type>
+static RayObject<ReturnType> Call(ActorFunc1<ActorType, ReturnType, Arg1Type> actor_func,
+                                  RayActor<ActorType> &actor, Arg1Type arg1);
+
+template <typename ReturnType, typename ActorType, typename Arg1Type>
+static RayObject<ReturnType> Call(ActorFunc1<ActorType, ReturnType, Arg1Type> actor_func,
+                                  RayActor<ActorType> &actor, RayObject<Arg1Type> &arg1);
+
+// 2 args
+template <typename ReturnType, typename ActorType, typename Arg1Type, typename Arg2Type>
+static RayObject<ReturnType> Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayActor<ActorType> &actor, Arg1Type arg1, Arg2Type arg2);
+
+template <typename ReturnType, typename ActorType, typename Arg1Type, typename Arg2Type>
+static RayObject<ReturnType> Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayActor<ActorType> &actor, RayObject<Arg1Type> &arg1, Arg2Type arg2);
+
+template <typename ReturnType, typename ActorType, typename Arg1Type, typename Arg2Type>
+static RayObject<ReturnType> Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayActor<ActorType> &actor, Arg1Type arg1, RayObject<Arg2Type> &arg2);
+
+template <typename ReturnType, typename ActorType, typename Arg1Type, typename Arg2Type>
+static RayObject<ReturnType> Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayActor<ActorType> &actor, RayObject<Arg1Type> &arg1, RayObject<Arg2Type> &arg2);
@@ -0,0 +1,60 @@
+// TODO(Guyang Song): code generation
+// 0 args
+template <typename ReturnType, typename ActorType>
+RayObject<ReturnType> Ray::Call(ActorFunc0<ActorType, ReturnType> actor_func,
+                                RayActor<ActorType> &actor) {
+  return CallActorInternal<ReturnType, ActorType>(
+      actor_func, ActorExecFunction<ReturnType, ActorType>, actor);
+}
+
+// 1 arg
+template <typename ReturnType, typename ActorType, typename Arg1Type>
+RayObject<ReturnType> Ray::Call(ActorFunc1<ActorType, ReturnType, Arg1Type> actor_func,
+                                RayActor<ActorType> &actor, Arg1Type arg1) {
+  return CallActorInternal<ReturnType, ActorType>(
+      actor_func, ActorExecFunction<ReturnType, ActorType, Arg1Type>, actor, arg1);
+}
+
+template <typename ReturnType, typename ActorType, typename Arg1Type>
+RayObject<ReturnType> Ray::Call(ActorFunc1<ActorType, ReturnType, Arg1Type> actor_func,
+                                RayActor<ActorType> &actor, RayObject<Arg1Type> &arg1) {
+  return CallActorInternal<ReturnType, ActorType>(
+      actor_func, ActorExecFunction<ReturnType, ActorType, Arg1Type>, actor, arg1);
+}
+
+// 2 args
+template <typename ReturnType, typename ActorType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Ray::Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayActor<ActorType> &actor, Arg1Type arg1, Arg2Type arg2) {
+  return CallActorInternal<ReturnType, ActorType>(
+      actor_func, ActorExecFunction<ReturnType, ActorType, Arg1Type, Arg2Type>, actor,
+      arg1, arg2);
+}
+
+template <typename ReturnType, typename ActorType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Ray::Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayActor<ActorType> &actor, RayObject<Arg1Type> &arg1, Arg2Type arg2) {
+  return CallActorInternal<ReturnType, ActorType>(
+      actor_func, ActorExecFunction<ReturnType, ActorType, Arg1Type, Arg2Type>, actor,
+      arg1, arg2);
+}
+
+template <typename ReturnType, typename ActorType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Ray::Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayActor<ActorType> &actor, Arg1Type arg1, RayObject<Arg2Type> &arg2) {
+  return CallActorInternal<ReturnType, ActorType>(
+      actor_func, ActorExecFunction<ReturnType, ActorType, Arg1Type, Arg2Type>, actor,
+      arg1, arg2);
+}
+
+template <typename ReturnType, typename ActorType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Ray::Call(
+    ActorFunc2<ActorType, ReturnType, Arg1Type, Arg2Type> actor_func,
+    RayActor<ActorType> &actor, RayObject<Arg1Type> &arg1, RayObject<Arg2Type> &arg2) {
+  return CallActorInternal<ReturnType, ActorType>(
+      actor_func, ActorExecFunction<ReturnType, ActorType, Arg1Type, Arg2Type>, actor,
+      arg1, arg2);
+}
@@ -0,0 +1,37 @@
+/// This file is auto-generated. DO NOT EDIT.
+/// The following `Call` methods are used to call remote functions.
+/// Their arguments and return types are as following:
+/// \param[in] func The function pointer to be remote execution.
+/// \param[in] arg1...argn The function arguments passed by a value or RayObject.
+/// \return RayObject.
+
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ReturnType>
+static RayObject<ReturnType> Call(Func0<ReturnType> func);
+
+// 1 arg
+template <typename ReturnType, typename Arg1Type>
+static RayObject<ReturnType> Call(Func1<ReturnType, Arg1Type> func, Arg1Type arg1);
+
+template <typename ReturnType, typename Arg1Type>
+static RayObject<ReturnType> Call(Func1<ReturnType, Arg1Type> func,
+                                  RayObject<Arg1Type> &arg1);
+
+// 2 args
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+static RayObject<ReturnType> Call(Func2<ReturnType, Arg1Type, Arg2Type> func,
+                                  Arg1Type arg1, Arg2Type arg2);
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+static RayObject<ReturnType> Call(Func2<ReturnType, Arg1Type, Arg2Type> func,
+                                  RayObject<Arg1Type> &arg1, Arg2Type arg2);
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+static RayObject<ReturnType> Call(Func2<ReturnType, Arg1Type, Arg2Type> func,
+                                  Arg1Type arg1, RayObject<Arg2Type> &arg2);
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+static RayObject<ReturnType> Call(Func2<ReturnType, Arg1Type, Arg2Type> func,
+                                  RayObject<Arg1Type> &arg1, RayObject<Arg2Type> &arg2);
@@ -0,0 +1,48 @@
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ReturnType>
+RayObject<ReturnType> Ray::Call(Func0<ReturnType> func) {
+  return CallInternal<ReturnType>(func, NormalExecFunction<ReturnType>);
+}
+
+// 1 arg
+template <typename ReturnType, typename Arg1Type>
+RayObject<ReturnType> Ray::Call(Func1<ReturnType, Arg1Type> func, Arg1Type arg1) {
+  return CallInternal<ReturnType>(func, NormalExecFunction<ReturnType, Arg1Type>, arg1);
+}
+
+template <typename ReturnType, typename Arg1Type>
+RayObject<ReturnType> Ray::Call(Func1<ReturnType, Arg1Type> func,
+                                RayObject<Arg1Type> &arg1) {
+  return CallInternal<ReturnType>(func, NormalExecFunction<ReturnType, Arg1Type>, arg1);
+}
+
+// 2 args
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Ray::Call(Func2<ReturnType, Arg1Type, Arg2Type> func, Arg1Type arg1,
+                                Arg2Type arg2) {
+  return CallInternal<ReturnType>(
+      func, NormalExecFunction<ReturnType, Arg1Type, Arg2Type>, arg1, arg2);
+}
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Ray::Call(Func2<ReturnType, Arg1Type, Arg2Type> func,
+                                RayObject<Arg1Type> &arg1, Arg2Type arg2) {
+  return CallInternal<ReturnType>(
+      func, NormalExecFunction<ReturnType, Arg1Type, Arg2Type>, arg1, arg2);
+}
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Ray::Call(Func2<ReturnType, Arg1Type, Arg2Type> func, Arg1Type arg1,
+                                RayObject<Arg2Type> &arg2) {
+  return CallInternal<ReturnType>(
+      func, NormalExecFunction<ReturnType, Arg1Type, Arg2Type>, arg1, arg2);
+}
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayObject<ReturnType> Ray::Call(Func2<ReturnType, Arg1Type, Arg2Type> func,
+                                RayObject<Arg1Type> &arg1, RayObject<Arg2Type> &arg2) {
+  return CallInternal<ReturnType>(
+      func, NormalExecFunction<ReturnType, Arg1Type, Arg2Type>, arg1, arg2);
+}
@@ -0,0 +1,42 @@
+/// This file is auto-generated. DO NOT EDIT.
+/// The following `Call` methods are used to call remote functions and create an actor.
+/// Their arguments and return types are as following:
+/// \param[in] create_func The function pointer to be remote execution.
+/// \param[in] arg1...argn The function arguments passed by a value or RayObject.
+/// \return RayActor.
+
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ReturnType>
+static RayActor<ReturnType> CreateActor(CreateActorFunc0<ReturnType> create_func);
+
+// 1 arg
+template <typename ReturnType, typename Arg1Type>
+static RayActor<ReturnType> CreateActor(
+    CreateActorFunc1<ReturnType, Arg1Type> create_func, Arg1Type arg1);
+
+template <typename ReturnType, typename Arg1Type>
+static RayActor<ReturnType> CreateActor(
+    CreateActorFunc1<ReturnType, Arg1Type> create_func, RayObject<Arg1Type> &arg1);
+
+// 2 args
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+static RayActor<ReturnType> CreateActor(
+    CreateActorFunc2<ReturnType, Arg1Type, Arg2Type> create_func, Arg1Type arg1,
+    Arg2Type arg2);
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+static RayActor<ReturnType> CreateActor(
+    CreateActorFunc2<ReturnType, Arg1Type, Arg2Type> create_func,
+    RayObject<Arg1Type> &arg1, Arg2Type arg2);
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+static RayActor<ReturnType> CreateActor(
+    CreateActorFunc2<ReturnType, Arg1Type, Arg2Type> create_func, Arg1Type arg1,
+    RayObject<Arg2Type> &arg2);
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+static RayActor<ReturnType> CreateActor(
+    CreateActorFunc2<ReturnType, Arg1Type, Arg2Type> create_func,
+    RayObject<Arg1Type> &arg1, RayObject<Arg2Type> &arg2);
@@ -0,0 +1,56 @@
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ReturnType>
+RayActor<ReturnType> Ray::CreateActor(CreateActorFunc0<ReturnType> create_func) {
+  return CreateActorInternal<ReturnType>(create_func,
+                                         CreateActorExecFunction<ReturnType *>);
+}
+
+// 1 arg
+template <typename ReturnType, typename Arg1Type>
+RayActor<ReturnType> Ray::CreateActor(CreateActorFunc1<ReturnType, Arg1Type> create_func,
+                                      Arg1Type arg1) {
+  return CreateActorInternal<ReturnType>(
+      create_func, CreateActorExecFunction<ReturnType *, Arg1Type>, arg1);
+}
+
+template <typename ReturnType, typename Arg1Type>
+RayActor<ReturnType> Ray::CreateActor(CreateActorFunc1<ReturnType, Arg1Type> create_func,
+                                      RayObject<Arg1Type> &arg1) {
+  return CreateActorInternal<ReturnType>(
+      create_func, CreateActorExecFunction<ReturnType *, Arg1Type>, arg1);
+}
+
+// 2 args
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayActor<ReturnType> Ray::CreateActor(
+    CreateActorFunc2<ReturnType, Arg1Type, Arg2Type> create_func, Arg1Type arg1,
+    Arg2Type arg2) {
+  return CreateActorInternal<ReturnType>(
+      create_func, CreateActorExecFunction<ReturnType *, Arg1Type, Arg2Type>, arg1, arg2);
+}
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayActor<ReturnType> Ray::CreateActor(
+    CreateActorFunc2<ReturnType, Arg1Type, Arg2Type> create_func,
+    RayObject<Arg1Type> &arg1, Arg2Type arg2) {
+  return CreateActorInternal<ReturnType>(
+      create_func, CreateActorExecFunction<ReturnType *, Arg1Type, Arg2Type>, arg1, arg2);
+}
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayActor<ReturnType> Ray::CreateActor(
+    CreateActorFunc2<ReturnType, Arg1Type, Arg2Type> create_func, Arg1Type arg1,
+    RayObject<Arg2Type> &arg2) {
+  return CreateActorInternal<ReturnType>(
+      create_func, CreateActorExecFunction<ReturnType *, Arg1Type, Arg2Type>, arg1, arg2);
+}
+
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+RayActor<ReturnType> Ray::CreateActor(
+    CreateActorFunc2<ReturnType, Arg1Type, Arg2Type> create_func,
+    RayObject<Arg1Type> &arg1, RayObject<Arg2Type> &arg2) {
+  return CreateActorInternal<ReturnType>(
+      create_func, CreateActorExecFunction<ReturnType *, Arg1Type, Arg2Type>, arg1, arg2);
+}
@@ -0,0 +1,14 @@
+
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ReturnType>
+using CreateActorFunc0 = ReturnType *(*)();
+
+// 1 arg
+template <typename ReturnType, typename Arg1Type>
+using CreateActorFunc1 = ReturnType *(*)(Arg1Type);
+
+// 2 args
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+using CreateActorFunc2 = ReturnType *(*)(Arg1Type, Arg2Type);
@@ -0,0 +1,159 @@
+/// This file is auto-generated. DO NOT EDIT.
+/// The following execution functions are wrappers of remote functions.
+/// Execution functions make remote functions executable in distributed system.
+/// NormalExecFunction the wrapper of normal remote function.
+/// CreateActorExecFunction the wrapper of actor creation function.
+/// ActorExecFunction the wrapper of actor member function.
+
+// TODO(Guyang Song): code generation
+
+template <typename ReturnType, typename CastReturnType, typename... OtherArgTypes>
+std::shared_ptr<msgpack::sbuffer> ExecuteNormalFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer, TaskType task_type,
+    std::shared_ptr<OtherArgTypes> &... args) {
+  msgpack::unpacker unpacker;
+  unpacker.reserve_buffer(args_buffer->size());
+  memcpy(unpacker.buffer(), args_buffer->data(), args_buffer->size());
+  unpacker.buffer_consumed(args_buffer->size());
+  Arguments::UnwrapArgs(unpacker, &args...);
+
+  ReturnType return_value;
+  typedef ReturnType (*Func)(OtherArgTypes...);
+  Func func = (Func)(base_addr + func_offset);
+  return_value = (*func)(*args...);
+
+  std::shared_ptr<msgpack::sbuffer> returnBuffer(new msgpack::sbuffer());
+  msgpack::packer<msgpack::sbuffer> packer(returnBuffer.get());
+  Serializer::Serialize(packer, (CastReturnType)(return_value));
+
+  return returnBuffer;
+}
+
+template <typename ReturnType, typename ActorType, typename... OtherArgTypes>
+std::shared_ptr<msgpack::sbuffer> ExecuteActorFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer,
+    std::shared_ptr<msgpack::sbuffer> &actor_buffer,
+    std::shared_ptr<OtherArgTypes> &... args) {
+  msgpack::unpacker actor_unpacker;
+  actor_unpacker.reserve_buffer(actor_buffer->size());
+  memcpy(actor_unpacker.buffer(), actor_buffer->data(), actor_buffer->size());
+  actor_unpacker.buffer_consumed(actor_buffer->size());
+  uintptr_t actor_ptr;
+  Serializer::Deserialize(actor_unpacker, &actor_ptr);
+  ActorType *actor_object = (ActorType *)actor_ptr;
+
+  msgpack::unpacker unpacker;
+  unpacker.reserve_buffer(args_buffer->size());
+  memcpy(unpacker.buffer(), args_buffer->data(), args_buffer->size());
+  unpacker.buffer_consumed(args_buffer->size());
+  Arguments::UnwrapArgs(unpacker, &args...);
+
+  ReturnType return_value;
+  typedef ReturnType (ActorType::*Func)(OtherArgTypes...);
+  MemberFunctionPtrHolder holder;
+  holder.value[0] = base_addr + func_offset;
+  holder.value[1] = 0;
+  Func func = *((Func *)&holder);
+  return_value = (actor_object->*func)(*args...);
+
+  std::shared_ptr<msgpack::sbuffer> returnBuffer(new msgpack::sbuffer());
+  msgpack::packer<msgpack::sbuffer> packer(returnBuffer.get());
+  Serializer::Serialize(packer, return_value);
+  return returnBuffer;
+}
+
+// 0 args
+template <typename ReturnType>
+std::shared_ptr<msgpack::sbuffer> NormalExecFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer) {
+  return ExecuteNormalFunction<ReturnType, ReturnType>(
+      base_addr, func_offset, args_buffer, TaskType::NORMAL_TASK);
+}
+
+// 1 arg
+template <typename ReturnType, typename Arg1Type>
+std::shared_ptr<msgpack::sbuffer> NormalExecFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer) {
+  std::shared_ptr<Arg1Type> arg1_ptr;
+  return ExecuteNormalFunction<ReturnType, ReturnType>(
+      base_addr, func_offset, args_buffer, TaskType::NORMAL_TASK, arg1_ptr);
+}
+
+// 2 args
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+std::shared_ptr<msgpack::sbuffer> NormalExecFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer) {
+  std::shared_ptr<Arg1Type> arg1_ptr;
+  std::shared_ptr<Arg2Type> arg2_ptr;
+  return ExecuteNormalFunction<ReturnType, ReturnType>(
+      base_addr, func_offset, args_buffer, TaskType::NORMAL_TASK, arg1_ptr, arg2_ptr);
+}
+
+// 0 args
+template <typename ReturnType>
+std::shared_ptr<msgpack::sbuffer> CreateActorExecFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer) {
+  return ExecuteNormalFunction<ReturnType, uintptr_t>(base_addr, func_offset, args_buffer,
+                                                      TaskType::ACTOR_CREATION_TASK);
+}
+
+// 1 arg
+template <typename ReturnType, typename Arg1Type>
+std::shared_ptr<msgpack::sbuffer> CreateActorExecFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer) {
+  std::shared_ptr<Arg1Type> arg1_ptr;
+  return ExecuteNormalFunction<ReturnType, uintptr_t>(
+      base_addr, func_offset, args_buffer, TaskType::ACTOR_CREATION_TASK, arg1_ptr);
+}
+
+// 2 args
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+std::shared_ptr<msgpack::sbuffer> CreateActorExecFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer) {
+  std::shared_ptr<Arg1Type> arg1_ptr;
+  std::shared_ptr<Arg2Type> arg2_ptr;
+  return ExecuteNormalFunction<ReturnType, uintptr_t>(base_addr, func_offset, args_buffer,
+                                                      TaskType::ACTOR_CREATION_TASK,
+                                                      arg1_ptr, arg2_ptr);
+}
+
+// 0 args
+template <typename ReturnType, typename ActorType>
+std::shared_ptr<msgpack::sbuffer> ActorExecFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer,
+    std::shared_ptr<msgpack::sbuffer> &actor_buffer) {
+  return ExecuteActorFunction<ReturnType, ActorType>(base_addr, func_offset, args_buffer,
+                                                     actor_buffer);
+}
+
+// 1 arg
+template <typename ReturnType, typename ActorType, typename Arg1Type>
+std::shared_ptr<msgpack::sbuffer> ActorExecFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer,
+    std::shared_ptr<msgpack::sbuffer> &actor_buffer) {
+  std::shared_ptr<Arg1Type> arg1_ptr;
+  return ExecuteActorFunction<ReturnType, ActorType>(base_addr, func_offset, args_buffer,
+                                                     actor_buffer, arg1_ptr);
+}
+
+// 2 args
+template <typename ReturnType, typename ActorType, typename Arg1Type, typename Arg2Type>
+std::shared_ptr<msgpack::sbuffer> ActorExecFunction(
+    uintptr_t base_addr, size_t func_offset,
+    std::shared_ptr<msgpack::sbuffer> &args_buffer,
+    std::shared_ptr<msgpack::sbuffer> &actor_buffer) {
+  std::shared_ptr<Arg1Type> arg1_ptr;
+  std::shared_ptr<Arg2Type> arg2_ptr;
+  return ExecuteActorFunction<ReturnType, ActorType>(base_addr, func_offset, args_buffer,
+                                                     actor_buffer, arg1_ptr, arg2_ptr);
+}
@@ -0,0 +1,15 @@
+
+
+// TODO(Guyang Song): code generation
+
+// 0 args
+template <typename ReturnType>
+using Func0 = ReturnType (*)();
+
+// 1 arg
+template <typename ReturnType, typename Arg1Type>
+using Func1 = ReturnType (*)(Arg1Type);
+
+// 2 args
+template <typename ReturnType, typename Arg1Type, typename Arg2Type>
+using Func2 = ReturnType (*)(Arg1Type, Arg2Type);
@@ -0,0 +1,52 @@
+
+#pragma once
+
+#include "ray/core.h"
+
+namespace ray {
+namespace api {
+
+#include <ray/api/generated/actor_funcs.generated.h>
+
+/// A handle to an actor which can be used to invoke a remote actor method, with the
+/// `Call` method.
+/// \param ActorType The type of the concrete actor class.
+/// Note, the `Call` method is defined in actor_call.generated.h.
+template <typename ActorType>
+class RayActor {
+ public:
+  RayActor();
+
+  RayActor(const ActorID &id);
+
+  /// Get a untyped ID of the actor
+  const ActorID &ID() const;
+
+  /// Include the `Call` methods for calling remote functions.
+#include <ray/api/generated/actor_call.generated.h>
+
+  /// Make RayActor serializable
+  MSGPACK_DEFINE(id_);
+
+ private:
+  ActorID id_;
+};
+
+// ---------- implementation ----------
+
+template <typename ActorType>
+RayActor<ActorType>::RayActor() {}
+
+template <typename ActorType>
+RayActor<ActorType>::RayActor(const ActorID &id) {
+  id_ = id;
+}
+
+template <typename ActorType>
+const ActorID &RayActor<ActorType>::ID() const {
+  return id_;
+}
+
+#include <ray/api/generated/actor_call_impl.generated.h>
+}  // namespace api
+}  // namespace ray
@@ -0,0 +1,20 @@
+
+#pragma once
+
+namespace ray {
+namespace api {
+
+enum class RunMode { SINGLE_PROCESS, SINGLE_BOX, CLUSTER };
+
+enum class WorkerMode { NONE, DRIVER, WORKER };
+
+/// TODO(Guyang Song): Make configuration complete and use to initialize.
+class RayConfig {
+ public:
+  WorkerMode workerMode = WorkerMode::DRIVER;
+
+  RunMode runMode = RunMode::SINGLE_PROCESS;
+};
+
+}  // namespace api
+}  // namespace ray
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <exception>
+#include <string>
+
+namespace ray {
+namespace api {
+
+class RayException : public std::exception {
+ public:
+  RayException(const std::string &msg) : msg_(msg){};
+
+  const char *what() const noexcept override { return msg_.c_str(); };
+
+  std::string msg_;
+};
+}  // namespace api
+}  // namespace ray
@@ -0,0 +1,67 @@
+
+#pragma once
+
+#include <memory>
+#include <utility>
+
+#include <msgpack.hpp>
+
+#include "ray/core.h"
+
+namespace ray {
+namespace api {
+
+/// Represents an object in the object store..
+/// \param T The type of object.
+template <typename T>
+class RayObject {
+ public:
+  RayObject();
+
+  RayObject(const ObjectID &id);
+
+  bool operator==(const RayObject<T> &object) const;
+
+  /// Get a untyped ID of the object
+  const ObjectID &ID() const;
+
+  /// Get the object from the object store.
+  /// This method will be blocked until the object is ready.
+  ///
+  /// \return shared pointer of the result.
+  std::shared_ptr<T> Get() const;
+
+  /// Make RayObject serializable
+  MSGPACK_DEFINE(id_);
+
+ private:
+  ObjectID id_;
+};
+
+// ---------- implementation ----------
+#include <ray/api.h>
+
+template <typename T>
+RayObject<T>::RayObject() {}
+
+template <typename T>
+RayObject<T>::RayObject(const ObjectID &id) {
+  id_ = id;
+}
+
+template <typename T>
+inline bool RayObject<T>::operator==(const RayObject<T> &object) const {
+  return id_ == object.id_;
+}
+
+template <typename T>
+const ObjectID &RayObject<T>::ID() const {
+  return id_;
+}
+
+template <typename T>
+inline std::shared_ptr<T> RayObject<T>::Get() const {
+  return Ray::Get(*this);
+}
+}  // namespace api
+}  // namespace ray
@@ -0,0 +1,46 @@
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <msgpack.hpp>
+#include <typeinfo>
+#include <vector>
+
+#include <ray/api/wait_result.h>
+#include "ray/core.h"
+
+namespace ray {
+namespace api {
+
+struct MemberFunctionPtrHolder {
+  uintptr_t value[2];
+};
+
+struct RemoteFunctionPtrHolder {
+  /// The remote function pointer
+  uintptr_t function_pointer;
+  /// The executable function pointer
+  uintptr_t exec_function_pointer;
+};
+
+class RayRuntime {
+ public:
+  virtual ObjectID Put(std::shared_ptr<msgpack::sbuffer> data) = 0;
+  virtual std::shared_ptr<msgpack::sbuffer> Get(const ObjectID &id) = 0;
+
+  virtual std::vector<std::shared_ptr<msgpack::sbuffer>> Get(
+      const std::vector<ObjectID> &ids) = 0;
+
+  virtual WaitResult Wait(const std::vector<ObjectID> &ids, int num_objects,
+                          int timeout_ms) = 0;
+
+  virtual ObjectID Call(RemoteFunctionPtrHolder &fptr,
+                        std::shared_ptr<msgpack::sbuffer> args) = 0;
+  virtual ActorID CreateActor(RemoteFunctionPtrHolder &fptr,
+                              std::shared_ptr<msgpack::sbuffer> args) = 0;
+  virtual ObjectID CallActor(const RemoteFunctionPtrHolder &fptr, const ActorID &actor,
+                             std::shared_ptr<msgpack::sbuffer> args) = 0;
+};
+}  // namespace api
+}  // namespace ray
@@ -0,0 +1,41 @@
+
+#pragma once
+
+#include <ray/api/ray_exception.h>
+#include <msgpack.hpp>
+
+namespace ray {
+namespace api {
+
+class Serializer {
+ public:
+  template <typename T>
+  static void Serialize(msgpack::packer<msgpack::sbuffer> &packer, const T &val);
+
+  template <typename T>
+  static void Deserialize(msgpack::unpacker &unpacker, T *val);
+};
+
+// ---------- implementation ----------
+
+template <typename T>
+inline void Serializer::Serialize(msgpack::packer<msgpack::sbuffer> &packer,
+                                  const T &val) {
+  packer.pack(val);
+  return;
+}
+
+template <typename T>
+inline void Serializer::Deserialize(msgpack::unpacker &unpacker, T *val) {
+  msgpack::object_handle oh;
+  bool result = unpacker.next(oh);
+  if (result == false) {
+    throw RayException("unpack error");
+  }
+  msgpack::object obj = oh.get();
+  obj.convert(*val);
+  return;
+}
+
+}  // namespace api
+}  // namespace ray
@@ -0,0 +1,22 @@
+
+#pragma once
+
+#include <vector>
+#include "ray/core.h"
+
+namespace ray {
+namespace api {
+
+class WaitResult {
+ public:
+  /// The object id array of ready objects
+  std::vector<ObjectID> ready;
+  /// The object id array of unready objects
+  std::vector<ObjectID> unready;
+  WaitResult(){};
+  WaitResult(std::vector<ObjectID> &&ready_objects,
+             std::vector<ObjectID> &&unready_objects)
+      : ready(std::move(ready_objects)), unready(std::move(unready_objects)){};
+};
+}  // namespace api
+}  // namespace ray
@@ -0,0 +1,13 @@
+
+#pragma once
+
+#include "ray/common/buffer.h"
+#include "ray/common/function_descriptor.h"
+#include "ray/common/id.h"
+#include "ray/common/status.h"
+#include "ray/common/task/task_common.h"
+#include "ray/common/task/task_spec.h"
+#include "ray/common/task/task_util.h"
+#include "ray/core_worker/context.h"
+#include "ray/core_worker/store_provider/memory_store/memory_store.h"
+#include "ray/util/logging.h"