wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-09 11:32:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

implement event based object store

Browse Source
This commit is contained in:
Philipp Moritz
2016-03-17 22:32:31 -07:00
parent b9d472342b
commit dd88bae487
10 changed files with 332 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/objstore.h
+17 -20
View File
@@ -1,17 +1,16 @@
#ifndef ORCHESTRA_OBJSTORE_SERVER_H
#define ORCHESTRA_OBJSTORE_SERVER_H
#ifndef ORCHESTRA_OBJSTORE_H
#define ORCHESTRA_OBJSTORE_H
#include <unordered_map>
#include <memory>
#include <thread>
#include <iostream>
#include <boost/interprocess/managed_shared_memory.hpp>
#include <grpc++/grpc++.h>
using namespace boost::interprocess;
#include "orchestra/orchestra.h"
#include "orchestra.grpc.pb.h"
#include "types.pb.h"
#include "ipc.h"
using grpc::Server;
using grpc::ServerBuilder;
@@ -29,34 +28,32 @@ public:
static Status upload_data_to(slice data, ObjRef objref, ObjStore::Stub& stub);
};
struct shared_object {
std::string name;
std::shared_ptr<managed_shared_memory> memory;
slice ptr;
};
class ObjStoreService final : public ObjStore::Service {
public:
ObjStoreService(const std::string& objstore_address, std::shared_ptr<Channel> scheduler_channel);
~ObjStoreService();
Status DeliverObj(ServerContext* context, const DeliverObjRequest* request, AckReply* reply) override;
// Status DeliverObj(ServerContext* context, const DeliverObjRequest* request, AckReply* reply) override;
// Status StreamObj(ServerContext* context, ServerReader<ObjChunk>* reader, AckReply* reply) override;
Status ObjStoreDebugInfo(ServerContext* context, const ObjStoreDebugInfoRequest* request, ObjStoreDebugInfoReply* reply) override;
Status GetObj(ServerContext* context, const GetObjRequest* request, GetObjReply* reply) override;
Status StreamObj(ServerContext* context, ServerReader<ObjChunk>* reader, AckReply* reply) override;
void start_objstore_service();
private:
void allocate_memory(ObjRef objref, size_t size);
// check if we already connected to the other objstore, if yes, return reference to connection, otherwise connect
ObjStore::Stub& get_objstore_stub(const std::string& objstore_address);
void process_requests();
std::vector<std::string> memory_names_;
std::unordered_map<ObjRef, shared_object> memory_;
std::string objstore_address_;
ObjStoreId objstoreid_; // id of this objectstore in the scheduler object store table
MemorySegmentPool segmentpool_;
std::vector<std::pair<ObjHandle, bool> > memory_; // object reference -> (memory address, memory finalized?)
std::mutex memory_lock_;
size_t page_size = mapped_region::get_page_size();
std::unordered_map<std::string, std::unique_ptr<ObjStore::Stub>> objstores_;
std::mutex objstores_lock_;
std::unique_ptr<Scheduler::Stub> scheduler_stub_;
ObjStoreId objstoreid_; // id of this objectstore in the scheduler object store table
std::vector<std::pair<WorkerId, ObjRef> > pull_queue_;
std::mutex pull_queue_lock_;
MessageQueue<ObjRequest> recv_queue_;
std::vector<MessageQueue<ObjHandle> > send_queues_; // workerid -> queue
std::thread communicator_thread_;
};
#endif