wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-15 12:45:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

support multiple object stores, part one

Browse Source
This commit is contained in:
Robert Nishihara
2016-04-24 19:06:14 -07:00
parent a12e6fd373
commit 6b846d2bc0
8 changed files with 111 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ipc.cc
+8 -3
View File
@@ -31,9 +31,9 @@ MemorySegmentPool::MemorySegmentPool(ObjStoreId objstoreid, bool create) : objst
// creates a memory segment if it is not already there; if the pool is in create mode,
// space is allocated, if it is in open mode, the shared memory is mapped into the process
void MemorySegmentPool::open_segment(SegmentId segmentid, size_t size) {
ORCH_LOG(ORCH_DEBUG, "opening segmentid " << segmentid);
ORCH_LOG(ORCH_DEBUG, "Opening segmentid " << segmentid << " on object store " << objstoreid_ << " with create_mode_ = " << create_mode_);
if (segmentid != segments_.size() && create_mode_) {
ORCH_LOG(ORCH_FATAL, "Attempting to open segmentid " << segmentid << " on the object store, but segments_.size() = " << segments_.size());
ORCH_LOG(ORCH_FATAL, "Object store " << objstoreid_ << " is attempting to open segmentid " << segmentid << " on the object store, but segments_.size() = " << segments_.size());
}
if (segmentid >= segments_.size()) { // resize and initialize segments_
int current_size = segments_.size();
@@ -90,7 +90,12 @@ void MemorySegmentPool::deallocate(ObjHandle pointer) {
// returns address of the object refered to by the handle, needs to be called on
// the process that will use the address
uint8_t* MemorySegmentPool::get_address(ObjHandle pointer) {
open_segment(pointer.segmentid());
if (create_mode_ && segments_[pointer.segmentid()].second != SegmentStatusType::OPENED) {
ORCH_LOG(ORCH_FATAL, "Object store " << objstoreid_ << " is attempting to call get_address on segmentid " << pointer.segmentid() << ", which has not been opened yet.");
}
if (!create_mode_) {
open_segment(pointer.segmentid());
}
managed_shared_memory* segment = segments_[pointer.segmentid()].first.get();
return static_cast<uint8_t*>(segment->get_address_from_handle(pointer.ipcpointer()));
}
src/objstore.h
+26 -12
View File
@@ -17,40 +17,50 @@ using grpc::ServerBuilder;
using grpc::ServerReader;
using grpc::ServerContext;
using grpc::ClientContext;
using grpc::ClientWriter;
using grpc::ServerWriter;
using grpc::ClientReader;
using grpc::Status;
using grpc::Channel;
class ObjStoreClient {
public:
static const size_t CHUNK_SIZE;
static Status upload_data_to(slice data, ObjRef objref, ObjStore::Stub& stub);
};
enum MemoryStatusType {READY = 0, NOT_READY = 1, DEALLOCATED = 2, NOT_PRESENT = 3};
// READY: This is used to indicate that the object has been copied from a
// worker and is ready to be used.
// NOT_READY: This is used to indicate that memory has been allocated for the
// object, but the object hasn't been copied from a worker yet.
// DEALLOCATED: This is used to indicate that the object has been deallocated.
// NOT_PRESENT: This is used to indicate that space has not been allocated for
// this object in this object store.
// PRE_ALLOCED: This is used to indicate that the memory has not yet been
// alloced, but it will be alloced soon. This is set when we call
// StartDelivery.
enum MemoryStatusType {READY = 0, NOT_READY = 1, DEALLOCATED = 2, NOT_PRESENT = 3, PRE_ALLOCED = 4};
class ObjStoreService final : public ObjStore::Service {
public:
ObjStoreService(const std::string& objstore_address, std::shared_ptr<Channel> scheduler_channel);
// Status DeliverObj(ServerContext* context, const DeliverObjRequest* request, AckReply* reply) override;
// Status StreamObj(ServerContext* context, ServerReader<ObjChunk>* reader, AckReply* reply) override;
Status StartDelivery(ServerContext* context, const StartDeliveryRequest* request, AckReply* reply) override;
Status StreamObjTo(ServerContext* context, const StreamObjToRequest* request, ServerWriter<ObjChunk>* writer) override;
Status NotifyAlias(ServerContext* context, const NotifyAliasRequest* request, AckReply* reply) override;
Status DeallocateObject(ServerContext* context, const DeallocateObjectRequest* request, AckReply* reply) override;
Status ObjStoreInfo(ServerContext* context, const ObjStoreInfoRequest* request, ObjStoreInfoReply* reply) override;
void start_objstore_service();
private:
void pull_data_from(ObjRef objref, ObjStore::Stub& stub);
// check if we already connected to the other objstore, if yes, return reference to connection, otherwise connect
ObjStore::Stub& get_objstore_stub(const std::string& objstore_address);
void process_worker_request(const ObjRequest request);
void process_objstore_request(const ObjRequest request);
void process_requests();
void process_pulls_for_objref(ObjRef objref);
ObjHandle alloc(ObjRef objref, size_t size);
void object_ready(ObjRef objref, size_t metadata_offset);
static const size_t CHUNK_SIZE;
std::string objstore_address_;
ObjStoreId objstoreid_; // id of this objectstore in the scheduler object store table
std::shared_ptr<MemorySegmentPool> segmentpool_;
std::mutex segmentpool_lock_;
std::vector<std::pair<ObjHandle, MemoryStatusType> > memory_; // object reference -> (memory address, memory status)
std::mutex memory_lock_;
std::unordered_map<std::string, std::unique_ptr<ObjStore::Stub>> objstores_;
@@ -58,9 +68,13 @@ private:
std::unique_ptr<Scheduler::Stub> scheduler_stub_;
std::vector<std::pair<WorkerId, ObjRef> > pull_queue_;
std::mutex pull_queue_lock_;
MessageQueue<ObjRequest> recv_queue_;
std::vector<MessageQueue<ObjHandle> > send_queues_; // workerid -> queue
MessageQueue<ObjRequest> recv_queue_; // This queue is used by workers to send tasks to the object store.
std::vector<MessageQueue<ObjHandle> > send_queues_; // This maps workerid -> queue. The object store uses these queues to send replies to the relevant workers.
std::thread communicator_thread_;
std::vector<std::shared_ptr<std::thread> > delivery_threads_; // TODO(rkn): document
// TODO(rkn): possibly add lock, and properly remove these threads from the delivery_threads_ when the deliveries are done
};
#endif
src/scheduler.cc
+20 -11
View File
@@ -191,8 +191,14 @@ Status SchedulerService::SchedulerInfo(ServerContext* context, const SchedulerIn
return Status::OK;
}
// TODO(rkn): This could execute multiple times with the same arguments before
// the delivery finishes, but we only want it to happen once. Currently, the
// redundancy is handled by the object store, which will only execute the
// delivery once. However, we may want to handle it in the scheduler in the
// future.
//
// deliver_object assumes that the aliasing for objref has already been completed. That is, has_canonical_objref(objref) == true
void SchedulerService::deliver_object(ObjRef objref, ObjStoreId from, ObjStoreId to) {
// deliver_object assumes that the aliasing for objref has already been completed. That is, has_canonical_objref(objref) == true
if (from == to) {
ORCH_LOG(ORCH_FATAL, "attempting to deliver objref " << objref << " from objstore " << from << " to itself.");
}
@@ -201,12 +207,12 @@ void SchedulerService::deliver_object(ObjRef objref, ObjStoreId from, ObjStoreId
}
ClientContext context;
AckReply reply;
DeliverObjRequest request;
StartDeliveryRequest request;
ObjRef canonical_objref = get_canonical_objref(objref);
request.set_objref(canonical_objref);
std::lock_guard<std::mutex> lock(objstores_lock_);
request.set_objstore_address(objstores_[to].address);
objstores_[from].objstore_stub->DeliverObj(&context, request, &reply);
request.set_objstore_address(objstores_[from].address);
objstores_[to].objstore_stub->StartDelivery(&context, request, &reply);
}
void SchedulerService::schedule() {
@@ -403,14 +409,16 @@ void SchedulerService::get_info(const SchedulerInfoRequest& request, SchedulerIn
}
// pick_objstore assumes that objtable_lock_ has been acquired
// pick_objstore must be called with a canonical_objref
ObjStoreId SchedulerService::pick_objstore(ObjRef canonical_objref) {
// pick_objstore must be called with a canonical_objref
std::mt19937 rng;
if (!is_canonical(canonical_objref)) {
ORCH_LOG(ORCH_FATAL, "Attempting to call pick_objstore with a non-canonical objref, (objref " << canonical_objref << ")");
}
std::uniform_int_distribution<int> uni(0, objtable_[canonical_objref].size() - 1);
return uni(rng);
ObjStoreId objstoreid = objtable_[canonical_objref][uni(rng)];
return objstoreid;
}
bool SchedulerService::is_canonical(ObjRef objref) {
@@ -433,7 +441,7 @@ void SchedulerService::perform_pulls() {
continue;
}
ObjRef canonical_objref = get_canonical_objref(objref);
ORCH_LOG(ORCH_DEBUG, "attempting to pull objref " << pull.second << " with canonical objref " << canonical_objref);
ORCH_LOG(ORCH_DEBUG, "attempting to pull objref " << pull.second << " with canonical objref " << canonical_objref << " to objstore " << get_store(workerid));
objtable_lock_.lock();
int num_stores = objtable_[canonical_objref].size();
@@ -570,16 +578,17 @@ void SchedulerService::deallocate_object(ObjRef canonical_objref) {
// so the lock must before outside of these methods (it is acquired in
// DecrementRefCount).
ORCH_LOG(ORCH_REFCOUNT, "Deallocating canonical_objref " << canonical_objref << ".");
ClientContext context;
AckReply reply;
DeallocateObjectRequest request;
request.set_canonical_objref(canonical_objref);
{
std::lock_guard<std::mutex> objtable_lock(objtable_lock_);
auto &objstores = objtable_[canonical_objref];
std::lock_guard<std::mutex> objstores_lock(objstores_lock_); // TODO(rkn): Should this be inside the for loop instead?
for (int i = 0; i < objstores.size(); ++i) {
ClientContext context;
AckReply reply;
DeallocateObjectRequest request;
request.set_canonical_objref(canonical_objref);
ObjStoreId objstoreid = objstores[i];
ORCH_LOG(ORCH_REFCOUNT, "Attempting to deallocate canonical_objref " << canonical_objref << " from objstore " << objstoreid);
objstores_[objstoreid].objstore_stub->DeallocateObject(&context, request, &reply);
}
}
src/serialize.cc
+1 -1
View File
@@ -156,7 +156,7 @@ std::shared_ptr<arrow::Array> read_flat_array(BufferMemorySource* source, int64_
std::shared_ptr<ipc::RowBatchReader> reader;
Status s = ipc::RowBatchReader::Open(source, metadata_offset, &reader);
if (!s.ok()) {
ORCH_LOG(ORCH_FATAL, s.ToString());
ORCH_LOG(ORCH_FATAL, "Error in read_flat_array: " << s.ToString());
}
auto field = std::make_shared<arrow::Field>("data", npy_traits<NpyType>::primitive_type);
std::shared_ptr<arrow::Schema> schema(new arrow::Schema({field}));