wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-10 12:38:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

[core worker] Python core worker object interface (#5272)

Browse Source
This commit is contained in:
Edward Oakes
2019-09-12 23:07:46 -07:00
committed by Eric Liang
parent 1b880191b0
commit 07c4c6367a
49 changed files with 1157 additions and 552 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/common/status.cc
+6
View File
@@ -59,6 +59,12 @@ std::string Status::CodeAsString() const {
case StatusCode::IOError:
type = "IOError";
break;
case StatusCode::ObjectExists:
type = "ObjectExists";
break;
case StatusCode::ObjectStoreFull:
type = "ObjectStoreFull";
break;
case StatusCode::UnknownError:
type = "Unknown error";
break;
src/ray/common/status.h
+12 -6
View File
@@ -75,9 +75,10 @@ enum class StatusCode : char {
Invalid = 4,
IOError = 5,
ObjectExists = 6,
ObjectStoreFull = 7,
UnknownError = 9,
NotImplemented = 10,
RedisError = 11
RedisError = 11,
};
#if defined(__clang__)
@@ -129,14 +130,18 @@ class RAY_EXPORT Status {
return Status(StatusCode::IOError, msg);
}
static Status RedisError(const std::string &msg) {
return Status(StatusCode::RedisError, msg);
}
static Status ObjectExists(const std::string &msg) {
return Status(StatusCode::ObjectExists, msg);
}
static Status ObjectStoreFull(const std::string &msg) {
return Status(StatusCode::ObjectStoreFull, msg);
}
static Status RedisError(const std::string &msg) {
return Status(StatusCode::RedisError, msg);
}
// Returns true iff the status indicates success.
bool ok() const { return (state_ == NULL); }
@@ -144,11 +149,12 @@ class RAY_EXPORT Status {
bool IsKeyError() const { return code() == StatusCode::KeyError; }
bool IsInvalid() const { return code() == StatusCode::Invalid; }
bool IsIOError() const { return code() == StatusCode::IOError; }
bool IsObjectExists() const { return code() == StatusCode::ObjectExists; }
bool IsObjectStoreFull() const { return code() == StatusCode::ObjectStoreFull; }
bool IsTypeError() const { return code() == StatusCode::TypeError; }
bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
bool IsRedisError() const { return code() == StatusCode::RedisError; }
bool IsObjectExists() const { return code() == StatusCode::ObjectExists; }
// Return a string representation of this status suitable for printing.
// Returns the string "OK" for success.
src/ray/core_worker/context.cc
+8
View File
@@ -74,6 +74,14 @@ const TaskID &WorkerContext::GetCurrentTaskID() const {
return GetThreadContext().GetCurrentTaskID();
}
// TODO(edoakes): remove this once Python core worker uses the task interfaces.
void WorkerContext::SetCurrentJobId(const JobID &job_id) { current_job_id_ = job_id; }
// TODO(edoakes): remove this once Python core worker uses the task interfaces.
void WorkerContext::SetCurrentTaskId(const TaskID &task_id) {
GetThreadContext().SetCurrentTaskId(task_id);
}
void WorkerContext::SetCurrentTask(const TaskSpecification &task_spec) {
current_job_id_ = task_spec.JobId();
GetThreadContext().SetCurrentTask(task_spec);
src/ray/core_worker/context.h
+6
View File
@@ -20,6 +20,12 @@ class WorkerContext {
const TaskID &GetCurrentTaskID() const;
// TODO(edoakes): remove this once Python core worker uses the task interfaces.
void SetCurrentJobId(const JobID &job_id);
// TODO(edoakes): remove this once Python core worker uses the task interfaces.
void SetCurrentTaskId(const TaskID &task_id);
void SetCurrentTask(const TaskSpecification &task_spec);
std::shared_ptr<const TaskSpecification> GetCurrentTask() const;
src/ray/core_worker/core_worker.cc
+46 -10
View File
@@ -7,30 +7,58 @@ CoreWorker::CoreWorker(
const WorkerType worker_type, const Language language,
const std::string &store_socket, const std::string &raylet_socket,
const JobID &job_id, const gcs::GcsClientOptions &gcs_options,
const CoreWorkerTaskExecutionInterface::TaskExecutor &execution_callback)
const std::string &log_dir,
const CoreWorkerTaskExecutionInterface::TaskExecutor &execution_callback,
bool use_memory_store)
: worker_type_(worker_type),
language_(language),
raylet_socket_(raylet_socket),
log_dir_(log_dir),
worker_context_(worker_type, job_id),
io_work_(io_service_) {
// Initialize gcs client
// Initialize logging if log_dir is passed. Otherwise, it must be initialized
// and cleaned up by the caller.
if (!log_dir_.empty()) {
std::stringstream app_name;
if (language_ == Language::PYTHON) {
app_name << "python-";
} else if (language == Language::JAVA) {
app_name << "java-";
}
if (worker_type_ == WorkerType::DRIVER) {
app_name << "core-driver-" << worker_context_.GetWorkerID();
} else {
app_name << "core-worker-" << worker_context_.GetWorkerID();
}
RayLog::StartRayLog(app_name.str(), RayLogLevel::INFO, log_dir_);
RayLog::InstallFailureSignalHandler();
}
// Initialize gcs client.
gcs_client_ =
std::unique_ptr<gcs::RedisGcsClient>(new gcs::RedisGcsClient(gcs_options));
RAY_CHECK_OK(gcs_client_->Connect(io_service_));
object_interface_ = std::unique_ptr<CoreWorkerObjectInterface>(
new CoreWorkerObjectInterface(worker_context_, raylet_client_, store_socket));
object_interface_ =
std::unique_ptr<CoreWorkerObjectInterface>(new CoreWorkerObjectInterface(
worker_context_, raylet_client_, store_socket, use_memory_store));
task_interface_ = std::unique_ptr<CoreWorkerTaskInterface>(new CoreWorkerTaskInterface(
worker_context_, raylet_client_, *object_interface_, io_service_, *gcs_client_));
// Initialize task execution.
int rpc_server_port = 0;
if (worker_type_ == WorkerType::WORKER) {
RAY_CHECK(execution_callback != nullptr);
task_execution_interface_ = std::unique_ptr<CoreWorkerTaskExecutionInterface>(
new CoreWorkerTaskExecutionInterface(worker_context_, raylet_client_,
*object_interface_, execution_callback));
rpc_server_port = task_execution_interface_->worker_server_.GetPort();
// TODO(edoakes): Remove this check once Python core worker migration is complete.
if (language != Language::PYTHON || execution_callback != nullptr) {
RAY_CHECK(execution_callback != nullptr);
task_execution_interface_ = std::unique_ptr<CoreWorkerTaskExecutionInterface>(
new CoreWorkerTaskExecutionInterface(worker_context_, raylet_client_,
*object_interface_, execution_callback));
rpc_server_port = task_execution_interface_->worker_server_.GetPort();
}
}
// Initialize raylet client.
// TODO(zhijunfu): currently RayletClient would crash in its constructor if it cannot
// connect to Raylet after a number of retries, this can be changed later
// so that the worker (java/python .etc) can retrieve and handle the error
@@ -44,12 +72,20 @@ CoreWorker::CoreWorker(
}
CoreWorker::~CoreWorker() {
gcs_client_->Disconnect();
io_service_.stop();
io_thread_.join();
if (task_execution_interface_) {
task_execution_interface_->Stop();
}
if (!log_dir_.empty()) {
RayLog::ShutDownRayLog();
}
}
void CoreWorker::Disconnect() {
if (gcs_client_) {
gcs_client_->Disconnect();
}
if (raylet_client_) {
RAY_IGNORE_EXPR(raylet_client_->Disconnect());
}
src/ray/core_worker/core_worker.h
+30 -23
View File
@@ -20,16 +20,32 @@ class CoreWorker {
/// Construct a CoreWorker instance.
///
/// \param[in] worker_type Type of this worker.
/// \param[in] langauge Language of this worker.
/// \param[in] language Language of this worker.
/// \param[in] store_socket Object store socket to connect to.
/// \param[in] raylet_socket Raylet socket to connect to.
/// \param[in] job_id Job ID of this worker.
/// \param[in] gcs_options Options for the GCS client.
/// \param[in] log_dir Directory to write logs to. If this is empty, logs
/// won't be written to a file.
/// \param[in] execution_callback Language worker callback to execute tasks.
/// \param[in] use_memory_store Whether or not to use the in-memory object store
/// in addition to the plasma store.
///
/// NOTE(zhijunfu): the constructor would throw if a failure happens.
/// NOTE(edoakes): the use_memory_store flag is a stop-gap solution to the issue
/// that randomly generated ObjectIDs may use the memory store
/// instead of the plasma store.
CoreWorker(const WorkerType worker_type, const Language language,
const std::string &store_socket, const std::string &raylet_socket,
const JobID &job_id, const gcs::GcsClientOptions &gcs_options,
const CoreWorkerTaskExecutionInterface::TaskExecutor &execution_callback);
const std::string &log_dir,
const CoreWorkerTaskExecutionInterface::TaskExecutor &execution_callback,
bool use_memory_store = true);
~CoreWorker();
void Disconnect();
/// Type of this worker.
WorkerType GetWorkerType() const { return worker_type_; }
@@ -55,44 +71,35 @@ class CoreWorker {
return *task_execution_interface_;
}
// TODO(edoakes): remove this once Python core worker uses the task interfaces.
void SetCurrentJobId(const JobID &job_id) { worker_context_.SetCurrentJobId(job_id); }
// TODO(edoakes): remove this once Python core worker uses the task interfaces.
void SetCurrentTaskId(const TaskID &task_id) {
worker_context_.SetCurrentTaskId(task_id);
}
private:
void StartIOService();
/// Type of this worker.
const WorkerType worker_type_;
/// Language of this worker.
const Language language_;
/// raylet socket name.
const std::string raylet_socket_;
/// Worker context.
const std::string log_dir_;
WorkerContext worker_context_;
/// event loop where the IO events are handled. e.g. async GCS operations.
/// Event loop where the IO events are handled. e.g. async GCS operations.
boost::asio::io_service io_service_;
/// keeps io_service_ alive.
/// Keeps the io_service_ alive.
boost::asio::io_service::work io_work_;
/// The thread to handle IO events.
std::thread io_thread_;
/// Raylet client.
std::unique_ptr<RayletClient> raylet_client_;
/// GCS client.
std::unique_ptr<gcs::RedisGcsClient> gcs_client_;
/// The `CoreWorkerTaskInterface` instance.
std::unique_ptr<CoreWorkerTaskInterface> task_interface_;
/// The `CoreWorkerObjectInterface` instance.
std::unique_ptr<CoreWorkerObjectInterface> object_interface_;
/// The `CoreWorkerTaskExecutionInterface` instance.
/// This is only available if it's not a driver.
/// Only available if it's not a driver.
std::unique_ptr<CoreWorkerTaskExecutionInterface> task_execution_interface_;
};
src/ray/core_worker/lib/java/org_ray_runtime_RayNativeRuntime.cc
+4 -2
View File
@@ -71,7 +71,7 @@ JNIEXPORT jlong JNICALL Java_org_ray_runtime_RayNativeRuntime_nativeInitCoreWork
try {
auto core_worker = new ray::CoreWorker(
static_cast<ray::WorkerType>(workerMode), ::Language::JAVA, native_store_socket,
native_raylet_socket, job_id, gcs_client_options, executor_func);
native_raylet_socket, job_id, gcs_client_options, /*log_dir=*/"", executor_func);
return reinterpret_cast<jlong>(core_worker);
} catch (const std::exception &e) {
std::ostringstream oss;
@@ -103,7 +103,9 @@ JNIEXPORT void JNICALL Java_org_ray_runtime_RayNativeRuntime_nativeRunTaskExecut
*/
JNIEXPORT void JNICALL Java_org_ray_runtime_RayNativeRuntime_nativeDestroyCoreWorker(
JNIEnv *env, jclass o, jlong nativeCoreWorkerPointer) {
delete reinterpret_cast<ray::CoreWorker *>(nativeCoreWorkerPointer);
auto core_worker = reinterpret_cast<ray::CoreWorker *>(nativeCoreWorkerPointer);
core_worker->Disconnect();
delete core_worker;
}
/*
src/ray/core_worker/object_interface.cc
+32 -3
View File
@@ -8,7 +8,7 @@
namespace ray {
// Group object ids according the the corresponding store providers.
void GroupObjectIdsByStoreProvider(
void CoreWorkerObjectInterface::GroupObjectIdsByStoreProvider(
const std::vector<ObjectID> &object_ids,
EnumUnorderedMap<StoreProviderType, std::unordered_set<ObjectID>> *results) {
// There are two cases:
@@ -25,7 +25,7 @@ void GroupObjectIdsByStoreProvider(
// and are only used locally.
// Thus we need to check whether this object is a task return object in additional
// to whether it's from direct actor call before we can choose memory store provider.
if (object_id.IsReturnObject() &&
if (use_memory_store_ && object_id.IsReturnObject() &&
object_id.GetTransportType() ==
static_cast<uint8_t>(TaskTransportType::DIRECT_ACTOR)) {
type = StoreProviderType::MEMORY;
@@ -37,15 +37,22 @@ void GroupObjectIdsByStoreProvider(
CoreWorkerObjectInterface::CoreWorkerObjectInterface(
WorkerContext &worker_context, std::unique_ptr<RayletClient> &raylet_client,
const std::string &store_socket)
const std::string &store_socket, bool use_memory_store)
: worker_context_(worker_context),
raylet_client_(raylet_client),
store_socket_(store_socket),
use_memory_store_(use_memory_store),
memory_store_(std::make_shared<CoreWorkerMemoryStore>()) {
AddStoreProvider(StoreProviderType::PLASMA);
AddStoreProvider(StoreProviderType::MEMORY);
}
Status CoreWorkerObjectInterface::SetClientOptions(std::string name,
int64_t limit_bytes) {
// Currently only the Plasma store supports client options.
return store_providers_[StoreProviderType::PLASMA]->SetClientOptions(name, limit_bytes);
}
Status CoreWorkerObjectInterface::Put(const RayObject &object, ObjectID *object_id) {
ObjectID put_id = ObjectID::ForPut(worker_context_.GetCurrentTaskID(),
worker_context_.GetNextPutIndex(),
@@ -62,6 +69,18 @@ Status CoreWorkerObjectInterface::Put(const RayObject &object,
return store_providers_[StoreProviderType::PLASMA]->Put(object, object_id);
}
Status CoreWorkerObjectInterface::Create(const std::shared_ptr<Buffer> &metadata,
const size_t data_size,
const ObjectID &object_id,
std::shared_ptr<Buffer> *data) {
return store_providers_[StoreProviderType::PLASMA]->Create(metadata, data_size,
object_id, data);
}
Status CoreWorkerObjectInterface::Seal(const ObjectID &object_id) {
return store_providers_[StoreProviderType::PLASMA]->Seal(object_id);
}
Status CoreWorkerObjectInterface::Get(const std::vector<ObjectID> &ids,
int64_t timeout_ms,
std::vector<std::shared_ptr<RayObject>> *results) {
@@ -124,6 +143,11 @@ Status CoreWorkerObjectInterface::Get(const std::vector<ObjectID> &ids,
return Status::OK();
}
Status CoreWorkerObjectInterface::Contains(const ObjectID &object_id, bool *has_object) {
// Currently only the Plasma store supports Contains().
return store_providers_[StoreProviderType::PLASMA]->Contains(object_id, has_object);
}
Status CoreWorkerObjectInterface::Wait(const std::vector<ObjectID> &ids, int num_objects,
int64_t timeout_ms, std::vector<bool> *results) {
(*results).resize(ids.size(), false);
@@ -231,6 +255,11 @@ Status CoreWorkerObjectInterface::Delete(const std::vector<ObjectID> &object_ids
return Status::OK();
}
std::string CoreWorkerObjectInterface::MemoryUsageString() {
// Currently only the Plasma store returns a debug string.
return store_providers_[StoreProviderType::PLASMA]->MemoryUsageString();
}
void CoreWorkerObjectInterface::AddStoreProvider(StoreProviderType type) {
store_providers_.emplace(type, CreateStoreProvider(type));
}
src/ray/core_worker/object_interface.h
+59 -5
View File
@@ -15,12 +15,24 @@ class CoreWorker;
class CoreWorkerStoreProvider;
class CoreWorkerMemoryStore;
/// The interface that contains all `CoreWorker` methods that are related to object store.
/// The interface that contains all `CoreWorker` methods related to the object store.
class CoreWorkerObjectInterface {
public:
/// \param[in] worker_context WorkerContext of the parent CoreWorker.
/// \param[in] store_socket Path to the plasma store socket.
/// \param[in] use_memory_store Whether or not to use the in-memory object store
/// in addition to the plasma store.
CoreWorkerObjectInterface(WorkerContext &worker_context,
std::unique_ptr<RayletClient> &raylet_client,
const std::string &store_socket);
const std::string &store_socket,
bool use_memory_store = true);
/// Set options for this client's interactions with the object store.
///
/// \param[in] name Unique name for this object store client.
/// \param[in] limit The maximum amount of memory in bytes that this client
/// can use in the object store.
Status SetClientOptions(std::string name, int64_t limit_bytes);
/// Put an object into object store.
///
@@ -32,11 +44,32 @@ class CoreWorkerObjectInterface {
/// Put an object with specified ID into object store.
///
/// \param[in] object The ray object.
/// \param[in] object_id Object ID specified by user.
/// \param[in] object_id Object ID specified by the user.
/// \return Status.
Status Put(const RayObject &object, const ObjectID &object_id);
/// Get a list of objects from the object store. Duplicate object ids are supported.
/// Create and return a buffer in the object store that can be directly written
/// into. After writing to the buffer, the caller must call `Seal()` to finalize
/// the object. The `Create()` and `Seal()` combination is an alternative interface
/// to `Put()` that allows frontends to avoid an extra copy when possible.
///
/// \param[in] metadata Metadata of the object to be written.
/// \param[in] data_size Size of the object to be written.
/// \param[in] object_id Object ID specified by the user.
/// \param[out] data Buffer for the user to write the object into.
/// \return Status.
Status Create(const std::shared_ptr<Buffer> &metadata, const size_t data_size,
const ObjectID &object_id, std::shared_ptr<Buffer> *data);
/// Finalize placing an object into the object store. This should be called after
/// a corresponding `Create()` call and then writing into the returned buffer.
///
/// \param[in] object_id Object ID corresponding to the object.
/// \return Status.
Status Seal(const ObjectID &object_id);
/// Get a list of objects from the object store. Objects that failed to be retrieved
/// will be returned as nullptrs.
///
/// \param[in] ids IDs of the objects to get.
/// \param[in] timeout_ms Timeout in milliseconds, wait infinitely if it's negative.
@@ -45,6 +78,13 @@ class CoreWorkerObjectInterface {
Status Get(const std::vector<ObjectID> &ids, int64_t timeout_ms,
std::vector<std::shared_ptr<RayObject>> *results);
/// Return whether or not the object store contains the given object.
///
/// \param[in] object_id ID of the objects to check for.
/// \param[out] has_object Whether or not the object is present.
/// \return Status.
Status Contains(const ObjectID &object_id, bool *has_object);
/// Wait for a list of objects to appear in the object store.
/// Duplicate object ids are supported, and `num_objects` includes duplicate ids in this
/// case.
@@ -70,7 +110,21 @@ class CoreWorkerObjectInterface {
Status Delete(const std::vector<ObjectID> &object_ids, bool local_only,
bool delete_creating_tasks);
/// Get a string describing object store memory usage for debugging purposes.
///
/// \return std::string The string describing memory usage.
std::string MemoryUsageString();
private:
/// Helper function to group object IDs by the store provider that should be used
/// for them.
///
/// \param[in] object_ids Object IDs to group.
/// \param[out] results Map of provider type to object IDs.
void GroupObjectIdsByStoreProvider(
const std::vector<ObjectID> &object_ids,
EnumUnorderedMap<StoreProviderType, std::unordered_set<ObjectID>> *results);
/// Helper function to get a set of objects from different store providers.
///
/// \param[in] ids_per_provider A map from store provider type to the set of
@@ -96,8 +150,8 @@ class CoreWorkerObjectInterface {
/// Reference to the parent CoreWorker's raylet client.
std::unique_ptr<RayletClient> &raylet_client_;
/// Store socket name.
std::string store_socket_;
bool use_memory_store_;
/// In-memory store for return objects. This is used for `MEMORY` store provider.
std::shared_ptr<CoreWorkerMemoryStore> memory_store_;
src/ray/core_worker/store_provider/memory_store/memory_store.h
+20
View File
@@ -26,6 +26,26 @@ class CoreWorkerMemoryStore {
/// \return Status.
Status Put(const ObjectID &object_id, const RayObject &object);
/// Create and return a buffer in the object store that can be directly written
/// into. After writing to the buffer, the caller must call `Seal()` to finalize
/// the object. The `Create()` and `Seal()` combination is an alternative interface
/// to `Put()` that allows frontends to avoid an extra copy when possible.
///
/// \param[in] metadata Metadata of the object to be written.
/// \param[in] data_size Size of the object to be written.
/// \param[in] object_id Object ID specified by the user.
/// \param[out] data Buffer for the user to write the object into.
/// \return Status.
Status Create(const std::shared_ptr<Buffer> &metadata, const size_t data_size,
const ObjectID &object_id, std::shared_ptr<Buffer> *data);
/// Finalize placing an object into the object store. This should be called after
/// a corresponding `Create()` call and then writing into the returned buffer.
///
/// \param[in] object_id Object ID corresponding to the object.
/// \return Status.
Status Seal(const ObjectID &object_id);
/// Get a list of objects from the object store.
///
/// \param[in] object_ids IDs of the objects to get. Duplicates are not allowed.
src/ray/core_worker/store_provider/memory_store_provider.cc
+28 -6
View File
@@ -7,24 +7,39 @@
namespace ray {
//
// CoreWorkerMemoryStoreProvider functions
//
CoreWorkerMemoryStoreProvider::CoreWorkerMemoryStoreProvider(
std::shared_ptr<CoreWorkerMemoryStore> store)
: store_(store) {
RAY_CHECK(store != nullptr);
}
Status CoreWorkerMemoryStoreProvider::SetClientOptions(std::string name,
int64_t limit_bytes) {
return Status::NotImplemented(
"SetClientOptions() not implemented for in-memory store.");
}
Status CoreWorkerMemoryStoreProvider::Put(const RayObject &object,
const ObjectID &object_id) {
auto status = store_->Put(object_id, object);
Status status = store_->Put(object_id, object);
if (status.IsObjectExists()) {
// Object already exists in store, treat it as ok.
return Status::OK();
} else {
return status;
}
return status;
}
Status CoreWorkerMemoryStoreProvider::Create(const std::shared_ptr<Buffer> &metadata,
const size_t data_size,
const ObjectID &object_id,
std::shared_ptr<Buffer> *data) {
return Status::NotImplemented(
"Create/Seal interface not implemented for in-memory store.");
}
Status CoreWorkerMemoryStoreProvider::Seal(const ObjectID &object_id) {
return Status::NotImplemented(
"Create/Seal interface not implemented for in-memory store.");
}
Status CoreWorkerMemoryStoreProvider::Get(
@@ -48,6 +63,11 @@ Status CoreWorkerMemoryStoreProvider::Get(
return Status::OK();
}
Status CoreWorkerMemoryStoreProvider::Contains(const ObjectID &object_id,
bool *has_object) {
return Status::NotImplemented("Contains() not implemented for in-memory store.");
}
Status CoreWorkerMemoryStoreProvider::Wait(const std::unordered_set<ObjectID> &object_ids,
int num_objects, int64_t timeout_ms,
const TaskID &task_id,
@@ -74,4 +94,6 @@ Status CoreWorkerMemoryStoreProvider::Delete(const std::vector<ObjectID> &object
return Status::OK();
}
std::string CoreWorkerMemoryStoreProvider::MemoryUsageString() { return ""; }
} // namespace ray
src/ray/core_worker/store_provider/memory_store_provider.h
+12 -4
View File
@@ -15,30 +15,38 @@ class CoreWorker;
/// The class provides implementations for accessing local process memory store.
/// An example usage for this is to retrieve the returned objects from direct
/// actor call (see direct_actor_transport.cc).
/// See `CoreWorkerStoreProvider` for the semantics of public methods.
class CoreWorkerMemoryStoreProvider : public CoreWorkerStoreProvider {
public:
CoreWorkerMemoryStoreProvider(std::shared_ptr<CoreWorkerMemoryStore> store);
/// See `CoreWorkerStoreProvider::Put` for semantics.
Status SetClientOptions(std::string name, int64_t limit_bytes) override;
Status Put(const RayObject &object, const ObjectID &object_id) override;
/// See `CoreWorkerStoreProvider::Get` for semantics.
Status Create(const std::shared_ptr<Buffer> &metadata, const size_t data_size,
const ObjectID &object_id, std::shared_ptr<Buffer> *data) override;
Status Seal(const ObjectID &object_id) override;
Status Get(const std::unordered_set<ObjectID> &object_ids, int64_t timeout_ms,
const TaskID &task_id,
std::unordered_map<ObjectID, std::shared_ptr<RayObject>> *results,
bool *got_exception) override;
/// See `CoreWorkerStoreProvider::Wait` for semantics.
Status Contains(const ObjectID &object_id, bool *has_object) override;
/// Note that `num_objects` must equal to number of items in `object_ids`.
Status Wait(const std::unordered_set<ObjectID> &object_ids, int num_objects,
int64_t timeout_ms, const TaskID &task_id,
std::unordered_set<ObjectID> *ready) override;
/// See `CoreWorkerStoreProvider::Delete` for semantics.
/// Note that `local_only` must be true, and `delete_creating_tasks` must be false here.
Status Delete(const std::vector<ObjectID> &object_ids, bool local_only = true,
bool delete_creating_tasks = false) override;
std::string MemoryUsageString() override;
private:
/// Implementation.
std::shared_ptr<CoreWorkerMemoryStore> store_;
src/ray/core_worker/store_provider/plasma_store_provider.cc
+61 -15
View File
@@ -13,32 +13,66 @@ CoreWorkerPlasmaStoreProvider::CoreWorkerPlasmaStoreProvider(
RAY_ARROW_CHECK_OK(store_client_.Connect(store_socket));
}
CoreWorkerPlasmaStoreProvider::~CoreWorkerPlasmaStoreProvider() {
RAY_IGNORE_EXPR(store_client_.Disconnect());
}
Status CoreWorkerPlasmaStoreProvider::SetClientOptions(std::string name,
int64_t limit_bytes) {
std::lock_guard<std::mutex> guard(store_client_mutex_);
RAY_ARROW_RETURN_NOT_OK(store_client_.SetClientOptions(name, limit_bytes));
return Status::OK();
}
Status CoreWorkerPlasmaStoreProvider::Put(const RayObject &object,
const ObjectID &object_id) {
std::shared_ptr<Buffer> data;
RAY_RETURN_NOT_OK(Create(object.GetMetadata(),
object.HasData() ? object.GetData()->Size() : 0, object_id,
&data));
// data could be a nullptr if the ObjectID already existed, but this does
// not throw an error.
if (data != nullptr) {
if (object.HasData()) {
memcpy(data->Data(), object.GetData()->Data(), object.GetData()->Size());
}
RAY_RETURN_NOT_OK(Seal(object_id));
}
return Status::OK();
}
Status CoreWorkerPlasmaStoreProvider::Create(const std::shared_ptr<Buffer> &metadata,
const size_t data_size,
const ObjectID &object_id,
std::shared_ptr<Buffer> *data) {
auto plasma_id = object_id.ToPlasmaId();
auto data = object.GetData();
auto metadata = object.GetMetadata();
std::shared_ptr<arrow::Buffer> out_buffer;
std::shared_ptr<arrow::Buffer> arrow_buffer;
{
std::unique_lock<std::mutex> guard(store_client_mutex_);
arrow::Status status = store_client_.Create(
plasma_id, data ? data->Size() : 0, metadata ? metadata->Data() : nullptr,
metadata ? metadata->Size() : 0, &out_buffer);
std::lock_guard<std::mutex> guard(store_client_mutex_);
arrow::Status status =
store_client_.Create(plasma_id, data_size, metadata ? metadata->Data() : nullptr,
metadata ? metadata->Size() : 0, &arrow_buffer);
if (plasma::IsPlasmaObjectExists(status)) {
// TODO(hchen): Should we propagate this error out of `ObjectInterface::put`?
RAY_LOG(WARNING) << "Trying to put an object that already existed in plasma: "
<< object_id << ".";
return Status::OK();
}
if (plasma::IsPlasmaStoreFull(status)) {
std::ostringstream message;
message << "Failed to put object " << object_id
<< " in object store because it is full: " << status.message();
return Status::ObjectStoreFull(message.str());
}
RAY_ARROW_RETURN_NOT_OK(status);
}
*data = std::make_shared<PlasmaBuffer>(PlasmaBuffer(arrow_buffer));
return Status::OK();
}
if (data != nullptr) {
memcpy(out_buffer->mutable_data(), data->Data(), data->Size());
}
Status CoreWorkerPlasmaStoreProvider::Seal(const ObjectID &object_id) {
auto plasma_id = object_id.ToPlasmaId();
{
std::unique_lock<std::mutex> guard(store_client_mutex_);
std::lock_guard<std::mutex> guard(store_client_mutex_);
RAY_ARROW_RETURN_NOT_OK(store_client_.Seal(plasma_id));
RAY_ARROW_RETURN_NOT_OK(store_client_.Release(plasma_id));
}
@@ -50,7 +84,7 @@ Status CoreWorkerPlasmaStoreProvider::FetchAndGetFromPlasmaStore(
int64_t timeout_ms, bool fetch_only, const TaskID &task_id,
std::unordered_map<ObjectID, std::shared_ptr<RayObject>> *results,
bool *got_exception) {
RAY_CHECK_OK(raylet_client_->FetchOrReconstruct(batch_ids, fetch_only, task_id));
RAY_RETURN_NOT_OK(raylet_client_->FetchOrReconstruct(batch_ids, fetch_only, task_id));
std::vector<plasma::ObjectID> plasma_batch_ids;
plasma_batch_ids.reserve(batch_ids.size());
@@ -59,7 +93,7 @@ Status CoreWorkerPlasmaStoreProvider::FetchAndGetFromPlasmaStore(
}
std::vector<plasma::ObjectBuffer> plasma_results;
{
std::unique_lock<std::mutex> guard(store_client_mutex_);
std::lock_guard<std::mutex> guard(store_client_mutex_);
RAY_ARROW_RETURN_NOT_OK(
store_client_.Get(plasma_batch_ids, timeout_ms, &plasma_results));
}
@@ -156,6 +190,13 @@ Status CoreWorkerPlasmaStoreProvider::Get(
return raylet_client_->NotifyUnblocked(task_id);
}
Status CoreWorkerPlasmaStoreProvider::Contains(const ObjectID &object_id,
bool *has_object) {
std::lock_guard<std::mutex> guard(store_client_mutex_);
RAY_ARROW_RETURN_NOT_OK(store_client_.Contains(object_id.ToPlasmaId(), has_object));
return Status::OK();
}
Status CoreWorkerPlasmaStoreProvider::Wait(const std::unordered_set<ObjectID> &object_ids,
int num_objects, int64_t timeout_ms,
const TaskID &task_id,
@@ -178,6 +219,11 @@ Status CoreWorkerPlasmaStoreProvider::Delete(const std::vector<ObjectID> &object
return raylet_client_->FreeObjects(object_ids, local_only, delete_creating_tasks);
}
std::string CoreWorkerPlasmaStoreProvider::MemoryUsageString() {
std::lock_guard<std::mutex> guard(store_client_mutex_);
return store_client_.DebugString();
}
void CoreWorkerPlasmaStoreProvider::WarnIfAttemptedTooManyTimes(
int num_attempts, const std::unordered_set<ObjectID> &remaining) {
if (num_attempts % RayConfig::instance().object_store_get_warn_per_num_attempts() ==
src/ray/core_worker/store_provider/plasma_store_provider.h
+17 -6
View File
@@ -14,30 +14,41 @@ namespace ray {
class CoreWorker;
/// The class provides implementations for accessing plasma store, which includes both
/// local and remote store, remote access is done via raylet.
/// local and remote stores. Local access goes is done via a
/// CoreWorkerLocalPlasmaStoreProvider and remote access goes through the raylet.
/// See `CoreWorkerStoreProvider` for the semantics of public methods.
class CoreWorkerPlasmaStoreProvider : public CoreWorkerStoreProvider {
public:
CoreWorkerPlasmaStoreProvider(const std::string &store_socket,
std::unique_ptr<RayletClient> &raylet_client);
/// See `CoreWorkerStoreProvider::Put` for semantics.
~CoreWorkerPlasmaStoreProvider();
Status SetClientOptions(std::string name, int64_t limit_bytes);
Status Put(const RayObject &object, const ObjectID &object_id) override;
/// See `CoreWorkerStoreProvider::Get` for semantics.
Status Create(const std::shared_ptr<Buffer> &metadata, const size_t data_size,
const ObjectID &object_id, std::shared_ptr<Buffer> *data) override;
Status Seal(const ObjectID &object_id) override;
Status Get(const std::unordered_set<ObjectID> &object_ids, int64_t timeout_ms,
const TaskID &task_id,
std::unordered_map<ObjectID, std::shared_ptr<RayObject>> *results,
bool *got_exception) override;
/// See `CoreWorkerStoreProvider::Wait` for semantics.
Status Contains(const ObjectID &object_id, bool *has_object) override;
Status Wait(const std::unordered_set<ObjectID> &object_ids, int num_objects,
int64_t timeout_ms, const TaskID &task_id,
std::unordered_set<ObjectID> *ready) override;
/// See `CoreWorkerStoreProvider::Delete` for semantics.
Status Delete(const std::vector<ObjectID> &object_ids, bool local_only = true,
bool delete_creating_tasks = false) override;
std::string MemoryUsageString() override;
private:
/// Ask the raylet to fetch a set of objects and then attempt to get them
/// from the local plasma store. Successfully fetched objects will be removed
@@ -62,7 +73,7 @@ class CoreWorkerPlasmaStoreProvider : public CoreWorkerStoreProvider {
bool *got_exception);
/// Print a warning if we've attempted too many times, but some objects are still
/// unavailable.
/// unavailable. Only the keys in the 'remaining' map are used.
///
/// \param[in] num_attemps The number of attempted times.
/// \param[in] remaining The remaining objects.
src/ray/core_worker/store_provider/store_provider.h
+41 -1
View File
@@ -18,6 +18,13 @@ class CoreWorkerStoreProvider {
virtual ~CoreWorkerStoreProvider() {}
/// Set options for this client's interactions with the object store.
///
/// \param[in] name Unique name for this object store client.
/// \param[in] limit The maximum amount of memory in bytes that this client
/// can use in the object store.
virtual Status SetClientOptions(std::string name, int64_t limit_bytes) = 0;
/// Put an object with specified ID into object store.
///
/// \param[in] object The ray object.
@@ -25,6 +32,26 @@ class CoreWorkerStoreProvider {
/// \return Status.
virtual Status Put(const RayObject &object, const ObjectID &object_id) = 0;
/// Create and return a buffer in the object store that can be directly written
/// into. After writing to the buffer, the caller must call `Seal()` to finalize
/// the object. The `Create()` and `Seal()` combination is an alternative interface
/// to `Put()` that allows frontends to avoid an extra copy when possible.
///
/// \param[in] metadata Metadata of the object to be written.
/// \param[in] data_size Size of the object to be written.
/// \param[in] object_id Object ID specified by the user.
/// \param[out] data Buffer for the user to write the object into.
/// \return Status.
virtual Status Create(const std::shared_ptr<Buffer> &metadata, const size_t data_size,
const ObjectID &object_id, std::shared_ptr<Buffer> *data) = 0;
/// Finalize placing an object into the object store. This should be called after
/// a corresponding `Create()` call and then writing into the returned buffer.
///
/// \param[in] object_id Object ID corresponding to the object.
/// \return Status.
virtual Status Seal(const ObjectID &object_id) = 0;
/// Get a set of objects from the object store.
///
/// \param[in] object_ids IDs of the objects to get.
@@ -33,12 +60,20 @@ class CoreWorkerStoreProvider {
/// \param[out] results Map of objects to write results into. Get will only add to this
/// map, not clear or remove from it, so the caller can pass in a non-empty map.
/// \param[out] got_exception Set to true if any of the fetched results were an
/// exception. \return Status.
/// exception.
/// \return Status.
virtual Status Get(const std::unordered_set<ObjectID> &object_ids, int64_t timeout_ms,
const TaskID &task_id,
std::unordered_map<ObjectID, std::shared_ptr<RayObject>> *results,
bool *got_exception) = 0;
/// Return whether or not the object store contains the given object.
///
/// \param[in] object_id ID of the objects to check for.
/// \param[out] has_object Whether or not the object is present.
/// \return Status.
virtual Status Contains(const ObjectID &object_id, bool *has_object) = 0;
/// Wait for a list of objects to appear in the object store. Objects that appear will
/// be added to the ready set.
///
@@ -63,6 +98,11 @@ class CoreWorkerStoreProvider {
/// \return Status.
virtual Status Delete(const std::vector<ObjectID> &object_ids, bool local_only = true,
bool delete_creating_tasks = false) = 0;
/// Get a string describing object store memory usage for debugging purposes.
///
/// \return std::string The string describing memory usage.
virtual std::string MemoryUsageString() = 0;
};
} // namespace ray
src/ray/core_worker/test/core_worker_test.cc
+11 -12
View File
@@ -30,8 +30,6 @@ std::string store_executable;
std::string raylet_executable;
std::string mock_worker_executable;
ray::ObjectID RandomObjectID() { return ObjectID::FromRandom(); }
static void flushall_redis(void) {
redisContext *context = redisConnect("127.0.0.1", 6379);
freeReplyObject(redisCommand(context, "FLUSHALL"));
@@ -112,7 +110,7 @@ class CoreWorkerTest : public ::testing::Test {
}
std::string StartStore() {
std::string store_socket_name = "/tmp/store" + RandomObjectID().Hex();
std::string store_socket_name = "/tmp/store" + ObjectID::FromRandom().Hex();
std::string store_pid = store_socket_name + ".pid";
std::string plasma_command = store_executable + " -m 10000000 -s " +
store_socket_name +
@@ -134,7 +132,7 @@ class CoreWorkerTest : public ::testing::Test {
std::string StartRaylet(std::string store_socket_name, std::string node_ip_address,
std::string redis_address, std::string resource) {
std::string raylet_socket_name = "/tmp/raylet" + RandomObjectID().Hex();
std::string raylet_socket_name = "/tmp/raylet" + ObjectID::FromRandom().Hex();
std::string ray_start_cmd = raylet_executable;
ray_start_cmd.append(" --raylet_socket_name=" + raylet_socket_name)
.append(" --store_socket_name=" + store_socket_name)
@@ -221,7 +219,7 @@ bool CoreWorkerTest::WaitForDirectCallActorState(CoreWorker &worker,
void CoreWorkerTest::TestNormalTask(
const std::unordered_map<std::string, double> &resources) {
CoreWorker driver(WorkerType::DRIVER, Language::PYTHON, raylet_store_socket_names_[0],
raylet_socket_names_[0], NextJobId(), gcs_options_, nullptr);
raylet_socket_names_[0], NextJobId(), gcs_options_, "", nullptr);
// Test for tasks with by-value and by-ref args.
{
@@ -263,7 +261,7 @@ void CoreWorkerTest::TestNormalTask(
void CoreWorkerTest::TestActorTask(
const std::unordered_map<std::string, double> &resources, bool is_direct_call) {
CoreWorker driver(WorkerType::DRIVER, Language::PYTHON, raylet_store_socket_names_[0],
raylet_socket_names_[0], NextJobId(), gcs_options_, nullptr);
raylet_socket_names_[0], NextJobId(), gcs_options_, "", nullptr);
auto actor_handle = CreateActorHelper(driver, resources, is_direct_call, 1000);
@@ -352,7 +350,7 @@ void CoreWorkerTest::TestActorTask(
void CoreWorkerTest::TestActorReconstruction(
const std::unordered_map<std::string, double> &resources, bool is_direct_call) {
CoreWorker driver(WorkerType::DRIVER, Language::PYTHON, raylet_store_socket_names_[0],
raylet_socket_names_[0], NextJobId(), gcs_options_, nullptr);
raylet_socket_names_[0], NextJobId(), gcs_options_, "", nullptr);
// creating actor.
auto actor_handle = CreateActorHelper(driver, resources, is_direct_call, 1000);
@@ -410,7 +408,7 @@ void CoreWorkerTest::TestActorReconstruction(
void CoreWorkerTest::TestActorFailure(
const std::unordered_map<std::string, double> &resources, bool is_direct_call) {
CoreWorker driver(WorkerType::DRIVER, Language::PYTHON, raylet_store_socket_names_[0],
raylet_socket_names_[0], NextJobId(), gcs_options_, nullptr);
raylet_socket_names_[0], NextJobId(), gcs_options_, "", nullptr);
// creating actor.
auto actor_handle =
@@ -698,7 +696,8 @@ TEST_F(ZeroNodeTest, TestTaskSpecPerf) {
TEST_F(SingleNodeTest, TestDirectActorTaskSubmissionPerf) {
CoreWorker driver(WorkerType::DRIVER, Language::PYTHON, raylet_store_socket_names_[0],
raylet_socket_names_[0], JobID::FromInt(1), gcs_options_, nullptr);
raylet_socket_names_[0], JobID::FromInt(1), gcs_options_, "",
nullptr);
std::unique_ptr<ActorHandle> actor_handle;
// Test creating actor.
@@ -802,7 +801,7 @@ TEST_F(SingleNodeTest, TestMemoryStoreProvider) {
TEST_F(SingleNodeTest, TestObjectInterface) {
CoreWorker core_worker(WorkerType::DRIVER, Language::PYTHON,
raylet_store_socket_names_[0], raylet_socket_names_[0],
JobID::FromInt(1), gcs_options_, nullptr);
JobID::FromInt(1), gcs_options_, "", nullptr);
uint8_t array1[] = {1, 2, 3, 4, 5, 6, 7, 8};
uint8_t array2[] = {10, 11, 12, 13, 14, 15};
@@ -873,10 +872,10 @@ TEST_F(SingleNodeTest, TestObjectInterface) {
TEST_F(TwoNodeTest, TestObjectInterfaceCrossNodes) {
CoreWorker worker1(WorkerType::DRIVER, Language::PYTHON, raylet_store_socket_names_[0],
raylet_socket_names_[0], NextJobId(), gcs_options_, nullptr);
raylet_socket_names_[0], NextJobId(), gcs_options_, "", nullptr);
CoreWorker worker2(WorkerType::DRIVER, Language::PYTHON, raylet_store_socket_names_[1],
raylet_socket_names_[1], NextJobId(), gcs_options_, nullptr);
raylet_socket_names_[1], NextJobId(), gcs_options_, "", nullptr);
uint8_t array1[] = {1, 2, 3, 4, 5, 6, 7, 8};
uint8_t array2[] = {10, 11, 12, 13, 14, 15};
src/ray/core_worker/test/mock_worker.cc
+1 -1
View File
@@ -23,7 +23,7 @@ class MockWorker {
MockWorker(const std::string &store_socket, const std::string &raylet_socket,
const gcs::GcsClientOptions &gcs_options)
: worker_(WorkerType::WORKER, Language::PYTHON, store_socket, raylet_socket,
JobID::FromInt(1), gcs_options,
JobID::FromInt(1), gcs_options, /*log_dir=*/"",
std::bind(&MockWorker::ExecuteTask, this, _1, _2, _3, _4)) {}
void Run() {
src/ray/core_worker/transport/direct_actor_transport.cc
+2 -2
View File
@@ -27,7 +27,7 @@ Status CoreWorkerDirectActorTaskSubmitter::SubmitTask(
const TaskSpecification &task_spec) {
RAY_LOG(DEBUG) << "Submitting task " << task_spec.TaskId();
if (HasByReferenceArgs(task_spec)) {
return Status::Invalid("direct actor call only supports by-value arguments");
return Status::Invalid("Direct actor call only supports by-value arguments");
}
RAY_CHECK(task_spec.IsActorTask());
@@ -243,7 +243,7 @@ void CoreWorkerDirectActorTaskReceiver::HandlePushTask(
RAY_LOG(DEBUG) << "Received task " << task_spec.TaskId();
if (HasByReferenceArgs(task_spec)) {
send_reply_callback(
Status::Invalid("direct actor call only supports by value arguments"), nullptr,
Status::Invalid("Direct actor call only supports by value arguments"), nullptr,
nullptr);
return;
}
src/ray/gcs/redis_context.cc
+1 -1
View File
@@ -187,7 +187,7 @@ Status AuthenticateRedis(redisAsyncContext *context, const std::string &password
}
void RedisAsyncContextDisconnectCallback(const redisAsyncContext *context, int status) {
RAY_LOG(WARNING) << "Redis async context disconnected. Status: " << status;
RAY_LOG(INFO) << "Redis async context disconnected. Status: " << status;
// Reset raw 'redisAsyncContext' to nullptr because hiredis will release this context.
reinterpret_cast<RedisAsyncContext *>(context->data)->ResetRawRedisAsyncContext();
}