wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-23 13:10:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

[core] Option to fallback to LRU on OutOfMemory (#7410)

Browse Source 
* Add a test for LRU fallback

* Update error message

* Upgrade arrow to master

* Integrate with arrow

* Revert "Bazel mirrors (#7385)"

This reverts commit 44aded5272.

* Don't LRU evict

* Revert "Revert "Bazel mirrors (#7385)""

This reverts commit b6359fea78d1bd3925452ca88ac71e0c9e5c7dd3.

* Add lru_evict flag

* fix internal config

* Fix

* upgrade arrow

* debug

* Set free period in config for lru_evict, override max retries to fix
test

* Fix test?

* fix test

* Revert "debug"

This reverts commit 98f01c63a267f38218f5047b1866e4c1c8280017.

* fix exception str

* Fix ref count test

* Shorten travis test?
This commit is contained in:
Stephanie Wang
2020-03-14 11:28:43 -07:00
committed by GitHub
parent 52cf77f5a9
commit 53549314c5
10 changed files with 123 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/core_worker/core_worker.cc
+1
View File
@@ -187,6 +187,7 @@ CoreWorker::CoreWorker(const WorkerType worker_type, const Language language,
plasma_store_provider_.reset(new CoreWorkerPlasmaStoreProvider(
store_socket, local_raylet_client_, check_signals_,
/*evict_if_full=*/RayConfig::instance().object_pinning_enabled(),
boost::bind(&CoreWorker::TriggerGlobalGC, this)));
memory_store_.reset(new CoreWorkerMemoryStore(
[this](const RayObject &obj, const ObjectID &obj_id) {
src/ray/core_worker/store_provider/plasma_store_provider.cc
+16 -9
View File
@@ -24,10 +24,12 @@ namespace ray {
CoreWorkerPlasmaStoreProvider::CoreWorkerPlasmaStoreProvider(
const std::string &store_socket,
const std::shared_ptr<raylet::RayletClient> raylet_client,
std::function<Status()> check_signals, std::function<void()> on_store_full)
: raylet_client_(raylet_client) {
check_signals_ = check_signals;
on_store_full_ = on_store_full;
std::function<Status()> check_signals, bool evict_if_full,
std::function<void()> on_store_full)
: raylet_client_(raylet_client),
check_signals_(check_signals),
evict_if_full_(evict_if_full),
on_store_full_(on_store_full) {
RAY_ARROW_CHECK_OK(store_client_.Connect(store_socket));
}
@@ -75,15 +77,20 @@ Status CoreWorkerPlasmaStoreProvider::Create(const std::shared_ptr<Buffer> &meta
uint32_t delay = RayConfig::instance().object_store_full_initial_delay_ms();
Status status;
bool should_retry = true;
// If we cannot retry, then always evict on the first attempt.
bool evict_if_full = max_retries == 0 ? true : evict_if_full_;
while (should_retry) {
should_retry = false;
arrow::Status plasma_status;
std::shared_ptr<arrow::Buffer> arrow_buffer;
{
std::lock_guard<std::mutex> guard(store_client_mutex_);
plasma_status = store_client_.Create(
object_id.ToPlasmaId(), data_size, metadata ? metadata->Data() : nullptr,
metadata ? metadata->Size() : 0, &arrow_buffer);
plasma_status = store_client_.Create(object_id.ToPlasmaId(), data_size,
metadata ? metadata->Data() : nullptr,
metadata ? metadata->Size() : 0, &arrow_buffer,
/*device_num=*/0, evict_if_full);
// Always try to evict after the first attempt.
evict_if_full = true;
}
if (plasma::IsPlasmaStoreFull(plasma_status)) {
std::ostringstream message;
@@ -101,8 +108,8 @@ Status CoreWorkerPlasmaStoreProvider::Create(const std::shared_ptr<Buffer> &meta
retries += 1;
should_retry = true;
} else {
RAY_LOG(ERROR) << "Failed to put object " << object_id << " after " << max_retries
<< " attempts. Plasma store status:\n"
RAY_LOG(ERROR) << "Failed to put object " << object_id << " after "
<< (max_retries + 1) << " attempts. Plasma store status:\n"
<< MemoryUsageString();
}
} else if (plasma::IsPlasmaObjectExists(plasma_status)) {
src/ray/core_worker/store_provider/plasma_store_provider.h
+2 -1
View File
@@ -35,7 +35,7 @@ class CoreWorkerPlasmaStoreProvider {
public:
CoreWorkerPlasmaStoreProvider(const std::string &store_socket,
const std::shared_ptr<raylet::RayletClient> raylet_client,
std::function<Status()> check_signals,
std::function<Status()> check_signals, bool evict_if_full,
std::function<void()> on_store_full = nullptr);
~CoreWorkerPlasmaStoreProvider();
@@ -134,6 +134,7 @@ class CoreWorkerPlasmaStoreProvider {
plasma::PlasmaClient store_client_;
std::mutex store_client_mutex_;
std::function<Status()> check_signals_;
const bool evict_if_full_;
std::function<void()> on_store_full_;
};