From b94b4a35e04d8d2c0af4420518a4e9a94c1c9b9f Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Wed, 31 May 2017 23:24:23 +0000 Subject: [PATCH] Make the Plasma store ready for Arrow integration (#579) * port plasma to arrow * fixes * refactor plasma client * more modernization * fix plasma manager tests * everything compiles * fix plasma client tests * update plasma serialization tests * fix plasma manager tests * fix bug * updates * fix bug * fix tests * fix rebase * address comments * fix travis valgrind build * fix linting * fix include order again * fix linting * address comments --- src/local_scheduler/local_scheduler.cc | 18 +- .../local_scheduler_algorithm.cc | 4 +- src/local_scheduler/local_scheduler_shared.h | 2 +- src/numbuf/CMakeLists.txt | 4 +- src/numbuf/python/src/pynumbuf/numbuf.cc | 61 +- src/plasma/CMakeLists.txt | 34 +- src/plasma/eviction_policy.cc | 17 +- src/plasma/eviction_policy.h | 1 + src/plasma/logging.h | 147 ++++ src/plasma/plasma.cc | 14 +- src/plasma/plasma.h | 28 +- src/plasma/plasma_client.cc | 749 ++++++++---------- src/plasma/plasma_client.h | 586 +++++++------- src/plasma/plasma_common.cc | 67 ++ src/plasma/plasma_common.h | 41 + src/plasma/plasma_events.cc | 6 +- src/plasma/plasma_events.h | 81 +- src/plasma/plasma_extension.cc | 141 ++-- src/plasma/plasma_extension.h | 15 +- src/plasma/plasma_io.cc | 220 +++++ src/plasma/plasma_io.h | 38 + src/plasma/plasma_manager.cc | 117 +-- src/plasma/plasma_protocol.cc | 634 +++++++-------- src/plasma/plasma_protocol.h | 273 +++---- src/plasma/plasma_store.cc | 475 ++++++----- src/plasma/plasma_store.h | 178 ++--- src/plasma/status.cc | 90 +++ src/plasma/status.h | 226 ++++++ src/plasma/test/client_tests.cc | 202 ++--- src/plasma/test/manager_tests.cc | 28 +- src/plasma/test/serialization_tests.cc | 311 ++++---- src/plasma/thirdparty/ae/ae.c | 465 +++++++++++ src/plasma/thirdparty/ae/ae.h | 123 +++ src/plasma/thirdparty/ae/ae_epoll.c | 135 ++++ src/plasma/thirdparty/ae/ae_evport.c | 320 ++++++++ src/plasma/thirdparty/ae/ae_kqueue.c | 138 ++++ src/plasma/thirdparty/ae/ae_select.c | 106 +++ src/plasma/thirdparty/ae/config.h | 54 ++ src/plasma/thirdparty/ae/zmalloc.h | 16 + test/component_failures_test.py | 2 +- 40 files changed, 4097 insertions(+), 2070 deletions(-) create mode 100644 src/plasma/logging.h create mode 100644 src/plasma/plasma_common.cc create mode 100644 src/plasma/plasma_common.h create mode 100644 src/plasma/plasma_io.cc create mode 100644 src/plasma/plasma_io.h create mode 100644 src/plasma/status.cc create mode 100644 src/plasma/status.h create mode 100644 src/plasma/thirdparty/ae/ae.c create mode 100644 src/plasma/thirdparty/ae/ae.h create mode 100644 src/plasma/thirdparty/ae/ae_epoll.c create mode 100644 src/plasma/thirdparty/ae/ae_evport.c create mode 100644 src/plasma/thirdparty/ae/ae_kqueue.c create mode 100644 src/plasma/thirdparty/ae/ae_select.c create mode 100644 src/plasma/thirdparty/ae/config.h create mode 100644 src/plasma/thirdparty/ae/zmalloc.h diff --git a/src/local_scheduler/local_scheduler.cc b/src/local_scheduler/local_scheduler.cc index 315a70280..891cff803 100644 --- a/src/local_scheduler/local_scheduler.cc +++ b/src/local_scheduler/local_scheduler.cc @@ -174,7 +174,8 @@ void LocalSchedulerState_free(LocalSchedulerState *state) { } /* Disconnect from plasma. */ - plasma_disconnect(state->plasma_conn); + ARROW_CHECK_OK(state->plasma_conn->Disconnect()); + delete state->plasma_conn; state->plasma_conn = NULL; /* Disconnect from the database. */ @@ -366,11 +367,18 @@ LocalSchedulerState *LocalSchedulerState_init( state->db = NULL; } /* Connect to Plasma. This method will retry if Plasma hasn't started yet. */ - state->plasma_conn = - plasma_connect(plasma_store_socket_name, plasma_manager_socket_name, - PLASMA_DEFAULT_RELEASE_DELAY); + state->plasma_conn = new PlasmaClient(); + if (plasma_manager_socket_name != NULL) { + ARROW_CHECK_OK(state->plasma_conn->Connect(plasma_store_socket_name, + plasma_manager_socket_name, + PLASMA_DEFAULT_RELEASE_DELAY)); + } else { + ARROW_CHECK_OK(state->plasma_conn->Connect(plasma_store_socket_name, "", + PLASMA_DEFAULT_RELEASE_DELAY)); + } /* Subscribe to notifications about sealed objects. */ - int plasma_fd = plasma_subscribe(state->plasma_conn); + int plasma_fd; + ARROW_CHECK_OK(state->plasma_conn->Subscribe(plasma_fd)); /* Add the callback that processes the notification to the event loop. */ event_loop_add_file(loop, plasma_fd, EVENT_LOOP_READ, process_plasma_notification, state); diff --git a/src/local_scheduler/local_scheduler_algorithm.cc b/src/local_scheduler/local_scheduler_algorithm.cc index d71fa65a7..7c4132ae5 100644 --- a/src/local_scheduler/local_scheduler_algorithm.cc +++ b/src/local_scheduler/local_scheduler_algorithm.cc @@ -457,7 +457,7 @@ void fetch_missing_dependency(LocalSchedulerState *state, /* We weren't actively fetching this object. Try the fetch once * immediately. */ if (plasma_manager_is_connected(state->plasma_conn)) { - plasma_fetch(state->plasma_conn, 1, &obj_id); + ARROW_CHECK_OK(state->plasma_conn->Fetch(1, &obj_id)); } /* Create an entry and add it to the list of active fetch requests to * ensure that the fetch actually happens. The entry will be moved to the @@ -546,7 +546,7 @@ int fetch_object_timeout_handler(event_loop *loop, timer_id id, void *context) { object_ids[i] = entry.second.object_id; i++; } - plasma_fetch(state->plasma_conn, num_object_ids, object_ids); + ARROW_CHECK_OK(state->plasma_conn->Fetch(num_object_ids, object_ids)); for (int i = 0; i < num_object_ids; ++i) { reconstruct_object(state, object_ids[i]); } diff --git a/src/local_scheduler/local_scheduler_shared.h b/src/local_scheduler/local_scheduler_shared.h index 4913af3fc..b10cff72c 100644 --- a/src/local_scheduler/local_scheduler_shared.h +++ b/src/local_scheduler/local_scheduler_shared.h @@ -62,7 +62,7 @@ struct LocalSchedulerState { /** The handle to the database. */ DBHandle *db; /** The Plasma client. */ - PlasmaConnection *plasma_conn; + PlasmaClient *plasma_conn; /** State for the scheduling algorithm. */ SchedulingAlgorithmState *algorithm_state; /** Input buffer, used for reading input in process_message to avoid diff --git a/src/numbuf/CMakeLists.txt b/src/numbuf/CMakeLists.txt index 0cbb4fc51..f281182aa 100644 --- a/src/numbuf/CMakeLists.txt +++ b/src/numbuf/CMakeLists.txt @@ -45,7 +45,6 @@ if(HAS_PLASMA) include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma") include_directories("${CMAKE_CURRENT_LIST_DIR}/../common") include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty") - set(COMMON_EXTENSION ../common/lib/python/common_extension.cc) endif() add_definitions(-fPIC) @@ -55,8 +54,7 @@ add_library(numbuf SHARED cpp/src/numbuf/sequence.cc python/src/pynumbuf/numbuf.cc python/src/pynumbuf/adapters/numpy.cc - python/src/pynumbuf/adapters/python.cc - ${COMMON_EXTENSION}) + python/src/pynumbuf/adapters/python.cc) if(APPLE) diff --git a/src/numbuf/python/src/pynumbuf/numbuf.cc b/src/numbuf/python/src/pynumbuf/numbuf.cc index d8a69b500..f3def5904 100644 --- a/src/numbuf/python/src/pynumbuf/numbuf.cc +++ b/src/numbuf/python/src/pynumbuf/numbuf.cc @@ -7,16 +7,13 @@ #include -#include -#include -#include - -#include - -#include "adapters/python.h" -#include "memory.h" - #ifdef HAS_PLASMA +// This needs to be included before plasma_protocol. We cannot include it in +// plasma_protocol, because that file is used both with the store and the +// manager, the store uses it the ObjectID from plasma_common.h and the +// manager uses it with the ObjectID from common.h. +#include "plasma_common.h" + #include "plasma_client.h" #include "plasma_protocol.h" @@ -25,11 +22,18 @@ PyObject* NumbufPlasmaOutOfMemoryError; PyObject* NumbufPlasmaObjectExistsError; } -#include "common_extension.h" #include "plasma_extension.h" #endif +#include +#include +#include +#include + +#include "adapters/python.h" +#include "memory.h" + using namespace arrow; using namespace numbuf; @@ -253,9 +257,9 @@ static void BufferCapsule_Destructor(PyObject* capsule) { * is (void*) 0x1). This is neccessary because the primary pointer of the * capsule cannot be NULL. */ if (PyCapsule_GetContext(context) == NULL) { - PlasmaConnection* conn; - CHECK(PyObjectToPlasmaConnection(context, &conn)); - plasma_release(conn, *id); + PlasmaClient* client; + ARROW_CHECK(PyObjectToPlasmaClient(context, &client)); + ARROW_CHECK_OK(client->Release(*id)); } Py_XDECREF(context); delete id; @@ -275,10 +279,10 @@ static void BufferCapsule_Destructor(PyObject* capsule) { */ static PyObject* store_list(PyObject* self, PyObject* args) { ObjectID obj_id; - PlasmaConnection* conn; + PlasmaClient* client; PyObject* value; if (!PyArg_ParseTuple(args, "O&O&O", PyStringToUniqueID, &obj_id, - PyObjectToPlasmaConnection, &conn, &value)) { + PyObjectToPlasmaClient, &client, &value)) { return NULL; } if (!PyList_Check(value)) { return NULL; } @@ -302,21 +306,20 @@ static PyObject* store_list(PyObject* self, PyObject* args) { * stored in the plasma data buffer. The header end offset is stored in * the first LENGTH_PREFIX_SIZE bytes of the data buffer. The RecordBatch * data is stored after that. */ - int error_code = - plasma_create(conn, obj_id, LENGTH_PREFIX_SIZE + total_size, NULL, 0, &data); - if (error_code == PlasmaError_ObjectExists) { + s = client->Create(obj_id, LENGTH_PREFIX_SIZE + total_size, NULL, 0, &data); + if (s.IsPlasmaObjectExists()) { PyErr_SetString(NumbufPlasmaObjectExistsError, "An object with this ID already exists in the plasma " "store."); return NULL; } - if (error_code == PlasmaError_OutOfMemory) { + if (s.IsPlasmaStoreFull()) { PyErr_SetString(NumbufPlasmaOutOfMemoryError, "The plasma store ran out of memory and could not create " "this object."); return NULL; } - CHECK(error_code == PlasmaError_OK); + ARROW_CHECK_OK(s); auto target = std::make_shared(LENGTH_PREFIX_SIZE + data, total_size); @@ -324,9 +327,9 @@ static PyObject* store_list(PyObject* self, PyObject* args) { *((int64_t*)data) = data_size; /* Do the plasma_release corresponding to the call to plasma_create. */ - plasma_release(conn, obj_id); + ARROW_CHECK_OK(client->Release(obj_id)); /* Seal the object. */ - plasma_seal(conn, obj_id); + ARROW_CHECK_OK(client->Seal(obj_id)); Py_RETURN_NONE; } @@ -350,13 +353,13 @@ static PyObject* store_list(PyObject* self, PyObject* args) { */ static PyObject* retrieve_list(PyObject* self, PyObject* args) { PyObject* object_id_list; - PyObject* plasma_conn; + PyObject* plasma_client; long long timeout_ms; - if (!PyArg_ParseTuple(args, "OOL", &object_id_list, &plasma_conn, &timeout_ms)) { + if (!PyArg_ParseTuple(args, "OOL", &object_id_list, &plasma_client, &timeout_ms)) { return NULL; } - PlasmaConnection* conn; - if (!PyObjectToPlasmaConnection(plasma_conn, &conn)) { return NULL; } + PlasmaClient* client; + if (!PyObjectToPlasmaClient(plasma_client, &client)) { return NULL; } Py_ssize_t num_object_ids = PyList_Size(object_id_list); ObjectID* object_ids = new ObjectID[num_object_ids]; @@ -367,7 +370,7 @@ static PyObject* retrieve_list(PyObject* self, PyObject* args) { } Py_BEGIN_ALLOW_THREADS; - plasma_get(conn, object_ids, num_object_ids, timeout_ms, object_buffers); + ARROW_CHECK_OK(client->Get(object_ids, num_object_ids, timeout_ms, object_buffers)); Py_END_ALLOW_THREADS; PyObject* returns = PyList_New(num_object_ids); @@ -384,8 +387,8 @@ static PyObject* retrieve_list(PyObject* self, PyObject* args) { * buffer is in scope. This prevents memory in the object store from getting * released while it is still being used to back a Python object. */ PyObject* base = PyCapsule_New(buffer_obj_id, "buffer", BufferCapsule_Destructor); - PyCapsule_SetContext(base, plasma_conn); - Py_XINCREF(plasma_conn); + PyCapsule_SetContext(base, plasma_client); + Py_XINCREF(plasma_client); auto batch = std::shared_ptr(); std::vector> tensors; diff --git a/src/plasma/CMakeLists.txt b/src/plasma/CMakeLists.txt index 69bcae616..a138c081e 100644 --- a/src/plasma/CMakeLists.txt +++ b/src/plasma/CMakeLists.txt @@ -43,7 +43,6 @@ include_directories("${CMAKE_CURRENT_LIST_DIR}/../") add_library(plasma SHARED plasma.cc plasma_extension.cc - ../common/lib/python/common_extension.cc plasma_protocol.cc plasma_client.cc thirdparty/xxhash.c @@ -52,9 +51,9 @@ add_library(plasma SHARED add_dependencies(plasma gen_plasma_fbs) if(APPLE) - target_link_libraries(plasma "-undefined dynamic_lookup" -Wl,-force_load,${FLATBUFFERS_STATIC_LIB} common ${FLATBUFFERS_STATIC_LIB} -lpthread) + target_link_libraries(plasma plasma_lib "-undefined dynamic_lookup" -Wl,-force_load,${FLATBUFFERS_STATIC_LIB} ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB} -lpthread) else(APPLE) - target_link_libraries(plasma -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive common ${FLATBUFFERS_STATIC_LIB} -lpthread) + target_link_libraries(plasma plasma_lib -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB} -lpthread) endif(APPLE) include_directories("${FLATBUFFERS_INCLUDE_DIR}") @@ -63,8 +62,22 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") set_source_files_properties(thirdparty/dlmalloc.c PROPERTIES COMPILE_FLAGS -Wno-all) +add_library(plasma_lib STATIC + plasma_client.cc + plasma.cc + plasma_common.cc + plasma_io.cc + plasma_protocol.cc + status.cc + fling.c + thirdparty/xxhash.c) + +target_link_libraries(plasma_lib ${FLATBUFFERS_STATIC_LIB} -lpthread) +add_dependencies(plasma_lib gen_plasma_fbs) + add_executable(plasma_store plasma_store.cc + thirdparty/ae/ae.c plasma.cc plasma_events.cc plasma_protocol.cc @@ -74,18 +87,7 @@ add_executable(plasma_store add_dependencies(plasma_store hiredis gen_plasma_fbs) -target_link_libraries(plasma_store common ${FLATBUFFERS_STATIC_LIB}) - -add_library(plasma_lib STATIC - plasma_client.cc - plasma.cc - plasma_events.cc - plasma_protocol.cc - fling.c - thirdparty/xxhash.c) - -target_link_libraries(plasma_lib common ${FLATBUFFERS_STATIC_LIB} -lpthread) -add_dependencies(plasma_lib gen_plasma_fbs) +target_link_libraries(plasma_store plasma_lib ${FLATBUFFERS_STATIC_LIB}) add_dependencies(plasma protocol_fbs) @@ -97,7 +99,7 @@ target_link_libraries(plasma_manager common plasma_lib ${FLATBUFFERS_STATIC_LIB} add_library(plasma_client SHARED plasma_client.cc) target_link_libraries(plasma_client ${FLATBUFFERS_STATIC_LIB}) -target_link_libraries(plasma_client common plasma_lib ${FLATBUFFERS_STATIC_LIB}) +target_link_libraries(plasma_client plasma_lib ${FLATBUFFERS_STATIC_LIB}) define_test(client_tests plasma_lib) define_test(manager_tests plasma_lib plasma_manager.cc) diff --git a/src/plasma/eviction_policy.cc b/src/plasma/eviction_policy.cc index cd1c3d27f..135c63ad0 100644 --- a/src/plasma/eviction_policy.cc +++ b/src/plasma/eviction_policy.cc @@ -2,7 +2,7 @@ void LRUCache::add(const ObjectID &key, int64_t size) { auto it = item_map_.find(key); - CHECK(it == item_map_.end()); + ARROW_CHECK(it == item_map_.end()); /* Note that it is important to use a list so the iterators stay valid. */ item_list_.emplace_front(key, size); item_map_.emplace(key, item_list_.begin()); @@ -10,7 +10,7 @@ void LRUCache::add(const ObjectID &key, int64_t size) { void LRUCache::remove(const ObjectID &key) { auto it = item_map_.find(key); - CHECK(it != item_map_.end()); + ARROW_CHECK(it != item_map_.end()); item_list_.erase(it->second); item_map_.erase(it); } @@ -58,15 +58,16 @@ bool EvictionPolicy::require_space(int64_t size, if (required_space > 0) { /* Try to free up at least as much space as we need right now but ideally * up to 20% of the total capacity. */ - int64_t space_to_free = MAX(size, store_info_->memory_capacity / 5); - LOG_DEBUG("not enough space to create this object, so evicting objects"); + int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5); + ARROW_LOG(DEBUG) + << "not enough space to create this object, so evicting objects"; /* Choose some objects to evict, and update the return pointers. */ num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict); - LOG_INFO( - "There is not enough space to create this object, so evicting " - "%zu objects to free up %" PRId64 " bytes.", - objects_to_evict.size(), num_bytes_evicted); + ARROW_LOG(INFO) + << "There is not enough space to create this object, so evicting " + << objects_to_evict.size() << " objects to free up " + << num_bytes_evicted << " bytes."; } else { num_bytes_evicted = 0; } diff --git a/src/plasma/eviction_policy.h b/src/plasma/eviction_policy.h index 4ff75f8b7..fd3861db4 100644 --- a/src/plasma/eviction_policy.h +++ b/src/plasma/eviction_policy.h @@ -4,6 +4,7 @@ #include #include +#include "plasma_common.h" #include "plasma.h" /* ==== The eviction policy ==== diff --git a/src/plasma/logging.h b/src/plasma/logging.h new file mode 100644 index 000000000..917d18140 --- /dev/null +++ b/src/plasma/logging.h @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_UTIL_LOGGING_H +#define ARROW_UTIL_LOGGING_H + +#include +#include + +namespace arrow { + +// Stubbed versions of macros defined in glog/logging.h, intended for +// environments where glog headers aren't available. +// +// Add more as needed. + +// Log levels. LOG ignores them, so their values are abitrary. + +#define ARROW_DEBUG (-1) +#define ARROW_INFO 0 +#define ARROW_WARNING 1 +#define ARROW_ERROR 2 +#define ARROW_FATAL 3 + +#define ARROW_LOG_INTERNAL(level) ::arrow::internal::CerrLog(level) +#define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level) + +#define ARROW_CHECK(condition) \ + (condition) ? 0 : ::arrow::internal::FatalLog(ARROW_FATAL) \ + << __FILE__ << __LINE__ \ + << " Check failed: " #condition " " + +#ifdef NDEBUG +#define ARROW_DFATAL ARROW_WARNING + +#define DCHECK(condition) \ + while (false) \ + ::arrow::internal::NullLog() +#define DCHECK_EQ(val1, val2) \ + while (false) \ + ::arrow::internal::NullLog() +#define DCHECK_NE(val1, val2) \ + while (false) \ + ::arrow::internal::NullLog() +#define DCHECK_LE(val1, val2) \ + while (false) \ + ::arrow::internal::NullLog() +#define DCHECK_LT(val1, val2) \ + while (false) \ + ::arrow::internal::NullLog() +#define DCHECK_GE(val1, val2) \ + while (false) \ + ::arrow::internal::NullLog() +#define DCHECK_GT(val1, val2) \ + while (false) \ + ::arrow::internal::NullLog() + +#else +#define ARROW_DFATAL ARROW_FATAL + +#define DCHECK(condition) ARROW_CHECK(condition) +#define DCHECK_EQ(val1, val2) ARROW_CHECK((val1) == (val2)) +#define DCHECK_NE(val1, val2) ARROW_CHECK((val1) != (val2)) +#define DCHECK_LE(val1, val2) ARROW_CHECK((val1) <= (val2)) +#define DCHECK_LT(val1, val2) ARROW_CHECK((val1) < (val2)) +#define DCHECK_GE(val1, val2) ARROW_CHECK((val1) >= (val2)) +#define DCHECK_GT(val1, val2) ARROW_CHECK((val1) > (val2)) + +#endif // NDEBUG + +namespace internal { + +class NullLog { + public: + template + NullLog &operator<<(const T &t) { + return *this; + } +}; + +class CerrLog { + public: + CerrLog(int severity) // NOLINT(runtime/explicit) + : severity_(severity), + has_logged_(false) {} + + virtual ~CerrLog() { + if (has_logged_) { + std::cerr << std::endl; + } + if (severity_ == ARROW_FATAL) { + std::exit(1); + } + } + + template + CerrLog &operator<<(const T &t) { + // TODO(pcm): Print this if in debug mode, but not if in valgrind + // mode + if (severity_ == ARROW_DEBUG) { + return *this; + } + + has_logged_ = true; + std::cerr << t; + return *this; + } + + protected: + const int severity_; + bool has_logged_; +}; + +// Clang-tidy isn't smart enough to determine that DCHECK using CerrLog doesn't +// return so we create a new class to give it a hint. +class FatalLog : public CerrLog { + public: + explicit FatalLog(int /* severity */) // NOLINT + : CerrLog(ARROW_FATAL){} // NOLINT + + [[noreturn]] ~FatalLog() { + if (has_logged_) { + std::cerr << std::endl; + } + std::exit(1); + } +}; + +} // namespace internal + +} // namespace arrow + +#endif // ARROW_UTIL_LOGGING_H diff --git a/src/plasma/plasma.cc b/src/plasma/plasma.cc index a420ab423..6273d3bbc 100644 --- a/src/plasma/plasma.cc +++ b/src/plasma/plasma.cc @@ -1,3 +1,4 @@ +#include "plasma_common.h" #include "plasma.h" #include "io.h" @@ -12,14 +13,15 @@ int warn_if_sigpipe(int status, int client_sock) { return 0; } if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) { - LOG_WARN( - "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " - "sending a message to client on fd %d. The client on the other end may " - "have hung up.", - client_sock); + ARROW_LOG(WARNING) + << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " + "sending a message to client on fd " + << client_sock << ". The client on the other end may " + "have hung up."; return errno; } - LOG_FATAL("Failed to write message to client on fd %d.", client_sock); + ARROW_LOG(FATAL) << "Failed to write message to client on fd " << client_sock + << "."; } /** diff --git a/src/plasma/plasma.h b/src/plasma/plasma.h index 4247fd5b4..4cd3f46dd 100644 --- a/src/plasma/plasma.h +++ b/src/plasma/plasma.h @@ -9,17 +9,41 @@ #include #include /* pid_t */ +extern "C" { +#include "sha256.h" +} + #include #include -#include "common.h" #include "format/common_generated.h" +#include "logging.h" +#include "status.h" #include +#define HANDLE_SIGPIPE(s, fd_) \ + do { \ + Status _s = (s); \ + if (!_s.ok()) { \ + if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) { \ + ARROW_LOG(WARNING) \ + << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " \ + "sending a message to client on fd " \ + << fd_ << ". " \ + "The client on the other end may have hung up."; \ + } else { \ + return _s; \ + } \ + } \ + } while (0); + /** Allocation granularity used in plasma for object allocation. */ #define BLOCK_SIZE 64 +// Size of object hash digests. +constexpr int64_t kDigestSize = SHA256_BLOCK_SIZE; + struct Client; /** @@ -113,7 +137,7 @@ struct ObjectTableEntry { /** The state of the object, e.g., whether it is open or sealed. */ object_state state; /** The digest of the object. Used to see if two objects are the same. */ - unsigned char digest[DIGEST_SIZE]; + unsigned char digest[kDigestSize]; }; /** The plasma store information that is exposed to the eviction policy. */ diff --git a/src/plasma/plasma_client.cc b/src/plasma/plasma_client.cc index 60985f8b9..c811b418b 100644 --- a/src/plasma/plasma_client.cc +++ b/src/plasma/plasma_client.cc @@ -1,4 +1,4 @@ -/* PLASMA CLIENT: Client library for using the plasma store and manager */ +// PLASMA CLIENT: Client library for using the plasma store and manager #ifdef _WIN32 #include @@ -16,18 +16,13 @@ #include #include #include -#include -#include -#include -#include "common.h" -#include "io.h" +#include "plasma_common.h" #include "plasma.h" +#include "plasma_io.h" #include "plasma_protocol.h" #include "plasma_client.h" -/* C++ includes */ -#include #include #include @@ -41,89 +36,39 @@ extern "C" { #define XXH64_DEFAULT_SEED 0 } -#define THREADPOOL_SIZE 8 -#define BYTES_IN_MB (1 << 20) -static std::vector threadpool_(THREADPOOL_SIZE); +// Number of threads used for memcopy and hash computations. +constexpr int64_t kThreadPoolSize = 8; +constexpr int64_t kBytesInMB = 1 << 20; +static std::vector threadpool_(kThreadPoolSize); struct ClientMmapTableEntry { - /** Key that uniquely identifies the memory mapped file. In practice, we - * take the numerical value of the file descriptor in the object store. */ - int key; - /** The result of mmap for this file descriptor. */ + /// The result of mmap for this file descriptor. uint8_t *pointer; - /** The length of the memory-mapped file. */ + /// The length of the memory-mapped file. size_t length; - /** The number of objects in this memory-mapped file that are currently being - * used by the client. When this count reaches zeros, we unmap the file. */ + /// The number of objects in this memory-mapped file that are currently being + /// used by the client. When this count reaches zeros, we unmap the file. int count; }; struct ObjectInUseEntry { - /** The ID of the object. This is used as the key in the hash table. */ - ObjectID object_id; - /** A count of the number of times this client has called plasma_create or - * plasma_get on this object ID minus the number of calls to plasma_release. - * When this count reaches zero, we remove the entry from the objects_in_use - * and decrement a count in the relevant ClientMmapTableEntry. */ + /// A count of the number of times this client has called PlasmaClient::Create + /// or + /// PlasmaClient::Get on this object ID minus the number of calls to + /// PlasmaClient::Release. + /// When this count reaches zero, we remove the entry from the ObjectsInUse + /// and decrement a count in the relevant ClientMmapTableEntry. int count; - /** Cached information to read the object. */ + /// Cached information to read the object. PlasmaObject object; - /** A flag representing whether the object has been sealed. */ + /// A flag representing whether the object has been sealed. bool is_sealed; }; -/** Configuration options for the plasma client. */ -typedef struct { - /** Number of release calls we wait until the object is actually released. - * This allows us to avoid invalidating the cpu cache on workers if objects - * are reused accross tasks. */ - int release_delay; -} plasma_client_config; - -/** Information about a connection between a Plasma Client and Plasma Store. - * This is used to avoid mapping the same files into memory multiple times. */ -struct PlasmaConnection { - /** File descriptor of the Unix domain socket that connects to the store. */ - int store_conn; - /** File descriptor of the Unix domain socket that connects to the manager. */ - int manager_conn; - /** File descriptor of the Unix domain socket on which client receives event - * notifications for the objects it subscribes for when these objects are - * sealed either locally or remotely. */ - int manager_conn_subscribe; - /** Buffer that holds memory for serializing plasma protocol messages. */ - protocol_builder *builder; - /** Table of dlmalloc buffer files that have been memory mapped so far. This - * is a hash table mapping a file descriptor to a struct containing the - * address of the corresponding memory-mapped file. */ - std::unordered_map mmap_table; - /** A hash table of the object IDs that are currently being used by this - * client. */ - std::unordered_map - objects_in_use; - /** Object IDs of the last few release calls. This is a deque and - * is used to delay releasing objects to see if they can be reused by - * subsequent tasks so we do not unneccessarily invalidate cpu caches. - * TODO(pcm): replace this with a proper lru cache using the size of the L3 - * cache. */ - std::deque release_history; - /** The number of bytes in the combined objects that are held in the release - * history doubly-linked list. If this is too large then the client starts - * releasing objects. */ - int64_t in_use_object_bytes; - /** Configuration options for the plasma client. */ - plasma_client_config config; - /** The amount of memory available to the Plasma store. The client needs this - * information to make sure that it does not delay in releasing so much - * memory that the store is unable to evict enough objects to free up space. - */ - int64_t store_capacity; -}; - -/* If the file descriptor fd has been mmapped in this client process before, - * return the pointer that was returned by mmap, otherwise mmap it and store the - * pointer in a hash table. */ -uint8_t *lookup_or_mmap(PlasmaConnection *conn, +// If the file descriptor fd has been mmapped in this client process before, +// return the pointer that was returned by mmap, otherwise mmap it and store the +// pointer in a hash table. +uint8_t *lookup_or_mmap(PlasmaClient *conn, int fd, int store_fd_val, int64_t map_size) { @@ -135,11 +80,10 @@ uint8_t *lookup_or_mmap(PlasmaConnection *conn, uint8_t *result = (uint8_t *) mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (result == MAP_FAILED) { - LOG_FATAL("mmap failed"); + ARROW_LOG(FATAL) << "mmap failed"; } close(fd); ClientMmapTableEntry *entry = new ClientMmapTableEntry(); - entry->key = store_fd_val; entry->pointer = result; entry->length = map_size; entry->count = 0; @@ -148,285 +92,277 @@ uint8_t *lookup_or_mmap(PlasmaConnection *conn, } } -/* Get a pointer to a file that we know has been memory mapped in this client - * process before. */ -uint8_t *lookup_mmapped_file(PlasmaConnection *conn, int store_fd_val) { +// Get a pointer to a file that we know has been memory mapped in this client +// process before. +uint8_t *lookup_mmapped_file(PlasmaClient *conn, int store_fd_val) { auto entry = conn->mmap_table.find(store_fd_val); - CHECK(entry != conn->mmap_table.end()); + ARROW_CHECK(entry != conn->mmap_table.end()); return entry->second->pointer; } -void increment_object_count(PlasmaConnection *conn, +void increment_object_count(PlasmaClient *conn, ObjectID object_id, PlasmaObject *object, bool is_sealed) { - /* Increment the count of the object to track the fact that it is being used. - * The corresponding decrement should happen in plasma_release. */ + // Increment the count of the object to track the fact that it is being used. + // The corresponding decrement should happen in PlasmaClient::Release. auto elem = conn->objects_in_use.find(object_id); ObjectInUseEntry *object_entry; if (elem == conn->objects_in_use.end()) { - /* Add this object ID to the hash table of object IDs in use. The - * corresponding call to free happens in plasma_release. */ + // Add this object ID to the hash table of object IDs in use. The + // corresponding call to free happens in PlasmaClient::Release. object_entry = new ObjectInUseEntry(); - object_entry->object_id = object_id; object_entry->object = *object; object_entry->count = 0; object_entry->is_sealed = is_sealed; conn->objects_in_use[object_id] = object_entry; - /* Increment the count of the number of objects in the memory-mapped file - * that are being used. The corresponding decrement should happen in - * plasma_release. */ + // Increment the count of the number of objects in the memory-mapped file + // that are being used. The corresponding decrement should happen in + // PlasmaClient::Release. auto entry = conn->mmap_table.find(object->handle.store_fd); - CHECK(entry != conn->mmap_table.end()); - CHECK(entry->second->count >= 0); - /* Update the in_use_object_bytes. */ + ARROW_CHECK(entry != conn->mmap_table.end()); + ARROW_CHECK(entry->second->count >= 0); + // Update the in_use_object_bytes. conn->in_use_object_bytes += (object_entry->object.data_size + object_entry->object.metadata_size); entry->second->count += 1; } else { object_entry = elem->second; - CHECK(object_entry->count > 0); + ARROW_CHECK(object_entry->count > 0); } - /* Increment the count of the number of instances of this object that are - * being used by this client. The corresponding decrement should happen in - * plasma_release. */ + // Increment the count of the number of instances of this object that are + // being used by this client. The corresponding decrement should happen in + // PlasmaClient::Release. object_entry->count += 1; } -int plasma_create(PlasmaConnection *conn, - ObjectID obj_id, - int64_t data_size, - uint8_t *metadata, - int64_t metadata_size, - uint8_t **data) { - LOG_DEBUG("called plasma_create on conn %d with size %" PRId64 - " and metadata size %" PRId64, - conn->store_conn, data_size, metadata_size); - CHECK(plasma_send_CreateRequest(conn->store_conn, conn->builder, obj_id, - data_size, metadata_size) >= 0); - uint8_t *reply_data = - plasma_receive(conn->store_conn, MessageType_PlasmaCreateReply); - int error; +Status PlasmaClient::Create(ObjectID object_id, + int64_t data_size, + uint8_t *metadata, + int64_t metadata_size, + uint8_t **data) { + ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn + << " with size " << data_size << " and metadata size " + << metadata_size; + RETURN_NOT_OK( + SendCreateRequest(store_conn, object_id, data_size, metadata_size)); + std::vector buffer; + RETURN_NOT_OK( + PlasmaReceive(store_conn, MessageType_PlasmaCreateReply, buffer)); ObjectID id; PlasmaObject object; - plasma_read_CreateReply(reply_data, &id, &object, &error); - free(reply_data); - if (error != PlasmaError_OK) { - LOG_DEBUG("returned from plasma_create with error %d", error); - CHECK(error == PlasmaError_OutOfMemory || - error == PlasmaError_ObjectExists); - return error; - } - /* If the CreateReply included an error, then the store will not send a file - * descriptor. */ - int fd = recv_fd(conn->store_conn); - CHECKM(fd >= 0, "recv not successful"); - CHECK(object.data_size == data_size); - CHECK(object.metadata_size == metadata_size); - /* The metadata should come right after the data. */ - CHECK(object.metadata_offset == object.data_offset + data_size); - *data = lookup_or_mmap(conn, fd, object.handle.store_fd, + RETURN_NOT_OK(ReadCreateReply(buffer.data(), &id, &object)); + // If the CreateReply included an error, then the store will not send a file + // descriptor. + int fd = recv_fd(store_conn); + ARROW_CHECK(fd >= 0) << "recv not successful"; + ARROW_CHECK(object.data_size == data_size); + ARROW_CHECK(object.metadata_size == metadata_size); + // The metadata should come right after the data. + ARROW_CHECK(object.metadata_offset == object.data_offset + data_size); + *data = lookup_or_mmap(this, fd, object.handle.store_fd, object.handle.mmap_size) + object.data_offset; - /* If plasma_create is being called from a transfer, then we will not copy the - * metadata here. The metadata will be written along with the data streamed - * from the transfer. */ + // If plasma_create is being called from a transfer, then we will not copy the + // metadata here. The metadata will be written along with the data streamed + // from the transfer. if (metadata != NULL) { - /* Copy the metadata to the buffer. */ + // Copy the metadata to the buffer. memcpy(*data + object.data_size, metadata, metadata_size); } - /* Increment the count of the number of instances of this object that this - * client is using. A call to plasma_release is required to decrement this - * count. Cache the reference to the object. */ - increment_object_count(conn, obj_id, &object, false); - /* We increment the count a second time (and the corresponding decrement will - * happen in a plasma_release call in plasma_seal) so even if the buffer - * returned by plasma_create goes out of scope, the object does not get - * released before the call to plasma_seal happens. */ - increment_object_count(conn, obj_id, &object, false); - return PlasmaError_OK; + // Increment the count of the number of instances of this object that this + // client is using. A call to PlasmaClient::Release is required to decrement + // this + // count. Cache the reference to the object. + increment_object_count(this, object_id, &object, false); + // We increment the count a second time (and the corresponding decrement will + // happen in a PlasmaClient::Release call in plasma_seal) so even if the + // buffer + // returned by PlasmaClient::Dreate goes out of scope, the object does not get + // released before the call to PlasmaClient::Seal happens. + increment_object_count(this, object_id, &object, false); + return Status::OK(); } -void plasma_get(PlasmaConnection *conn, - ObjectID object_ids[], - int64_t num_objects, - int64_t timeout_ms, - ObjectBuffer object_buffers[]) { - /* Fill out the info for the objects that are already in use locally. */ +Status PlasmaClient::Get(ObjectID object_ids[], + int64_t num_objects, + int64_t timeout_ms, + ObjectBuffer object_buffers[]) { + // Fill out the info for the objects that are already in use locally. bool all_present = true; for (int i = 0; i < num_objects; ++i) { - auto object_entry = conn->objects_in_use.find(object_ids[i]); - if (object_entry == conn->objects_in_use.end()) { - /* This object is not currently in use by this client, so we need to send - * a request to the store. */ + auto object_entry = objects_in_use.find(object_ids[i]); + if (object_entry == objects_in_use.end()) { + // This object is not currently in use by this client, so we need to send + // a request to the store. all_present = false; - /* Make a note to ourselves that the object is not present. */ + // Make a note to ourselves that the object is not present. object_buffers[i].data_size = -1; } else { - /* NOTE: If the object is still unsealed, we will deadlock, since we must - * have been the one who created it. */ - CHECKM(object_entry->second->is_sealed, - "Plasma client called get on an unsealed object that it created"); + // NOTE: If the object is still unsealed, we will deadlock, since we must + // have been the one who created it. + ARROW_CHECK(object_entry->second->is_sealed) + << "Plasma client called get on an unsealed object that it created"; PlasmaObject *object = &object_entry->second->object; object_buffers[i].data = - lookup_mmapped_file(conn, object->handle.store_fd); + lookup_mmapped_file(this, object->handle.store_fd); object_buffers[i].data = object_buffers[i].data + object->data_offset; object_buffers[i].data_size = object->data_size; object_buffers[i].metadata = object_buffers[i].data + object->data_size; object_buffers[i].metadata_size = object->metadata_size; - /* Increment the count of the number of instances of this object that this - * client is using. A call to plasma_release is required to decrement this - * count. Cache the reference to the object. */ - increment_object_count(conn, object_ids[i], object, true); + // Increment the count of the number of instances of this object that this + // client is using. A call to PlasmaClient::Release is required to + // decrement this + // count. Cache the reference to the object. + increment_object_count(this, object_ids[i], object, true); } } if (all_present) { - return; + return Status::OK(); } - /* If we get here, then the objects aren't all currently in use by this - * client, so we need to send a request to the plasma store. */ - CHECK(plasma_send_GetRequest(conn->store_conn, conn->builder, object_ids, - num_objects, timeout_ms) >= 0); - uint8_t *reply_data = - plasma_receive(conn->store_conn, MessageType_PlasmaGetReply); - ObjectID *received_obj_ids = - (ObjectID *) malloc(num_objects * sizeof(ObjectID)); - PlasmaObject *object_data = - (PlasmaObject *) malloc(num_objects * sizeof(PlasmaObject)); + // If we get here, then the objects aren't all currently in use by this + // client, so we need to send a request to the plasma store. + RETURN_NOT_OK( + SendGetRequest(store_conn, object_ids, num_objects, timeout_ms)); + std::vector buffer; + RETURN_NOT_OK(PlasmaReceive(store_conn, MessageType_PlasmaGetReply, buffer)); + std::vector received_object_ids(num_objects); + std::vector object_data(num_objects); PlasmaObject *object; - plasma_read_GetReply(reply_data, received_obj_ids, object_data, num_objects); - free(reply_data); + RETURN_NOT_OK(ReadGetReply(buffer.data(), received_object_ids.data(), + object_data.data(), num_objects)); for (int i = 0; i < num_objects; ++i) { - DCHECK(ObjectID_equal(received_obj_ids[i], object_ids[i])); + DCHECK(received_object_ids[i] == object_ids[i]); object = &object_data[i]; if (object_buffers[i].data_size != -1) { - /* If the object was already in use by the client, then the store should - * have returned it. */ + // If the object was already in use by the client, then the store should + // have returned it. DCHECK(object->data_size != -1); - /* We won't use this file descriptor, but the store sent us one, so we - * need to receive it and then close it right away so we don't leak file - * descriptors. */ - int fd = recv_fd(conn->store_conn); + // We won't use this file descriptor, but the store sent us one, so we + // need to receive it and then close it right away so we don't leak file + // descriptors. + int fd = recv_fd(store_conn); close(fd); - CHECK(fd >= 0); - /* We've already filled out the information for this object, so we can - * just continue. */ + ARROW_CHECK(fd >= 0); + // We've already filled out the information for this object, so we can + // just continue. continue; } - /* If we are here, the object was not currently in use, so we need to - * process the reply from the object store. */ + // If we are here, the object was not currently in use, so we need to + // process the reply from the object store. if (object->data_size != -1) { - /* The object was retrieved. The user will be responsible for releasing - * this object. */ - int fd = recv_fd(conn->store_conn); - CHECK(fd >= 0); - object_buffers[i].data = lookup_or_mmap(conn, fd, object->handle.store_fd, + // The object was retrieved. The user will be responsible for releasing + // this object. + int fd = recv_fd(store_conn); + ARROW_CHECK(fd >= 0); + object_buffers[i].data = lookup_or_mmap(this, fd, object->handle.store_fd, object->handle.mmap_size); - /* Finish filling out the return values. */ + // Finish filling out the return values. object_buffers[i].data = object_buffers[i].data + object->data_offset; object_buffers[i].data_size = object->data_size; object_buffers[i].metadata = object_buffers[i].data + object->data_size; object_buffers[i].metadata_size = object->metadata_size; - /* Increment the count of the number of instances of this object that this - * client is using. A call to plasma_release is required to decrement this - * count. Cache the reference to the object. */ - increment_object_count(conn, received_obj_ids[i], object, true); + // Increment the count of the number of instances of this object that this + // client is using. A call to PlasmaClient::Release is required to + // decrement this + // count. Cache the reference to the object. + increment_object_count(this, received_object_ids[i], object, true); } else { - /* The object was not retrieved. Make sure we already put a -1 here to - * indicate that the object was not retrieved. The caller is not - * responsible for releasing this object. */ + // The object was not retrieved. Make sure we already put a -1 here to + // indicate that the object was not retrieved. The caller is not + // responsible for releasing this object. DCHECK(object_buffers[i].data_size == -1); object_buffers[i].data_size = -1; } } - free(object_data); - free(received_obj_ids); + return Status::OK(); } -/** - * This is a helper method for implementing plasma_release. We maintain a buffer - * of release calls and only perform them once the buffer becomes full (as - * judged by the aggregate sizes of the objects). There may be multiple release - * calls for the same object ID in the buffer. In this case, the first release - * calls will not do anything. The client will only send a message to the store - * releasing the object when the client is truly done with the object. - * - * @param conn The plasma connection. - * @param object_id The object ID to attempt to release. - */ -void plasma_perform_release(PlasmaConnection *conn, ObjectID object_id) { - /* Decrement the count of the number of instances of this object that are - * being used by this client. The corresponding increment should have happened - * in plasma_get. */ - auto object_entry = conn->objects_in_use.find(object_id); - CHECK(object_entry != conn->objects_in_use.end()); +/// This is a helper method for implementing plasma_release. We maintain a +/// buffer +/// of release calls and only perform them once the buffer becomes full (as +/// judged by the aggregate sizes of the objects). There may be multiple release +/// calls for the same object ID in the buffer. In this case, the first release +/// calls will not do anything. The client will only send a message to the store +/// releasing the object when the client is truly done with the object. +/// +/// @param conn The plasma connection. +/// @param object_id The object ID to attempt to release. +Status PlasmaClient::PerformRelease(ObjectID object_id) { + // Decrement the count of the number of instances of this object that are + // being used by this client. The corresponding increment should have happened + // in PlasmaClient::Get. + auto object_entry = objects_in_use.find(object_id); + ARROW_CHECK(object_entry != objects_in_use.end()); object_entry->second->count -= 1; - CHECK(object_entry->second->count >= 0); - /* Check if the client is no longer using this object. */ + ARROW_CHECK(object_entry->second->count >= 0); + // Check if the client is no longer using this object. if (object_entry->second->count == 0) { - /* Decrement the count of the number of objects in this memory-mapped file - * that the client is using. The corresponding increment should have - * happened in plasma_get. */ + // Decrement the count of the number of objects in this memory-mapped file + // that the client is using. The corresponding increment should have + // happened in plasma_get. int fd = object_entry->second->object.handle.store_fd; - auto entry = conn->mmap_table.find(fd); - CHECK(entry != conn->mmap_table.end()); + auto entry = mmap_table.find(fd); + ARROW_CHECK(entry != mmap_table.end()); entry->second->count -= 1; - CHECK(entry->second->count >= 0); - /* If none are being used then unmap the file. */ + ARROW_CHECK(entry->second->count >= 0); + // If none are being used then unmap the file. if (entry->second->count == 0) { munmap(entry->second->pointer, entry->second->length); - /* Remove the corresponding entry from the hash table. */ + // Remove the corresponding entry from the hash table. delete entry->second; - conn->mmap_table.erase(fd); + mmap_table.erase(fd); } - /* Tell the store that the client no longer needs the object. */ - CHECK(plasma_send_ReleaseRequest(conn->store_conn, conn->builder, - object_id) >= 0); - /* Update the in_use_object_bytes. */ - conn->in_use_object_bytes -= (object_entry->second->object.data_size + - object_entry->second->object.metadata_size); - DCHECK(conn->in_use_object_bytes >= 0); - /* Remove the entry from the hash table of objects currently in use. */ + // Tell the store that the client no longer needs the object. + RETURN_NOT_OK(SendReleaseRequest(store_conn, object_id)); + // Update the in_use_object_bytes. + in_use_object_bytes -= (object_entry->second->object.data_size + + object_entry->second->object.metadata_size); + DCHECK(in_use_object_bytes >= 0); + // Remove the entry from the hash table of objects currently in use. delete object_entry->second; - conn->objects_in_use.erase(object_id); + objects_in_use.erase(object_id); } + return Status::OK(); } -void plasma_release(PlasmaConnection *conn, ObjectID obj_id) { - /* Add the new object to the release history. */ - conn->release_history.push_front(obj_id); - /* If there are too many bytes in use by the client or if there are too many - * pending release calls, and there are at least some pending release calls in - * the release_history list, then release some objects. */ - while ((conn->in_use_object_bytes > - MIN(L3_CACHE_SIZE_BYTES, conn->store_capacity / 100) || - conn->release_history.size() > conn->config.release_delay) && - conn->release_history.size() > 0) { - /* Perform a release for the object ID for the first pending release. */ - plasma_perform_release(conn, conn->release_history.back()); - /* Remove the last entry from the release history. */ - conn->release_history.pop_back(); +Status PlasmaClient::Release(ObjectID object_id) { + // Add the new object to the release history. + release_history.push_front(object_id); + // If there are too many bytes in use by the client or if there are too many + // pending release calls, and there are at least some pending release calls in + // the release_history list, then release some objects. + while ((in_use_object_bytes > + std::min(kL3CacheSizeBytes, store_capacity / 100) || + release_history.size() > config.release_delay) && + release_history.size() > 0) { + // Perform a release for the object ID for the first pending release. + RETURN_NOT_OK(PerformRelease(release_history.back())); + // Remove the last entry from the release history. + release_history.pop_back(); } + return Status::OK(); } -/* This method is used to query whether the plasma store contains an object. */ -void plasma_contains(PlasmaConnection *conn, ObjectID obj_id, int *has_object) { - /* Check if we already have a reference to the object. */ - if (conn->objects_in_use.count(obj_id) > 0) { +// This method is used to query whether the plasma store contains an object. +Status PlasmaClient::Contains(ObjectID object_id, int *has_object) { + // Check if we already have a reference to the object. + if (objects_in_use.count(object_id) > 0) { *has_object = 1; } else { - /* If we don't already have a reference to the object, check with the store - * to see if we have the object. */ - plasma_send_ContainsRequest(conn->store_conn, conn->builder, obj_id); - uint8_t *reply_data = - plasma_receive(conn->store_conn, MessageType_PlasmaContainsReply); + // If we don't already have a reference to the object, check with the store + // to see if we have the object. + RETURN_NOT_OK(SendContainsRequest(store_conn, object_id)); + std::vector buffer; + RETURN_NOT_OK( + PlasmaReceive(store_conn, MessageType_PlasmaContainsReply, buffer)); ObjectID object_id2; - plasma_read_ContainsReply(reply_data, &object_id2, has_object); - free(reply_data); + RETURN_NOT_OK(ReadContainsReply(buffer.data(), &object_id2, has_object)); } + return Status::OK(); } static void compute_block_hash(const unsigned char *data, @@ -441,18 +377,18 @@ static void compute_block_hash(const unsigned char *data, static inline bool compute_object_hash_parallel(XXH64_state_t *hash_state, const unsigned char *data, int64_t nbytes) { - /* Note that this function will likely be faster if the address of data is - * aligned on a 64-byte boundary. */ - const uint64_t num_threads = THREADPOOL_SIZE; + // Note that this function will likely be faster if the address of data is + // aligned on a 64-byte boundary. + const uint64_t num_threads = kThreadPoolSize; uint64_t threadhash[num_threads + 1]; const uint64_t data_address = reinterpret_cast(data); const uint64_t num_blocks = nbytes / BLOCK_SIZE; const uint64_t chunk_size = (num_blocks / num_threads) * BLOCK_SIZE; const uint64_t right_address = data_address + chunk_size * num_threads; const uint64_t suffix = (data_address + nbytes) - right_address; - /* Now the data layout is | k * num_threads * block_size | suffix | == - * | num_threads * chunk_size | suffix |, where chunk_size = k * block_size. - * Each thread gets a "chunk" of k blocks, except the suffix thread. */ + // Now the data layout is | k * num_threads * block_size | suffix | == + // | num_threads * chunk_size | suffix |, where chunk_size = k * block_size. + // Each thread gets a "chunk" of k blocks, except the suffix thread. for (int i = 0; i < num_threads; i++) { threadpool_[i] = @@ -463,7 +399,7 @@ static inline bool compute_object_hash_parallel(XXH64_state_t *hash_state, compute_block_hash(reinterpret_cast(right_address), suffix, &threadhash[num_threads]); - /* Join the threads. */ + // Join the threads. for (auto &t : threadpool_) { if (t.joinable()) { t.join(); @@ -477,7 +413,7 @@ static inline bool compute_object_hash_parallel(XXH64_state_t *hash_state, static uint64_t compute_object_hash(const ObjectBuffer &obj_buffer) { XXH64_state_t hash_state; XXH64_reset(&hash_state, XXH64_DEFAULT_SEED); - if (obj_buffer.data_size >= BYTES_IN_MB) { + if (obj_buffer.data_size >= kBytesInMB) { compute_object_hash_parallel(&hash_state, (unsigned char *) obj_buffer.data, obj_buffer.data_size); } else { @@ -489,198 +425,181 @@ static uint64_t compute_object_hash(const ObjectBuffer &obj_buffer) { return XXH64_digest(&hash_state); } -bool plasma_compute_object_hash(PlasmaConnection *conn, +bool plasma_compute_object_hash(PlasmaClient *conn, ObjectID obj_id, unsigned char *digest) { - /* Get the plasma object data. We pass in a timeout of 0 to indicate that - * the operation should timeout immediately. */ + // Get the plasma object data. We pass in a timeout of 0 to indicate that + // the operation should timeout immediately. ObjectBuffer obj_buffer; ObjectID obj_id_array[1] = {obj_id}; uint64_t hash; - plasma_get(conn, obj_id_array, 1, 0, &obj_buffer); - /* If the object was not retrieved, return false. */ + ARROW_CHECK_OK(conn->Get(obj_id_array, 1, 0, &obj_buffer)); + // If the object was not retrieved, return false. if (obj_buffer.data_size == -1) { return false; } - /* Compute the hash. */ + // Compute the hash. hash = compute_object_hash(obj_buffer); memcpy(digest, &hash, sizeof(hash)); - /* Release the plasma object. */ - plasma_release(conn, obj_id); + // Release the plasma object. + ARROW_CHECK_OK(conn->Release(obj_id)); return true; } -void plasma_seal(PlasmaConnection *conn, ObjectID object_id) { - /* Make sure this client has a reference to the object before sending the - * request to Plasma. */ - auto object_entry = conn->objects_in_use.find(object_id); - CHECKM(object_entry != conn->objects_in_use.end(), - "Plasma client called seal an object without a reference to it"); - CHECKM(!object_entry->second->is_sealed, - "Plasma client called seal an already sealed object"); +Status PlasmaClient::Seal(ObjectID object_id) { + // Make sure this client has a reference to the object before sending the + // request to Plasma. + auto object_entry = objects_in_use.find(object_id); + ARROW_CHECK(object_entry != objects_in_use.end()) + << "Plasma client called seal an object without a reference to it"; + ARROW_CHECK(!object_entry->second->is_sealed) + << "Plasma client called seal an already sealed object"; object_entry->second->is_sealed = true; - /* Send the seal request to Plasma. */ - static unsigned char digest[DIGEST_SIZE]; - CHECK(plasma_compute_object_hash(conn, object_id, &digest[0])); - CHECK(plasma_send_SealRequest(conn->store_conn, conn->builder, object_id, - &digest[0]) >= 0); - /* We call plasma_release to decrement the number of instances of this object - * that are currently being used by this client. The corresponding increment - * happened in plasma_create and was used to ensure that the object was not - * released before the call to plasma_seal. */ - plasma_release(conn, object_id); + /// Send the seal request to Plasma. + static unsigned char digest[kDigestSize]; + ARROW_CHECK(plasma_compute_object_hash(this, object_id, &digest[0])); + RETURN_NOT_OK(SendSealRequest(store_conn, object_id, &digest[0])); + // We call PlasmaClient::Release to decrement the number of instances of this + // object + // that are currently being used by this client. The corresponding increment + // happened in plasma_create and was used to ensure that the object was not + // released before the call to PlasmaClient::Seal. + return Release(object_id); } -void plasma_delete(PlasmaConnection *conn, ObjectID object_id) { - /* TODO(rkn): In the future, we can use this method to give hints to the - * eviction policy about when an object will no longer be needed. */ +Status PlasmaClient::Delete(ObjectID object_id) { + // TODO(rkn): In the future, we can use this method to give hints to the + // eviction policy about when an object will no longer be needed. + return Status::NotImplemented("PlasmaClient::Delete is not implemented."); } -int64_t plasma_evict(PlasmaConnection *conn, int64_t num_bytes) { - /* Send a request to the store to evict objects. */ - CHECK(plasma_send_EvictRequest(conn->store_conn, conn->builder, num_bytes) >= - 0); - /* Wait for a response with the number of bytes actually evicted. */ +Status PlasmaClient::Evict(int64_t num_bytes, int64_t &num_bytes_evicted) { + // Send a request to the store to evict objects. + RETURN_NOT_OK(SendEvictRequest(store_conn, num_bytes)); + // Wait for a response with the number of bytes actually evicted. + std::vector buffer; int64_t type; - int64_t length; - uint8_t *reply_data; - read_message(conn->store_conn, &type, &length, &reply_data); - int64_t num_bytes_evicted; - plasma_read_EvictReply(reply_data, &num_bytes_evicted); - free(reply_data); - return num_bytes_evicted; + RETURN_NOT_OK(ReadMessage(store_conn, &type, buffer)); + return ReadEvictReply(buffer.data(), num_bytes_evicted); } -int plasma_subscribe(PlasmaConnection *conn) { - int fd[2]; - /* TODO: Just create 1 socket, bind it to port 0 to find a free port, and - * send the port number instead, and let the client connect. */ - /* Create a non-blocking socket pair. This will only be used to send - * notifications from the Plasma store to the client. */ - socketpair(AF_UNIX, SOCK_STREAM, 0, fd); - /* Make the socket non-blocking. */ - int flags = fcntl(fd[1], F_GETFL, 0); - CHECK(fcntl(fd[1], F_SETFL, flags | O_NONBLOCK) == 0); - /* Tell the Plasma store about the subscription. */ - CHECK(plasma_send_SubscribeRequest(conn->store_conn, conn->builder) >= 0); - /* Send the file descriptor that the Plasma store should use to push - * notifications about sealed objects to this client. */ - CHECK(send_fd(conn->store_conn, fd[1]) >= 0); - close(fd[1]); - /* Return the file descriptor that the client should use to read notifications - * about sealed objects. */ - return fd[0]; +Status PlasmaClient::Subscribe(int &fd) { + int sock[2]; + // Create a non-blocking socket pair. This will only be used to send + // notifications from the Plasma store to the client. + socketpair(AF_UNIX, SOCK_STREAM, 0, sock); + // Make the socket non-blocking. + int flags = fcntl(sock[1], F_GETFL, 0); + ARROW_CHECK(fcntl(sock[1], F_SETFL, flags | O_NONBLOCK) == 0); + // Tell the Plasma store about the subscription. + RETURN_NOT_OK(SendSubscribeRequest(store_conn)); + // Send the file descriptor that the Plasma store should use to push + // notifications about sealed objects to this client. + ARROW_CHECK(send_fd(store_conn, sock[1]) >= 0); + close(sock[1]); + // Return the file descriptor that the client should use to read notifications + // about sealed objects. + fd = sock[0]; + return Status::OK(); } -PlasmaConnection *plasma_connect(const char *store_socket_name, - const char *manager_socket_name, - int release_delay) { - /* Initialize the store connection struct */ - PlasmaConnection *result = new PlasmaConnection(); - result->store_conn = connect_ipc_sock_retry(store_socket_name, -1, -1); - if (manager_socket_name != NULL) { - result->manager_conn = connect_ipc_sock_retry(manager_socket_name, -1, -1); +Status PlasmaClient::Connect(const std::string &store_socket_name, + const std::string &manager_socket_name, + int release_delay) { + store_conn = connect_ipc_sock_retry(store_socket_name, -1, -1); + if (manager_socket_name != "") { + manager_conn = connect_ipc_sock_retry(manager_socket_name, -1, -1); } else { - result->manager_conn = -1; + manager_conn = -1; } - result->builder = make_protocol_builder(); - result->config.release_delay = release_delay; - result->in_use_object_bytes = 0; - /* Send a ConnectRequest to the store to get its memory capacity. */ - plasma_send_ConnectRequest(result->store_conn, result->builder); - uint8_t *reply_data = - plasma_receive(result->store_conn, MessageType_PlasmaConnectReply); - plasma_read_ConnectReply(reply_data, &result->store_capacity); - free(reply_data); - return result; + config.release_delay = release_delay; + in_use_object_bytes = 0; + // Send a ConnectRequest to the store to get its memory capacity. + RETURN_NOT_OK(SendConnectRequest(store_conn)); + std::vector buffer; + RETURN_NOT_OK( + PlasmaReceive(store_conn, MessageType_PlasmaConnectReply, buffer)); + RETURN_NOT_OK(ReadConnectReply(buffer.data(), &store_capacity)); + return Status::OK(); } -void plasma_disconnect(PlasmaConnection *conn) { - /* NOTE: We purposefully do not finish sending release calls for objects in - * use, so that we don't duplicate plasma_release calls (when handling a - * SIGTERM, for example). */ - for (auto &entry : conn->objects_in_use) { +Status PlasmaClient::Disconnect() { + // NOTE: We purposefully do not finish sending release calls for objects in + // use, so that we don't duplicate PlasmaClient::Release calls (when handling + // a + // SIGTERM, for example). + for (auto &entry : objects_in_use) { delete entry.second; } - for (auto &entry : conn->mmap_table) { + for (auto &entry : mmap_table) { delete entry.second; } - free_protocol_builder(conn->builder); - /* Close the connections to Plasma. The Plasma store will release the objects - * that were in use by us when handling the SIGPIPE. */ - close(conn->store_conn); - if (conn->manager_conn >= 0) { - close(conn->manager_conn); + // Close the connections to Plasma. The Plasma store will release the objects + // that were in use by us when handling the SIGPIPE. + close(store_conn); + if (manager_conn >= 0) { + close(manager_conn); } - delete conn; + return Status::OK(); } -bool plasma_manager_is_connected(PlasmaConnection *conn) { +bool plasma_manager_is_connected(PlasmaClient *conn) { return conn->manager_conn >= 0; } #define h_addr h_addr_list[0] -void plasma_transfer(PlasmaConnection *conn, - const char *address, - int port, - ObjectID object_id) { - CHECK(plasma_send_DataRequest(conn->manager_conn, conn->builder, object_id, - address, port) >= 0); +Status PlasmaClient::Transfer(const char *address, + int port, + ObjectID object_id) { + return SendDataRequest(manager_conn, object_id, address, port); } -void plasma_fetch(PlasmaConnection *conn, - int num_object_ids, - ObjectID object_ids[]) { - CHECK(conn != NULL); - CHECK(conn->manager_conn >= 0); - CHECK(plasma_send_FetchRequest(conn->manager_conn, conn->builder, object_ids, - num_object_ids) >= 0); +Status PlasmaClient::Fetch(int num_object_ids, ObjectID object_ids[]) { + ARROW_CHECK(manager_conn >= 0); + return SendFetchRequest(manager_conn, object_ids, num_object_ids); } -int get_manager_fd(PlasmaConnection *conn) { +int get_manager_fd(PlasmaClient *conn) { return conn->manager_conn; } -int plasma_status(PlasmaConnection *conn, ObjectID object_id) { - CHECK(conn != NULL); - CHECK(conn->manager_conn >= 0); +Status PlasmaClient::Info(ObjectID object_id, int *object_status) { + ARROW_CHECK(manager_conn >= 0); - plasma_send_StatusRequest(conn->manager_conn, conn->builder, &object_id, 1); - uint8_t *reply_data = - plasma_receive(conn->manager_conn, MessageType_PlasmaStatusReply); - int object_status; - plasma_read_StatusReply(reply_data, &object_id, &object_status, 1); - free(reply_data); - return object_status; + RETURN_NOT_OK(SendStatusRequest(manager_conn, &object_id, 1)); + std::vector buffer; + RETURN_NOT_OK( + PlasmaReceive(manager_conn, MessageType_PlasmaStatusReply, buffer)); + return ReadStatusReply(buffer.data(), &object_id, object_status, 1); } -int plasma_wait(PlasmaConnection *conn, - int num_object_requests, - ObjectRequest object_requests[], - int num_ready_objects, - uint64_t timeout_ms) { - CHECK(conn != NULL); - CHECK(conn->manager_conn >= 0); - CHECK(num_object_requests > 0); - CHECK(num_ready_objects > 0); - CHECK(num_ready_objects <= num_object_requests); +Status PlasmaClient::Wait(int num_object_requests, + ObjectRequest object_requests[], + int num_ready_objects, + uint64_t timeout_ms, + int &num_objects_ready) { + ARROW_CHECK(manager_conn >= 0); + ARROW_CHECK(num_object_requests > 0); + ARROW_CHECK(num_ready_objects > 0); + ARROW_CHECK(num_ready_objects <= num_object_requests); for (int i = 0; i < num_object_requests; ++i) { - CHECK(object_requests[i].type == PLASMA_QUERY_LOCAL || - object_requests[i].type == PLASMA_QUERY_ANYWHERE); + ARROW_CHECK(object_requests[i].type == PLASMA_QUERY_LOCAL || + object_requests[i].type == PLASMA_QUERY_ANYWHERE); } - CHECK(plasma_send_WaitRequest(conn->manager_conn, conn->builder, - object_requests, num_object_requests, - num_ready_objects, timeout_ms) >= 0); - uint8_t *reply_data = - plasma_receive(conn->manager_conn, MessageType_PlasmaWaitReply); - plasma_read_WaitReply(reply_data, object_requests, &num_ready_objects); - free(reply_data); + RETURN_NOT_OK(SendWaitRequest(manager_conn, object_requests, + num_object_requests, num_ready_objects, + timeout_ms)); + std::vector buffer; + RETURN_NOT_OK( + PlasmaReceive(manager_conn, MessageType_PlasmaWaitReply, buffer)); + RETURN_NOT_OK( + ReadWaitReply(buffer.data(), object_requests, &num_ready_objects)); - int num_objects_ready = 0; + num_objects_ready = 0; for (int i = 0; i < num_object_requests; ++i) { int type = object_requests[i].type; int status = object_requests[i].status; @@ -694,12 +613,12 @@ int plasma_wait(PlasmaConnection *conn, if (status == ObjectStatus_Local || status == ObjectStatus_Remote) { num_objects_ready += 1; } else { - CHECK(status == ObjectStatus_Nonexistent); + ARROW_CHECK(status == ObjectStatus_Nonexistent); } break; default: - LOG_FATAL("This code should be unreachable."); + ARROW_LOG(FATAL) << "This code should be unreachable."; } } - return num_objects_ready; + return Status::OK(); } diff --git a/src/plasma/plasma_client.h b/src/plasma/plasma_client.h index d8f65eb8e..cd0384041 100644 --- a/src/plasma/plasma_client.h +++ b/src/plasma/plasma_client.h @@ -4,258 +4,311 @@ #include #include +#include + #include "plasma.h" +using arrow::Status; + #define PLASMA_DEFAULT_RELEASE_DELAY 64 -/* Use 100MB as an overestimate of the L3 cache size. */ -#define L3_CACHE_SIZE_BYTES 100000000 -typedef struct PlasmaConnection PlasmaConnection; +// Use 100MB as an overestimate of the L3 cache size. +constexpr int64_t kL3CacheSizeBytes = 100000000; -/** - * Try to connect to the socket several times. If unsuccessful, fail. - * - * @param socket_name Name of the Unix domain socket to connect to. - * @param num_retries Number of retries. - * @param timeout Timeout in milliseconds. - * @return File descriptor of the socket. - */ -int socket_connect_retry(const char *socket_name, - int num_retries, - int64_t timeout); - -/** - * Connect to the local plasma store and plasma manager. Return - * the resulting connection. - * - * @param store_socket_name The name of the UNIX domain socket to use to - * connect to the Plasma store. - * @param manager_socket_name The name of the UNIX domain socket to use to - * connect to the local Plasma manager. If this is NULL, then this - * function will not connect to a manager. - * @return The object containing the connection state. - */ -PlasmaConnection *plasma_connect(const char *store_socket_name, - const char *manager_socket_name, - int release_delay); - -/** - * Disconnect from the local plasma instance, including the local store and - * manager. - * - * @param conn The connection to the local plasma store and plasma manager. - * @return Void. - */ -void plasma_disconnect(PlasmaConnection *conn); - -/** - * Return true if the plasma manager is connected. - * - * @param conn The connection to the local plasma store and plasma manager. - * @return True if the plasma manager is connected and false otherwise. - */ -bool plasma_manager_is_connected(PlasmaConnection *conn); - -/** - * Try to connect to a possibly remote Plasma Manager. - * - * @param addr The IP address of the Plasma Manager to connect to. - * @param port The port of the Plasma Manager to connect to. - * @return The file descriptor to use to send messages to the - * Plasma Manager. If connection was unsuccessful, this - * value is -1. - */ -int plasma_manager_connect(const char *addr, int port); - -/** - * Create an object in the Plasma Store. Any metadata for this object must be - * be passed in when the object is created. - * - * @param conn The object containing the connection state. - * @param object_id The ID to use for the newly created object. - * @param size The size in bytes of the space to be allocated for this object's - data (this does not include space used for metadata). - * @param metadata The object's metadata. If there is no metadata, this pointer - should be NULL. - * @param metadata_size The size in bytes of the metadata. If there is no - metadata, this should be 0. - * @param data The address of the newly created object will be written here. - * @return One of the following error codes: - * - PlasmaError_OK, if the object was created successfully. - * - PlasmaError_ObjectExists, if an object with this ID is already - * present in the store. In this case, the client should not call - * plasma_release. - * - PlasmaError_OutOfMemory, if the store is out of memory and cannot - * create the object. In this case, the client should not call - * plasma_release. - */ -int plasma_create(PlasmaConnection *conn, - ObjectID object_id, - int64_t size, - uint8_t *metadata, - int64_t metadata_size, - uint8_t **data); - -/** - * Object buffer data structure. - */ -typedef struct { - /** The size in bytes of the data object. */ +/// Object buffer data structure. +struct ObjectBuffer { + /// The size in bytes of the data object. int64_t data_size; - /** The address of the data object. */ + /// The address of the data object. uint8_t *data; - /** The metadata size in bytes. */ + /// The metadata size in bytes. int64_t metadata_size; - /** The address of the metadata. */ + /// The address of the metadata. uint8_t *metadata; -} ObjectBuffer; +}; -/** - * Get some objects from the Plasma Store. This function will block until the - * objects have all been created and sealed in the Plasma Store or the timeout - * expires. The caller is responsible for releasing any retrieved objects, but - * the caller should not release objects that were not retrieved. - * - * @param conn The object containing the connection state. - * @param object_ids The IDs of the objects to get. - * @param num_object_ids The number of object IDs to get. - * @param timeout_ms The amount of time in milliseconds to wait before this - * request times out. If this value is -1, then no timeout is set. - * @param object_buffers An array where the results will be stored. If the data - * size field is -1, then the object was not retrieved. - * @return Void. - */ -void plasma_get(PlasmaConnection *conn, - ObjectID object_ids[], - int64_t num_objects, - int64_t timeout_ms, - ObjectBuffer object_buffers[]); +/// Configuration options for the plasma client. +struct PlasmaClientConfig { + /// Number of release calls we wait until the object is actually released. + /// This allows us to avoid invalidating the cpu cache on workers if objects + /// are reused accross tasks. + int release_delay; +}; -/** - * Tell Plasma that the client no longer needs the object. This should be called - * after plasma_get when the client is done with the object. After this call, - * the address returned by plasma_get is no longer valid. This should be called - * once for each call to plasma_get (with the same object ID). - * - * @param conn The object containing the connection state. - * @param object_id The ID of the object that is no longer needed. - * @return Void. - */ -void plasma_release(PlasmaConnection *conn, ObjectID object_id); +struct ClientMmapTableEntry; +struct ObjectInUseEntry; -/** - * Check if the object store contains a particular object and the object has - * been sealed. The result will be stored in has_object. - * - * @todo: We may want to indicate if the object has been created but not sealed. - * - * @param conn The object containing the connection state. - * @param object_id The ID of the object whose presence we are checking. - * @param has_object The function will write 1 at this address if the object is - * present and 0 if it is not present. - * @return Void. - */ -void plasma_contains(PlasmaConnection *conn, - ObjectID object_id, - int *has_object); +class PlasmaClient { + public: + /// Connect to the local plasma store and plasma manager. Return + /// the resulting connection. + /// + /// @param store_socket_name The name of the UNIX domain socket to use to + /// connect to the Plasma store. + /// @param manager_socket_name The name of the UNIX domain socket to use to + /// connect to the local Plasma manager. If this is NULL, then this + /// function will not connect to a manager. + /// @param release_delay Number of released objects that are kept around + /// and not evicted to avoid too many munmaps. + /// @return The return status. + Status Connect(const std::string &store_socket_name, + const std::string &manager_socket_name, + int release_delay); -/** - * Compute the hash of an object in the object store. - * - * @param conn The object containing the connection state. - * @param object_id The ID of the object we want to hash. - * @param digest A pointer at which to return the hash digest of the object. - * The pointer must have at least DIGEST_SIZE bytes allocated. - * @return A boolean representing whether the hash operation succeeded. - */ -bool plasma_compute_object_hash(PlasmaConnection *conn, + /// Create an object in the Plasma Store. Any metadata for this object must be + /// be passed in when the object is created. + /// + /// @param object_id The ID to use for the newly created object. + /// @param data_size The size in bytes of the space to be allocated for this + /// object's + /// data (this does not include space used for metadata). + /// @param metadata The object's metadata. If there is no metadata, this + /// pointer + /// should be NULL. + /// @param metadata_size The size in bytes of the metadata. If there is no + /// metadata, this should be 0. + /// @param data The address of the newly created object will be written here. + /// @return The return status. + Status Create(ObjectID object_id, + int64_t data_size, + uint8_t *metadata, + int64_t metadata_size, + uint8_t **data); + + /// Get some objects from the Plasma Store. This function will block until the + /// objects have all been created and sealed in the Plasma Store or the + /// timeout + /// expires. The caller is responsible for releasing any retrieved objects, + /// but + /// the caller should not release objects that were not retrieved. + /// + /// @param object_ids The IDs of the objects to get. + /// @param num_object_ids The number of object IDs to get. + /// @param timeout_ms The amount of time in milliseconds to wait before this + /// request times out. If this value is -1, then no timeout is set. + /// @param object_buffers An array where the results will be stored. If the + /// data + /// size field is -1, then the object was not retrieved. + /// @return The return status. + Status Get(ObjectID object_ids[], + int64_t num_objects, + int64_t timeout_ms, + ObjectBuffer object_buffers[]); + + /// Tell Plasma that the client no longer needs the object. This should be + /// called + /// after Get when the client is done with the object. After this call, + /// the address returned by Get is no longer valid. This should be called + /// once for each call to Get (with the same object ID). + /// + /// @param object_id The ID of the object that is no longer needed. + /// @return The return status. + Status Release(ObjectID object_id); + + /// Check if the object store contains a particular object and the object has + /// been sealed. The result will be stored in has_object. + /// + /// @todo: We may want to indicate if the object has been created but not + /// sealed. + /// + /// @param object_id The ID of the object whose presence we are checking. + /// @param has_object The function will write 1 at this address if the object + /// is + /// present and 0 if it is not present. + /// @return The return status. + Status Contains(ObjectID object_id, int *has_object); + + /// Seal an object in the object store. The object will be immutable after + /// this + /// call. + /// + /// @param object_id The ID of the object to seal. + /// @return The return status. + Status Seal(ObjectID object_id); + + /// Delete an object from the object store. This currently assumes that the + /// object is present and has been sealed. + /// + /// @todo We may want to allow the deletion of objects that are not present or + /// haven't been sealed. + /// + /// @param object_id The ID of the object to delete. + /// @return The return status. + Status Delete(ObjectID object_id); + + /// Delete objects until we have freed up num_bytes bytes or there are no more + /// released objects that can be deleted. + /// + /// @param num_bytes The number of bytes to try to free up. + /// @param num_bytes_evicted Out parameter for total number of bytes of space + /// retrieved. + /// @return The return status. + Status Evict(int64_t num_bytes, int64_t &num_bytes_evicted); + + /// Subscribe to notifications when objects are sealed in the object store. + /// Whenever an object is sealed, a message will be written to the client + /// socket + /// that is returned by this method. + /// + /// @param fd Out parameter for the file descriptor the client should use to + /// read notifications + /// from the object store about sealed objects. + /// @return The return status. + Status Subscribe(int &fd); + + /// Disconnect from the local plasma instance, including the local store and + /// manager. + /// + /// @return The return status. + Status Disconnect(); + + /// Attempt to initiate the transfer of some objects from remote Plasma + /// Stores. + /// This method does not guarantee that the fetched objects will arrive + /// locally. + /// + /// For an object that is available in the local Plasma Store, this method + /// will + /// not do anything. For an object that is not available locally, it will + /// check + /// if the object are already being fetched. If so, it will not do anything. + /// If + /// not, it will query the object table for a list of Plasma Managers that + /// have + /// the object. The object table will return a non-empty list, and this Plasma + /// Manager will attempt to initiate transfers from one of those Plasma + /// Managers. + /// + /// This function is non-blocking. + /// + /// This method is idempotent in the sense that it is ok to call it multiple + /// times. + /// + /// @param num_object_ids The number of object IDs fetch is being called on. + /// @param object_ids The IDs of the objects that fetch is being called on. + /// @return The return status. + Status Fetch(int num_object_ids, ObjectID object_ids[]); + + /// Wait for (1) a specified number of objects to be available (sealed) in the + /// local Plasma Store or in a remote Plasma Store, or (2) for a timeout to + /// expire. This is a blocking call. + /// + /// @param num_object_requests Size of the object_requests array. + /// @param object_requests Object event array. Each element contains a request + /// for a particular object_id. The type of request is specified in the + /// "type" field. + /// - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes + /// available in the local Plasma Store. In this case, this function + /// sets the "status" field to ObjectStatus_Local. Note, if the + /// status + /// is not ObjectStatus_Local, it will be ObjectStatus_Nonexistent, + /// but it may exist elsewhere in the system. + /// - A PLASMA_QUERY_ANYWHERE request is satisfied when object_id + /// becomes + /// available either at the local Plasma Store or on a remote Plasma + /// Store. In this case, the functions sets the "status" field to + /// ObjectStatus_Local or ObjectStatus_Remote. + /// @param num_ready_objects The number of requests in object_requests array + /// that + /// must be satisfied before the function returns, unless it timeouts. + /// The num_ready_objects should be no larger than num_object_requests. + /// @param timeout_ms Timeout value in milliseconds. If this timeout expires + /// before min_num_ready_objects of requests are satisfied, the + /// function + /// returns. + /// @param num_objects_ready Out parameter for number of satisfied requests in + /// the object_requests list. If the returned number is less than + /// min_num_ready_objects this means that timeout expired. + /// @return The return status. + Status Wait(int num_object_requests, + ObjectRequest object_requests[], + int num_ready_objects, + uint64_t timeout_ms, + int &num_objects_ready); + + /// Transfer local object to a different plasma manager. + /// + /// @param conn The object containing the connection state. + /// @param addr IP address of the plasma manager we are transfering to. + /// @param port Port of the plasma manager we are transfering to. + /// @object_id ObjectID of the object we are transfering. + /// @return The return status. + Status Transfer(const char *addr, int port, ObjectID object_id); + + /// Return the status of a given object. This method may query the object + /// table. + /// + /// @param conn The object containing the connection state. + /// @param object_id The ID of the object whose status we query. + /// @param object_status Out parameter for object status. Can take the + /// following values. + /// - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma + /// Store. + /// has been already scheduled by the Plasma Manager. + /// - PLASMA_CLIENT_TRANSFER, if the object is either currently being + /// transferred or just scheduled. + /// - PLASMA_CLIENT_REMOTE, if the object is stored at a remote + /// Plasma Store. + /// - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the + /// system. + /// @return The return status. + Status Info(ObjectID object_id, int *object_status); + + // private: + + Status PerformRelease(ObjectID object_id); + + /// File descriptor of the Unix domain socket that connects to the store. + int store_conn; + /// File descriptor of the Unix domain socket that connects to the manager. + int manager_conn; + /// File descriptor of the Unix domain socket on which client receives event + /// notifications for the objects it subscribes for when these objects are + /// sealed either locally or remotely. + int manager_conn_subscribe; + /// Table of dlmalloc buffer files that have been memory mapped so far. This + /// is a hash table mapping a file descriptor to a struct containing the + /// address of the corresponding memory-mapped file. + std::unordered_map mmap_table; + /// A hash table of the object IDs that are currently being used by this + /// client. + std::unordered_map + objects_in_use; + /// Object IDs of the last few release calls. This is a deque and + /// is used to delay releasing objects to see if they can be reused by + /// subsequent tasks so we do not unneccessarily invalidate cpu caches. + /// TODO(pcm): replace this with a proper lru cache using the size of the L3 + /// cache. + std::deque release_history; + /// The number of bytes in the combined objects that are held in the release + /// history doubly-linked list. If this is too large then the client starts + /// releasing objects. + int64_t in_use_object_bytes; + /// Configuration options for the plasma client. + PlasmaClientConfig config; + /// The amount of memory available to the Plasma store. The client needs this + /// information to make sure that it does not delay in releasing so much + /// memory that the store is unable to evict enough objects to free up space. + int64_t store_capacity; +}; + +/// Return true if the plasma manager is connected. +/// +/// @param conn The connection to the local plasma store and plasma manager. +/// @return True if the plasma manager is connected and false otherwise. +bool plasma_manager_is_connected(PlasmaClient *conn); + +/// Compute the hash of an object in the object store. +/// +/// @param conn The object containing the connection state. +/// @param object_id The ID of the object we want to hash. +/// @param digest A pointer at which to return the hash digest of the object. +/// The pointer must have at least DIGEST_SIZE bytes allocated. +/// @return A boolean representing whether the hash operation succeeded. +bool plasma_compute_object_hash(PlasmaClient *conn, ObjectID object_id, unsigned char *digest); -/** - * Seal an object in the object store. The object will be immutable after this - * call. - * - * @param conn The object containing the connection state. - * @param object_id The ID of the object to seal. - * @return Void. - */ -void plasma_seal(PlasmaConnection *conn, ObjectID object_id); - -/** - * Delete an object from the object store. This currently assumes that the - * object is present and has been sealed. - * - * @todo We may want to allow the deletion of objects that are not present or - * haven't been sealed. - * - * @param conn The object containing the connection state. - * @param object_id The ID of the object to delete. - * @return Void. - */ -void plasma_delete(PlasmaConnection *conn, ObjectID object_id); - -/** - * Delete objects until we have freed up num_bytes bytes or there are no more - * released objects that can be deleted. - * - * @param conn The object containing the connection state. - * @param num_bytes The number of bytes to try to free up. - * @return The total number of bytes of space retrieved. - */ -int64_t plasma_evict(PlasmaConnection *conn, int64_t num_bytes); - -/** - * Attempt to initiate the transfer of some objects from remote Plasma Stores. - * This method does not guarantee that the fetched objects will arrive locally. - * - * For an object that is available in the local Plasma Store, this method will - * not do anything. For an object that is not available locally, it will check - * if the object are already being fetched. If so, it will not do anything. If - * not, it will query the object table for a list of Plasma Managers that have - * the object. The object table will return a non-empty list, and this Plasma - * Manager will attempt to initiate transfers from one of those Plasma Managers. - * - * This function is non-blocking. - * - * This method is idempotent in the sense that it is ok to call it multiple - * times. - * - * @param conn The object containing the connection state. - * @param num_object_ids The number of object IDs fetch is being called on. - * @param object_ids The IDs of the objects that fetch is being called on. - * @return Void. - */ -void plasma_fetch(PlasmaConnection *conn, - int num_object_ids, - ObjectID object_ids[]); - -/** - * Transfer local object to a different plasma manager. - * - * @param conn The object containing the connection state. - * @param addr IP address of the plasma manager we are transfering to. - * @param port Port of the plasma manager we are transfering to. - * @object_id ObjectID of the object we are transfering. - * - * @return Void. - */ -void plasma_transfer(PlasmaConnection *conn, - const char *addr, - int port, - ObjectID object_id); - -/** - * Subscribe to notifications when objects are sealed in the object store. - * Whenever an object is sealed, a message will be written to the client socket - * that is returned by this method. - * - * @param conn The object containing the connection state. - * @return The file descriptor that the client should use to read notifications - from the object store about sealed objects. - */ -int plasma_subscribe(PlasmaConnection *conn); - /** * Get the file descriptor for the socket connection to the plasma manager. * @@ -263,25 +316,7 @@ int plasma_subscribe(PlasmaConnection *conn); * @return The file descriptor for the manager connection. If there is no * connection to the manager, this is -1. */ -int get_manager_fd(PlasmaConnection *conn); - -/** - * Return the status of a given object. This method may query the object table. - * - * @param conn The object containing the connection state. - * @param object_id The ID of the object whose status we query. - * @return Status as returned by get_status() function. Status can take the - * following values. - * - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma Store. - * has been already scheduled by the Plasma Manager. - * - PLASMA_CLIENT_TRANSFER, if the object is either currently being - * transferred or just scheduled. - * - PLASMA_CLIENT_REMOTE, if the object is stored at a remote - * Plasma Store. - * - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the - * system. - */ -int plasma_status(PlasmaConnection *conn, ObjectID object_id); +int get_manager_fd(PlasmaClient *conn); /** * Return the information associated to a given object. @@ -293,43 +328,8 @@ int plasma_status(PlasmaConnection *conn, ObjectID object_id); * PLASMA_CLIENT_NOT_LOCAL, if not. In this case, the caller needs to * ignore data, metadata_size, and metadata fields. */ -int plasma_info(PlasmaConnection *conn, - ObjectID object_id, - ObjectInfo *object_info); - -/** - * Wait for (1) a specified number of objects to be available (sealed) in the - * local Plasma Store or in a remote Plasma Store, or (2) for a timeout to - * expire. This is a blocking call. - * - * @param conn The object containing the connection state. - * @param num_object_requests Size of the object_requests array. - * @param object_requests Object event array. Each element contains a request - * for a particular object_id. The type of request is specified in the - * "type" field. - * - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes - * available in the local Plasma Store. In this case, this function - * sets the "status" field to ObjectStatus_Local. Note, if the status - * is not ObjectStatus_Local, it will be ObjectStatus_Nonexistent, - * but it may exist elsewhere in the system. - * - A PLASMA_QUERY_ANYWHERE request is satisfied when object_id becomes - * available either at the local Plasma Store or on a remote Plasma - * Store. In this case, the functions sets the "status" field to - * ObjectStatus_Local or ObjectStatus_Remote. - * @param num_ready_objects The number of requests in object_requests array that - * must be satisfied before the function returns, unless it timeouts. - * The num_ready_objects should be no larger than num_object_requests. - * @param timeout_ms Timeout value in milliseconds. If this timeout expires - * before min_num_ready_objects of requests are satisfied, the function - * returns. - * @return Number of satisfied requests in the object_requests list. If the - * returned number is less than min_num_ready_objects this means that - * timeout expired. - */ -int plasma_wait(PlasmaConnection *conn, - int num_object_requests, - ObjectRequest object_requests[], - int num_ready_objects, - uint64_t timeout_ms); +// int plasma_info(PlasmaConnection *conn, +// ObjectID object_id, +// ObjectInfo *object_info); #endif /* PLASMA_CLIENT_H */ diff --git a/src/plasma/plasma_common.cc b/src/plasma/plasma_common.cc new file mode 100644 index 000000000..d09be2d35 --- /dev/null +++ b/src/plasma/plasma_common.cc @@ -0,0 +1,67 @@ +#include "plasma_common.h" + +#include + +#include "format/plasma_generated.h" + +using arrow::Status; + +UniqueID UniqueID::from_random() { + UniqueID id; + uint8_t *data = id.mutable_data(); + std::random_device engine; + for (int i = 0; i < kUniqueIDSize; i++) { + data[i] = engine(); + } + return id; +} + +UniqueID UniqueID::from_binary(const std::string &binary) { + UniqueID id; + std::memcpy(&id, binary.data(), sizeof(id)); + return id; +} + +const uint8_t *UniqueID::data() const { + return id_; +} + +uint8_t *UniqueID::mutable_data() { + return id_; +} + +std::string UniqueID::binary() const { + return std::string(reinterpret_cast(id_), kUniqueIDSize); +} + +std::string UniqueID::hex() const { + constexpr char hex[] = "0123456789abcdef"; + std::string result; + for (int i = 0; i < sizeof(UniqueID); i++) { + unsigned int val = id_[i]; + result.push_back(hex[val >> 4]); + result.push_back(hex[val & 0xf]); + } + return result; +} + +bool UniqueID::operator==(const UniqueID &rhs) const { + return std::memcmp(data(), rhs.data(), kUniqueIDSize) == 0; +} + +Status plasma_error_status(int plasma_error) { + switch (plasma_error) { + case PlasmaError_OK: + return Status::OK(); + case PlasmaError_ObjectExists: + return Status::PlasmaObjectExists( + "object already exists in the plasma store"); + case PlasmaError_ObjectNonexistent: + return Status::PlasmaObjectNonexistent( + "object does not exist in the plasma store"); + case PlasmaError_OutOfMemory: + return Status::PlasmaStoreFull("object does not fit in the plasma store"); + default: + ARROW_LOG(FATAL) << "unknown plasma error code " << plasma_error; + } +} diff --git a/src/plasma/plasma_common.h b/src/plasma/plasma_common.h new file mode 100644 index 000000000..a6fff3803 --- /dev/null +++ b/src/plasma/plasma_common.h @@ -0,0 +1,41 @@ +#ifndef PLASMA_COMMON_H +#define PLASMA_COMMON_H + +#include +#include + +#include "logging.h" +#include "status.h" + +constexpr int64_t kUniqueIDSize = 20; + +class UniqueID { + public: + static UniqueID from_random(); + static UniqueID from_binary(const std::string &binary); + bool operator==(const UniqueID &rhs) const; + const uint8_t *data() const; + uint8_t *mutable_data(); + std::string binary() const; + std::string hex() const; + + private: + uint8_t id_[kUniqueIDSize]; +}; + +static_assert(std::is_pod::value, "UniqueID must be plain old data"); + +struct UniqueIDHasher { + /* ObjectID hashing function. */ + size_t operator()(const UniqueID &id) const { + size_t result; + std::memcpy(&result, id.data(), sizeof(size_t)); + return result; + } +}; + +typedef UniqueID ObjectID; + +arrow::Status plasma_error_status(int plasma_error); + +#endif // PLASMA_COMMON_H diff --git a/src/plasma/plasma_events.cc b/src/plasma/plasma_events.cc index 0ce3c5de3..883b745d5 100644 --- a/src/plasma/plasma_events.cc +++ b/src/plasma/plasma_events.cc @@ -29,10 +29,10 @@ bool EventLoop::add_file_event(int fd, int events, FileCallback callback) { } auto data = std::unique_ptr(new FileCallback(callback)); void *context = reinterpret_cast(data.get()); - /* Try to add the file descriptor. */ + // Try to add the file descriptor. int err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context); - /* If it cannot be added, increase the size of the event loop. */ + // If it cannot be added, increase the size of the event loop. if (err == AE_ERR && errno == ERANGE) { err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2); if (err != AE_OK) { @@ -41,7 +41,7 @@ bool EventLoop::add_file_event(int fd, int events, FileCallback callback) { err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, context); } - /* In any case, test if there were errors. */ + // In any case, test if there were errors. if (err == AE_OK) { file_callbacks_.emplace(fd, std::move(data)); return true; diff --git a/src/plasma/plasma_events.h b/src/plasma/plasma_events.h index 099a6e0ef..c94025f96 100644 --- a/src/plasma/plasma_events.h +++ b/src/plasma/plasma_events.h @@ -9,73 +9,62 @@ extern "C" { #include "ae/ae.h" } -/** Constant specifying that the timer is done and it will be removed. */ +/// Constant specifying that the timer is done and it will be removed. constexpr int kEventLoopTimerDone = AE_NOMORE; -/** Read event on the file descriptor. */ +/// Read event on the file descriptor. constexpr int kEventLoopRead = AE_READABLE; -/** Write event on the file descriptor. */ +/// Write event on the file descriptor. constexpr int kEventLoopWrite = AE_WRITABLE; class EventLoop { public: - /* Signature of the handler that will be called when there is a new event - * on the file descriptor that this handler has been registered for. - * - * The arguments are the event flags (read or write). - */ + // Signature of the handler that will be called when there is a new event + // on the file descriptor that this handler has been registered for. + // + // The arguments are the event flags (read or write). typedef std::function FileCallback; - /* This handler will be called when a timer times out. The timer id is - * passed as an argument. The return is the number of milliseconds the timer - * shall be reset to or kEventLoopTimerDone if the timer shall not be - * triggered again. - */ + // This handler will be called when a timer times out. The timer id is + // passed as an argument. The return is the number of milliseconds the timer + // shall be reset to or kEventLoopTimerDone if the timer shall not be + // triggered again. typedef std::function TimerCallback; EventLoop(); - /** - * Add a new file event handler to the event loop. - * - * @param fd The file descriptor we are listening to. - * @param events The flags for events we are listening to (read or write). - * @param callback The callback that will be called when the event happens. - * @return Returns true if the event handler was added successfully. - */ + /// Add a new file event handler to the event loop. + /// + /// @param fd The file descriptor we are listening to. + /// @param events The flags for events we are listening to (read or write). + /// @param callback The callback that will be called when the event happens. + /// @return Returns true if the event handler was added successfully. bool add_file_event(int fd, int events, FileCallback callback); - /** - * Remove a file event handler from the event loop. - * - * @param fd The file descriptor of the event handler. - * @return Void. - */ + /// Remove a file event handler from the event loop. + /// + /// @param fd The file descriptor of the event handler. + /// @return Void. void remove_file_event(int fd); - /** Register a handler that will be called after a time slice of - * "timeout" milliseconds. - * - * @param timeout The timeout in milliseconds. - * @param callback The callback for the timeout. - * @return The ID of the newly created timer. - */ + /// Register a handler that will be called after a time slice of + /// "timeout" milliseconds. + /// + /// @param timeout The timeout in milliseconds. + /// @param callback The callback for the timeout. + /// @return The ID of the newly created timer. int64_t add_timer(int64_t timeout, TimerCallback callback); - /** - * Remove a timer handler from the event loop. - * - * @param timer_id The ID of the timer that is to be removed. - * @return The ae.c error code. TODO(pcm): needs to be standardized - */ + /// Remove a timer handler from the event loop. + /// + /// @param timer_id The ID of the timer that is to be removed. + /// @return The ae.c error code. TODO(pcm): needs to be standardized int remove_timer(int64_t timer_id); - /** - * Run the event loop. - * - * @return Void. - */ + /// Run the event loop. + /// + /// @return Void. void run(); private: @@ -93,4 +82,4 @@ class EventLoop { std::unordered_map> timer_callbacks_; }; -#endif /* PLASMA_EVENTS */ +#endif // PLASMA_EVENTS diff --git a/src/plasma/plasma_extension.cc b/src/plasma/plasma_extension.cc index d884dfaeb..58d45ec3c 100644 --- a/src/plasma/plasma_extension.cc +++ b/src/plasma/plasma_extension.cc @@ -1,9 +1,8 @@ #include #include "bytesobject.h" -#include "common_extension.h" -#include "common.h" -#include "io.h" +#include "plasma_io.h" +#include "plasma_common.h" #include "plasma_protocol.h" #include "plasma_client.h" @@ -20,38 +19,35 @@ PyObject *PyPlasma_connect(PyObject *self, PyObject *args) { &release_delay)) { return NULL; } - PlasmaConnection *conn; - if (strlen(manager_socket_name) == 0) { - conn = plasma_connect(store_socket_name, NULL, release_delay); - } else { - conn = - plasma_connect(store_socket_name, manager_socket_name, release_delay); - } - return PyCapsule_New(conn, "plasma", NULL); + PlasmaClient *client = new PlasmaClient(); + ARROW_CHECK_OK( + client->Connect(store_socket_name, manager_socket_name, release_delay)); + + return PyCapsule_New(client, "plasma", NULL); } PyObject *PyPlasma_disconnect(PyObject *self, PyObject *args) { - PyObject *conn_capsule; - PlasmaConnection *conn; - if (!PyArg_ParseTuple(args, "O", &conn_capsule)) { + PyObject *client_capsule; + if (!PyArg_ParseTuple(args, "O", &client_capsule)) { return NULL; } - CHECK(PyObjectToPlasmaConnection(conn_capsule, &conn)); - plasma_disconnect(conn); + PlasmaClient *client; + ARROW_CHECK(PyObjectToPlasmaClient(client_capsule, &client)); + ARROW_CHECK_OK(client->Disconnect()); /* We use the context of the connection capsule to indicate if the connection * is still active (if the context is NULL) or if it is closed (if the context * is (void*) 0x1). This is neccessary because the primary pointer of the * capsule cannot be NULL. */ - PyCapsule_SetContext(conn_capsule, (void *) 0x1); + PyCapsule_SetContext(client_capsule, (void *) 0x1); Py_RETURN_NONE; } PyObject *PyPlasma_create(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; ObjectID object_id; long long size; PyObject *metadata; - if (!PyArg_ParseTuple(args, "O&O&LO", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&O&LO", PyObjectToPlasmaClient, &client, PyStringToUniqueID, &object_id, &size, &metadata)) { return NULL; } @@ -60,22 +56,22 @@ PyObject *PyPlasma_create(PyObject *self, PyObject *args) { return NULL; } uint8_t *data; - int error_code = plasma_create(conn, object_id, size, - (uint8_t *) PyByteArray_AsString(metadata), - PyByteArray_Size(metadata), &data); - if (error_code == PlasmaError_ObjectExists) { + Status s = client->Create(object_id, size, + (uint8_t *) PyByteArray_AsString(metadata), + PyByteArray_Size(metadata), &data); + if (s.IsPlasmaObjectExists()) { PyErr_SetString(PlasmaObjectExistsError, "An object with this ID already exists in the plasma " "store."); return NULL; } - if (error_code == PlasmaError_OutOfMemory) { + if (s.IsPlasmaStoreFull()) { PyErr_SetString(PlasmaOutOfMemoryError, "The plasma store ran out of memory and could not create " "this object."); return NULL; } - CHECK(error_code == PlasmaError_OK); + ARROW_CHECK(s.ok()); #if PY_MAJOR_VERSION >= 3 return PyMemoryView_FromMemory((char *) data, (Py_ssize_t) size, PyBUF_WRITE); @@ -85,17 +81,17 @@ PyObject *PyPlasma_create(PyObject *self, PyObject *args) { } PyObject *PyPlasma_hash(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID, &object_id)) { return NULL; } - unsigned char digest[DIGEST_SIZE]; - bool success = plasma_compute_object_hash(conn, object_id, digest); + unsigned char digest[kDigestSize]; + bool success = plasma_compute_object_hash(client, object_id, digest); if (success) { PyObject *digest_string = - PyBytes_FromStringAndSize((char *) digest, DIGEST_SIZE); + PyBytes_FromStringAndSize((char *) digest, kDigestSize); return digest_string; } else { Py_RETURN_NONE; @@ -103,32 +99,32 @@ PyObject *PyPlasma_hash(PyObject *self, PyObject *args) { } PyObject *PyPlasma_seal(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID, &object_id)) { return NULL; } - plasma_seal(conn, object_id); + ARROW_CHECK_OK(client->Seal(object_id)); Py_RETURN_NONE; } PyObject *PyPlasma_release(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID, &object_id)) { return NULL; } - plasma_release(conn, object_id); + ARROW_CHECK_OK(client->Release(object_id)); Py_RETURN_NONE; } PyObject *PyPlasma_get(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; PyObject *object_id_list; long long timeout_ms; - if (!PyArg_ParseTuple(args, "O&OL", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&OL", PyObjectToPlasmaClient, &client, &object_id_list, &timeout_ms)) { return NULL; } @@ -143,7 +139,8 @@ PyObject *PyPlasma_get(PyObject *self, PyObject *args) { } Py_BEGIN_ALLOW_THREADS; - plasma_get(conn, object_ids, num_object_ids, timeout_ms, object_buffers); + ARROW_CHECK_OK( + client->Get(object_ids, num_object_ids, timeout_ms, object_buffers)); Py_END_ALLOW_THREADS; free(object_ids); @@ -182,14 +179,14 @@ PyObject *PyPlasma_get(PyObject *self, PyObject *args) { } PyObject *PyPlasma_contains(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID, &object_id)) { return NULL; } int has_object; - plasma_contains(conn, object_id, &has_object); + ARROW_CHECK_OK(client->Contains(object_id, &has_object)); if (has_object) Py_RETURN_TRUE; @@ -198,13 +195,13 @@ PyObject *PyPlasma_contains(PyObject *self, PyObject *args) { } PyObject *PyPlasma_fetch(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; PyObject *object_id_list; - if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaClient, &client, &object_id_list)) { return NULL; } - if (!plasma_manager_is_connected(conn)) { + if (!plasma_manager_is_connected(client)) { PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager"); return NULL; } @@ -213,23 +210,23 @@ PyObject *PyPlasma_fetch(PyObject *self, PyObject *args) { for (int i = 0; i < n; ++i) { PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]); } - plasma_fetch(conn, (int) n, object_ids); + ARROW_CHECK_OK(client->Fetch((int) n, object_ids)); free(object_ids); Py_RETURN_NONE; } PyObject *PyPlasma_wait(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; PyObject *object_id_list; long long timeout; int num_returns; - if (!PyArg_ParseTuple(args, "O&OLi", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&OLi", PyObjectToPlasmaClient, &client, &object_id_list, &timeout, &num_returns)) { return NULL; } Py_ssize_t n = PyList_Size(object_id_list); - if (!plasma_manager_is_connected(conn)) { + if (!plasma_manager_is_connected(client)) { PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager"); return NULL; } @@ -254,19 +251,19 @@ PyObject *PyPlasma_wait(PyObject *self, PyObject *args) { ObjectRequest *object_requests = (ObjectRequest *) malloc(sizeof(ObjectRequest) * n); for (int i = 0; i < n; ++i) { - CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i), - &object_requests[i].object_id) == 1); + ARROW_CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i), + &object_requests[i].object_id) == 1); object_requests[i].type = PLASMA_QUERY_ANYWHERE; } /* Drop the global interpreter lock while we are waiting, so other threads can * run. */ int num_return_objects; Py_BEGIN_ALLOW_THREADS; - num_return_objects = plasma_wait(conn, (int) n, object_requests, num_returns, - (uint64_t) timeout); + ARROW_CHECK_OK(client->Wait((int) n, object_requests, num_returns, + (uint64_t) timeout, num_return_objects)); Py_END_ALLOW_THREADS; - int num_to_return = MIN(num_return_objects, num_returns); + int num_to_return = std::min(num_return_objects, num_returns); PyObject *ready_ids = PyList_New(num_to_return); PyObject *waiting_ids = PySet_New(object_id_list); int num_returned = 0; @@ -277,16 +274,16 @@ PyObject *PyPlasma_wait(PyObject *self, PyObject *args) { if (object_requests[i].status == ObjectStatus_Local || object_requests[i].status == ObjectStatus_Remote) { PyObject *ready = - PyBytes_FromStringAndSize((char *) object_requests[i].object_id.id, + PyBytes_FromStringAndSize((char *) &object_requests[i].object_id, sizeof(object_requests[i].object_id)); PyList_SetItem(ready_ids, num_returned, ready); PySet_Discard(waiting_ids, ready); num_returned += 1; } else { - CHECK(object_requests[i].status == ObjectStatus_Nonexistent); + ARROW_CHECK(object_requests[i].status == ObjectStatus_Nonexistent); } } - CHECK(num_returned == num_to_return); + ARROW_CHECK(num_returned == num_to_return); /* Return both the ready IDs and the remaining IDs. */ PyObject *t = PyTuple_New(2); PyTuple_SetItem(t, 0, ready_ids); @@ -295,53 +292,55 @@ PyObject *PyPlasma_wait(PyObject *self, PyObject *args) { } PyObject *PyPlasma_evict(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; long long num_bytes; - if (!PyArg_ParseTuple(args, "O&L", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&L", PyObjectToPlasmaClient, &client, &num_bytes)) { return NULL; } - int64_t evicted_bytes = plasma_evict(conn, (int64_t) num_bytes); + int64_t evicted_bytes; + ARROW_CHECK_OK(client->Evict((int64_t) num_bytes, evicted_bytes)); return PyLong_FromLong((long) evicted_bytes); } PyObject *PyPlasma_delete(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, PyStringToUniqueID, &object_id)) { return NULL; } - plasma_delete(conn, object_id); + ARROW_CHECK_OK(client->Delete(object_id)); Py_RETURN_NONE; } PyObject *PyPlasma_transfer(PyObject *self, PyObject *args) { - PlasmaConnection *conn; + PlasmaClient *client; ObjectID object_id; const char *addr; int port; - if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaConnection, &conn, + if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaClient, &client, PyStringToUniqueID, &object_id, &addr, &port)) { return NULL; } - if (!plasma_manager_is_connected(conn)) { + if (!plasma_manager_is_connected(client)) { PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager"); return NULL; } - plasma_transfer(conn, addr, port, object_id); + ARROW_CHECK_OK(client->Transfer(addr, port, object_id)); Py_RETURN_NONE; } PyObject *PyPlasma_subscribe(PyObject *self, PyObject *args) { - PlasmaConnection *conn; - if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaConnection, &conn)) { + PlasmaClient *client; + if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaClient, &client)) { return NULL; } - int sock = plasma_subscribe(conn); + int sock; + ARROW_CHECK_OK(client->Subscribe(sock)); return PyLong_FromLong(sock); } @@ -355,7 +354,7 @@ PyObject *PyPlasma_receive_notification(PyObject *self, PyObject *args) { * object was added, return a tuple of its fields: ObjectID, data_size, * metadata_size. If the object was deleted, data_size and metadata_size will * be set to -1. */ - uint8_t *notification = read_message_async(NULL, plasma_sock); + uint8_t *notification = read_message_async(plasma_sock); if (notification == NULL) { PyErr_SetString(PyExc_RuntimeError, "Failed to read object notification from Plasma socket"); diff --git a/src/plasma/plasma_extension.h b/src/plasma/plasma_extension.h index 6be357dc5..6c7bf5954 100644 --- a/src/plasma/plasma_extension.h +++ b/src/plasma/plasma_extension.h @@ -1,10 +1,9 @@ #ifndef PLASMA_EXTENSION_H #define PLASMA_EXTENSION_H -static int PyObjectToPlasmaConnection(PyObject *object, - PlasmaConnection **conn) { +static int PyObjectToPlasmaClient(PyObject *object, PlasmaClient **client) { if (PyCapsule_IsValid(object, "plasma")) { - *conn = (PlasmaConnection *) PyCapsule_GetPointer(object, "plasma"); + *client = (PlasmaClient *) PyCapsule_GetPointer(object, "plasma"); return 1; } else { PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule"); @@ -12,4 +11,14 @@ static int PyObjectToPlasmaConnection(PyObject *object, } } +int PyStringToUniqueID(PyObject *object, ObjectID *object_id) { + if (PyBytes_Check(object)) { + memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID)); + return 1; + } else { + PyErr_SetString(PyExc_TypeError, "must be a 20 character string"); + return 0; + } +} + #endif /* PLASMA_EXTENSION_H */ diff --git a/src/plasma/plasma_io.cc b/src/plasma/plasma_io.cc new file mode 100644 index 000000000..7bad98518 --- /dev/null +++ b/src/plasma/plasma_io.cc @@ -0,0 +1,220 @@ +#include "plasma_io.h" +#include "plasma_common.h" + +using arrow::Status; + +/* Number of times we try binding to a socket. */ +#define NUM_BIND_ATTEMPTS 5 +#define BIND_TIMEOUT_MS 100 + +/* Number of times we try connecting to a socket. */ +#define NUM_CONNECT_ATTEMPTS 50 +#define CONNECT_TIMEOUT_MS 100 + +Status WriteBytes(int fd, uint8_t *cursor, size_t length) { + ssize_t nbytes = 0; + size_t bytesleft = length; + size_t offset = 0; + while (bytesleft > 0) { + /* While we haven't written the whole message, write to the file descriptor, + * advance the cursor, and decrease the amount left to write. */ + nbytes = write(fd, cursor + offset, bytesleft); + if (nbytes < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { + continue; + } + return Status::IOError(std::string(strerror(errno))); + } else if (nbytes == 0) { + return Status::IOError("Encountered unexpected EOF"); + } + ARROW_CHECK(nbytes > 0); + bytesleft -= nbytes; + offset += nbytes; + } + + return Status::OK(); +} + +Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t *bytes) { + int64_t version = PLASMA_PROTOCOL_VERSION; + RETURN_NOT_OK( + WriteBytes(fd, reinterpret_cast(&version), sizeof(version))); + RETURN_NOT_OK( + WriteBytes(fd, reinterpret_cast(&type), sizeof(type))); + RETURN_NOT_OK( + WriteBytes(fd, reinterpret_cast(&length), sizeof(length))); + return WriteBytes(fd, bytes, length * sizeof(char)); +} + +Status ReadBytes(int fd, uint8_t *cursor, size_t length) { + ssize_t nbytes = 0; + /* Termination condition: EOF or read 'length' bytes total. */ + size_t bytesleft = length; + size_t offset = 0; + while (bytesleft > 0) { + nbytes = read(fd, cursor + offset, bytesleft); + if (nbytes < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { + continue; + } + return Status::IOError(std::string(strerror(errno))); + } else if (0 == nbytes) { + return Status::IOError("Encountered unexpected EOF"); + } + ARROW_CHECK(nbytes > 0); + bytesleft -= nbytes; + offset += nbytes; + } + + return Status::OK(); +} + +Status ReadMessage(int fd, int64_t *type, std::vector &buffer) { + int64_t version; + RETURN_NOT_OK_ELSE( + ReadBytes(fd, reinterpret_cast(&version), sizeof(version)), + *type = DISCONNECT_CLIENT); + ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version; + int64_t length; + RETURN_NOT_OK_ELSE( + ReadBytes(fd, reinterpret_cast(type), sizeof(*type)), + *type = DISCONNECT_CLIENT); + RETURN_NOT_OK_ELSE( + ReadBytes(fd, reinterpret_cast(&length), sizeof(length)), + *type = DISCONNECT_CLIENT); + if (length > buffer.size()) { + buffer.resize(length); + } + RETURN_NOT_OK_ELSE(ReadBytes(fd, buffer.data(), length), + *type = DISCONNECT_CLIENT); + return Status::OK(); +} + +int bind_ipc_sock(const std::string &pathname, bool shall_listen) { + struct sockaddr_un socket_address; + int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (socket_fd < 0) { + ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname; + return -1; + } + /* Tell the system to allow the port to be reused. */ + int on = 1; + if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, + sizeof(on)) < 0) { + ARROW_LOG(ERROR) << "setsockopt failed for pathname " << pathname; + close(socket_fd); + return -1; + } + + unlink(pathname.c_str()); + memset(&socket_address, 0, sizeof(socket_address)); + socket_address.sun_family = AF_UNIX; + if (pathname.size() + 1 > sizeof(socket_address.sun_path)) { + ARROW_LOG(ERROR) << "Socket pathname is too long."; + close(socket_fd); + return -1; + } + strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1); + + if (bind(socket_fd, (struct sockaddr *) &socket_address, + sizeof(socket_address)) != 0) { + ARROW_LOG(ERROR) << "Bind failed for pathname " << pathname; + close(socket_fd); + return -1; + } + if (shall_listen && listen(socket_fd, 5) == -1) { + ARROW_LOG(ERROR) << "Could not listen to socket " << pathname; + close(socket_fd); + return -1; + } + return socket_fd; +} + +int connect_ipc_sock_retry(const std::string &pathname, + int num_retries, + int64_t timeout) { + /* Pick the default values if the user did not specify. */ + if (num_retries < 0) { + num_retries = NUM_CONNECT_ATTEMPTS; + } + if (timeout < 0) { + timeout = CONNECT_TIMEOUT_MS; + } + + int fd = -1; + for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) { + fd = connect_ipc_sock(pathname); + if (fd >= 0) { + break; + } + if (num_attempts == 0) { + ARROW_LOG(ERROR) << "Connection to socket failed for pathname " + << pathname; + } + /* Sleep for timeout milliseconds. */ + usleep(timeout * 1000); + } + /* If we could not connect to the socket, exit. */ + if (fd == -1) { + ARROW_LOG(FATAL) << "Could not connect to socket " << pathname; + } + return fd; +} + +int connect_ipc_sock(const std::string &pathname) { + struct sockaddr_un socket_address; + int socket_fd; + + socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (socket_fd < 0) { + ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname; + return -1; + } + + memset(&socket_address, 0, sizeof(socket_address)); + socket_address.sun_family = AF_UNIX; + if (pathname.size() + 1 > sizeof(socket_address.sun_path)) { + ARROW_LOG(ERROR) << "Socket pathname is too long."; + return -1; + } + strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1); + + if (connect(socket_fd, (struct sockaddr *) &socket_address, + sizeof(socket_address)) != 0) { + close(socket_fd); + return -1; + } + + return socket_fd; +} + +int AcceptClient(int socket_fd) { + int client_fd = accept(socket_fd, NULL, NULL); + if (client_fd < 0) { + ARROW_LOG(ERROR) << "Error reading from socket."; + return -1; + } + return client_fd; +} + +uint8_t *read_message_async(int sock) { + int64_t size; + Status s = ReadBytes(sock, (uint8_t *) &size, sizeof(int64_t)); + if (!s.ok()) { + /* The other side has closed the socket. */ + ARROW_LOG(DEBUG) + << "Socket has been closed, or some other error has occurred."; + close(sock); + return NULL; + } + uint8_t *message = (uint8_t *) malloc(size); + s = ReadBytes(sock, message, size); + if (!s.ok()) { + /* The other side has closed the socket. */ + ARROW_LOG(DEBUG) + << "Socket has been closed, or some other error has occurred."; + close(sock); + return NULL; + } + return message; +} diff --git a/src/plasma/plasma_io.h b/src/plasma/plasma_io.h new file mode 100644 index 000000000..512763394 --- /dev/null +++ b/src/plasma/plasma_io.h @@ -0,0 +1,38 @@ +#include +#include +#include +#include + +#include +#include + +#include "status.h" + +// TODO(pcm): Replace our own custom message header (message type, +// message length, plasma protocol verion) with one that is serialized +// using flatbuffers. +#define PLASMA_PROTOCOL_VERSION 0x0000000000000000 +#define DISCONNECT_CLIENT 0 + +arrow::Status WriteBytes(int fd, uint8_t *cursor, size_t length); + +arrow::Status WriteMessage(int fd, + int64_t type, + int64_t length, + uint8_t *bytes); + +arrow::Status ReadBytes(int fd, uint8_t *cursor, size_t length); + +arrow::Status ReadMessage(int fd, int64_t *type, std::vector &buffer); + +int bind_ipc_sock(const std::string &pathname, bool shall_listen); + +int connect_ipc_sock(const std::string &pathname); + +int connect_ipc_sock_retry(const std::string &pathname, + int num_retries, + int64_t timeout); + +int AcceptClient(int socket_fd); + +uint8_t *read_message_async(int sock); diff --git a/src/plasma/plasma_manager.cc b/src/plasma/plasma_manager.cc index 31b3d9662..7412c589c 100644 --- a/src/plasma/plasma_manager.cc +++ b/src/plasma/plasma_manager.cc @@ -43,6 +43,21 @@ #include "state/task_table.h" #include "state/db_client_table.h" +int handle_sigpipe(Status s, int fd) { + if (s.ok()) { + return 0; + } + if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) { + ARROW_LOG(WARNING) + << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " + "sending a message to client on fd " + << fd << ". The client on the other end may " + "have hung up."; + return errno; + } + ARROW_LOG(FATAL) << "Failed to write message to client on fd " << fd << "."; +} + /** * Process either the fetch or the status request. * @@ -200,7 +215,7 @@ struct PlasmaManagerState { /** Event loop. */ event_loop *loop; /** Connection to the local plasma store for reading or writing data. */ - PlasmaConnection *plasma_conn; + PlasmaClient *plasma_conn; /** Hash table of all contexts for active connections to * other plasma managers. These are used for writing data to * other plasma stores. */ @@ -221,8 +236,6 @@ struct PlasmaManagerState { ObjectWaitRequests *object_wait_requests_remote; /** Initialize an empty hash map for the cache of local available object. */ AvailableObject *local_available_objects; - /** Buffer that holds memory for serializing plasma protocol messages. */ - protocol_builder *builder; }; PlasmaManagerState *g_manager_state = NULL; @@ -400,10 +413,10 @@ void remove_wait_request(PlasmaManagerState *manager_state, void return_from_wait(PlasmaManagerState *manager_state, WaitRequest *wait_req) { /* Send the reply to the client. */ - warn_if_sigpipe(plasma_send_WaitReply( - wait_req->client_conn->fd, manager_state->builder, - wait_req->object_requests, wait_req->num_object_requests), - wait_req->client_conn->fd); + handle_sigpipe( + SendWaitReply(wait_req->client_conn->fd, wait_req->object_requests, + wait_req->num_object_requests), + wait_req->client_conn->fd); /* Iterate over all object IDs requested as part of this wait request. * Remove the wait request from each of the relevant object_wait_requests hash * tables if it is present there. */ @@ -498,8 +511,9 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name, PlasmaManagerState *state = (PlasmaManagerState *) malloc(sizeof(PlasmaManagerState)); state->loop = event_loop_create(); - state->plasma_conn = - plasma_connect(store_socket_name, NULL, PLASMA_DEFAULT_RELEASE_DELAY); + state->plasma_conn = new PlasmaClient(); + ARROW_CHECK_OK(state->plasma_conn->Connect(store_socket_name, "", + PLASMA_DEFAULT_RELEASE_DELAY)); state->manager_connections = NULL; state->fetch_requests = NULL; state->object_wait_requests_local = NULL; @@ -534,11 +548,11 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name, /* Initialize an empty hash map for the cache of local available objects. */ state->local_available_objects = NULL; /* Subscribe to notifications about sealed objects. */ - int plasma_fd = plasma_subscribe(state->plasma_conn); + int plasma_fd; + ARROW_CHECK_OK(state->plasma_conn->Subscribe(plasma_fd)); /* Add the callback that processes the notification to the event loop. */ event_loop_add_file(state->loop, plasma_fd, EVENT_LOOP_READ, process_object_notification, state); - state->builder = make_protocol_builder(); return state; } @@ -582,9 +596,9 @@ void PlasmaManagerState_free(PlasmaManagerState *state) { free(wait_reqs); } - plasma_disconnect(state->plasma_conn); + ARROW_CHECK_OK(state->plasma_conn->Disconnect()); + delete state->plasma_conn; event_loop_destroy(state->loop); - free_protocol_builder(state->builder); free(state); } @@ -624,7 +638,7 @@ int write_object_chunk(ClientConnection *conn, PlasmaRequestBuffer *buf) { conn->cursor = 0; /* We are done sending the object, so release it. The corresponding call to * plasma_get occurred in process_transfer_request. */ - plasma_release(conn->manager_state->plasma_conn, buf->object_id); + ARROW_CHECK_OK(conn->manager_state->plasma_conn->Release(buf->object_id)); } return 0; @@ -649,9 +663,8 @@ void send_queued_request(event_loop *loop, int err = 0; switch (buf->type) { case MessageType_PlasmaDataRequest: - err = warn_if_sigpipe( - plasma_send_DataRequest(conn->fd, state->builder, buf->object_id, - state->addr, state->port), + err = handle_sigpipe( + SendDataRequest(conn->fd, buf->object_id, state->addr, state->port), conn->fd); break; case MessageType_PlasmaDataReply: @@ -659,10 +672,9 @@ void send_queued_request(event_loop *loop, if (conn->cursor == 0) { /* If the cursor is zero, we haven't sent any requests for this object * yet, so send the initial data request. */ - err = warn_if_sigpipe( - plasma_send_DataReply(conn->fd, state->builder, buf->object_id, - buf->data_size, buf->metadata_size), - conn->fd); + err = handle_sigpipe(SendDataReply(conn->fd, buf->object_id, + buf->data_size, buf->metadata_size), + conn->fd); } if (err == 0) { err = write_object_chunk(conn, buf); @@ -743,8 +755,8 @@ void process_data_chunk(event_loop *loop, LOG_DEBUG("reading on channel %d finished", data_sock); /* The following seal also triggers notification of clients for fetch or * wait requests, see process_object_notification. */ - plasma_seal(conn->manager_state->plasma_conn, buf->object_id); - plasma_release(conn->manager_state->plasma_conn, buf->object_id); + ARROW_CHECK_OK(conn->manager_state->plasma_conn->Seal(buf->object_id)); + ARROW_CHECK_OK(conn->manager_state->plasma_conn->Release(buf->object_id)); /* Remove the request buffer used for reading this object's data. */ DL_DELETE(conn->transfer_queue, buf); free(buf); @@ -824,7 +836,8 @@ void process_transfer_request(event_loop *loop, /* Allocate and append the request to the transfer queue. */ ObjectBuffer object_buffer; /* We pass in 0 to indicate that the command should return immediately. */ - plasma_get(conn->manager_state->plasma_conn, &obj_id, 1, 0, &object_buffer); + ARROW_CHECK_OK( + conn->manager_state->plasma_conn->Get(&obj_id, 1, 0, &object_buffer)); if (object_buffer.data_size == -1) { /* If the object wasn't locally available, exit immediately. If the object * later appears locally, the requesting plasma manager should request the @@ -890,12 +903,12 @@ void process_data_request(event_loop *loop, /* The corresponding call to plasma_release should happen in * process_data_chunk. */ - int error_code = plasma_create(conn->manager_state->plasma_conn, object_id, - data_size, NULL, metadata_size, &(buf->data)); + Status s = conn->manager_state->plasma_conn->Create( + object_id, data_size, NULL, metadata_size, &(buf->data)); /* If success_create == true, a new object has been created. * If success_create == false the object creation has failed, possibly * due to an object with the same ID already existing in the Plasma Store. */ - if (error_code == PlasmaError_OK) { + if (s.ok()) { /* Add buffer where the fetched data is to be stored to * conn->transfer_queue. */ DL_APPEND(conn->transfer_queue, buf); @@ -905,7 +918,7 @@ void process_data_request(event_loop *loop, /* Switch to reading the data from this socket, instead of listening for * other requests. */ event_loop_remove_file(loop, client_sock); - if (error_code == PlasmaError_OK) { + if (s.ok()) { bool success = event_loop_add_file(loop, client_sock, EVENT_LOOP_READ, process_data_chunk, conn); if (!success) { @@ -1175,12 +1188,12 @@ int wait_timeout_handler(event_loop *loop, timer_id id, void *context) { } void process_wait_request(ClientConnection *client_conn, - int num_object_requests, ObjectRequestMap &&object_requests, uint64_t timeout_ms, int num_ready_objects) { CHECK(client_conn != NULL); PlasmaManagerState *manager_state = client_conn->manager_state; + int num_object_requests = object_requests.size(); /* Create a wait request for this object. */ WaitRequest *wait_req = @@ -1267,10 +1280,8 @@ void request_status_done(ObjectID object_id, ClientConnection *client_conn = (ClientConnection *) context; int status = request_status(object_id, manager_count, manager_vector, context); - warn_if_sigpipe(plasma_send_StatusReply(client_conn->fd, - client_conn->manager_state->builder, - &object_id, &status, 1), - client_conn->fd); + handle_sigpipe(SendStatusReply(client_conn->fd, &object_id, &status, 1), + client_conn->fd); } int request_status(ObjectID object_id, @@ -1302,19 +1313,15 @@ void process_status_request(ClientConnection *client_conn, ObjectID object_id) { /* Return success immediately if we already have this object. */ if (is_object_local(client_conn->manager_state, object_id)) { int status = ObjectStatus_Local; - warn_if_sigpipe(plasma_send_StatusReply(client_conn->fd, - client_conn->manager_state->builder, - &object_id, &status, 1), - client_conn->fd); + handle_sigpipe(SendStatusReply(client_conn->fd, &object_id, &status, 1), + client_conn->fd); return; } if (client_conn->manager_state->db == NULL) { int status = ObjectStatus_Nonexistent; - warn_if_sigpipe(plasma_send_StatusReply(client_conn->fd, - client_conn->manager_state->builder, - &object_id, &status, 1), - client_conn->fd); + handle_sigpipe(SendStatusReply(client_conn->fd, &object_id, &status, 1), + client_conn->fd); return; } @@ -1531,7 +1538,7 @@ void process_message(event_loop *loop, ObjectID object_id; char *address; int port; - plasma_read_DataRequest(data, &object_id, &address, &port); + ARROW_CHECK_OK(ReadDataRequest(data, &object_id, &address, &port)); process_transfer_request(loop, object_id, address, port, conn); free(address); } break; @@ -1540,38 +1547,34 @@ void process_message(event_loop *loop, ObjectID object_id; int64_t object_size; int64_t metadata_size; - plasma_read_DataReply(data, &object_id, &object_size, &metadata_size); + ARROW_CHECK_OK( + ReadDataReply(data, &object_id, &object_size, &metadata_size)); process_data_request(loop, client_sock, object_id, object_size, metadata_size, conn); } break; case MessageType_PlasmaFetchRequest: { LOG_DEBUG("Processing fetch remote"); - int64_t num_objects = plasma_read_FetchRequest_num_objects(data); - ObjectID *object_ids_to_fetch = - (ObjectID *) malloc(num_objects * sizeof(ObjectID)); + std::vector object_ids_to_fetch; /* TODO(pcm): process_fetch_requests allocates an array of num_objects * object_ids too so these should be shared in the future. */ - plasma_read_FetchRequest(data, object_ids_to_fetch, num_objects); - process_fetch_requests(conn, num_objects, &object_ids_to_fetch[0]); - free(object_ids_to_fetch); + ARROW_CHECK_OK(ReadFetchRequest(data, object_ids_to_fetch)); + process_fetch_requests(conn, object_ids_to_fetch.size(), + object_ids_to_fetch.data()); } break; case MessageType_PlasmaWaitRequest: { LOG_DEBUG("Processing wait"); - int num_object_ids = plasma_read_WaitRequest_num_object_ids(data); ObjectRequestMap object_requests; int64_t timeout_ms; int num_ready_objects; - plasma_read_WaitRequest(data, object_requests, num_object_ids, &timeout_ms, - &num_ready_objects); - process_wait_request(conn, num_object_ids, std::move(object_requests), - timeout_ms, num_ready_objects); + ARROW_CHECK_OK(ReadWaitRequest(data, object_requests, &timeout_ms, + &num_ready_objects)); + process_wait_request(conn, std::move(object_requests), timeout_ms, + num_ready_objects); } break; case MessageType_PlasmaStatusRequest: { LOG_DEBUG("Processing status"); ObjectID object_id; - int64_t num_objects = plasma_read_StatusRequest_num_objects(data); - CHECK(num_objects == 1); - plasma_read_StatusRequest(data, &object_id, 1); + ARROW_CHECK_OK(ReadStatusRequest(data, &object_id, 1)); process_status_request(conn, object_id); } break; case DISCONNECT_CLIENT: { diff --git a/src/plasma/plasma_protocol.cc b/src/plasma/plasma_protocol.cc index 7baabf431..134d09e80 100644 --- a/src/plasma/plasma_protocol.cc +++ b/src/plasma/plasma_protocol.cc @@ -1,406 +1,381 @@ #include "flatbuffers/flatbuffers.h" #include "format/plasma_generated.h" +#include "plasma_common.h" #include "plasma_protocol.h" -#include "common_protocol.h" -#include "io.h" +#include "plasma_io.h" -#define FLATBUFFER_BUILDER_DEFAULT_SIZE 1024 - -protocol_builder *make_protocol_builder(void) { - return NULL; +flatbuffers::Offset< + flatbuffers::Vector>> +to_flatbuffer(flatbuffers::FlatBufferBuilder &fbb, + ObjectID object_ids[], + int64_t num_objects) { + std::vector> results; + for (size_t i = 0; i < num_objects; i++) { + results.push_back(fbb.CreateString(object_ids[i].binary())); + } + return fbb.CreateVector(results); } -void free_protocol_builder(protocol_builder *builder) {} - -uint8_t *plasma_receive(int sock, int64_t message_type) { +Status PlasmaReceive(int sock, + int64_t message_type, + std::vector &buffer) { int64_t type; - int64_t length; - uint8_t *reply_data; - read_message(sock, &type, &length, &reply_data); - CHECKM(type == message_type, "type = %" PRId64 ", message_type = %" PRId64, - type, message_type); - return reply_data; + RETURN_NOT_OK(ReadMessage(sock, &type, buffer)); + ARROW_CHECK(type == message_type) << "type = " << type + << ", message_type = " << message_type; + return Status::OK(); } /* Create messages. */ -int plasma_send_CreateRequest(int sock, - protocol_builder *B, - ObjectID object_id, - int64_t data_size, - int64_t metadata_size) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = CreatePlasmaCreateRequest(fbb, to_flatbuf(fbb, object_id), - data_size, metadata_size); +Status SendCreateRequest(int sock, + ObjectID object_id, + int64_t data_size, + int64_t metadata_size) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaCreateRequest( + fbb, fbb.CreateString(object_id.binary()), data_size, metadata_size); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaCreateRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaCreateRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_CreateRequest(uint8_t *data, - ObjectID *object_id, - int64_t *data_size, - int64_t *metadata_size) { - CHECK(data); +Status ReadCreateRequest(uint8_t *data, + ObjectID *object_id, + int64_t *data_size, + int64_t *metadata_size) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); *data_size = message->data_size(); *metadata_size = message->metadata_size(); - *object_id = from_flatbuf(message->object_id()); + *object_id = ObjectID::from_binary(message->object_id()->str()); + return Status::OK(); } -int plasma_send_CreateReply(int sock, - protocol_builder *B, - ObjectID object_id, - PlasmaObject *object, - int error_code) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendCreateReply(int sock, + ObjectID object_id, + PlasmaObject *object, + int error_code) { + flatbuffers::FlatBufferBuilder fbb; PlasmaObjectSpec plasma_object( object->handle.store_fd, object->handle.mmap_size, object->data_offset, object->data_size, object->metadata_offset, object->metadata_size); auto message = - CreatePlasmaCreateReply(fbb, to_flatbuf(fbb, object_id), &plasma_object, - (PlasmaError) error_code); + CreatePlasmaCreateReply(fbb, fbb.CreateString(object_id.binary()), + &plasma_object, (PlasmaError) error_code); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaCreateReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaCreateReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_CreateReply(uint8_t *data, - ObjectID *object_id, - PlasmaObject *object, - int *error_code) { - CHECK(data); +Status ReadCreateReply(uint8_t *data, + ObjectID *object_id, + PlasmaObject *object) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); + *object_id = ObjectID::from_binary(message->object_id()->str()); object->handle.store_fd = message->plasma_object()->segment_index(); object->handle.mmap_size = message->plasma_object()->mmap_size(); object->data_offset = message->plasma_object()->data_offset(); object->data_size = message->plasma_object()->data_size(); object->metadata_offset = message->plasma_object()->metadata_offset(); object->metadata_size = message->plasma_object()->metadata_size(); - *error_code = message->error(); + return plasma_error_status(message->error()); } /* Seal messages. */ -int plasma_send_SealRequest(int sock, - protocol_builder *B, - ObjectID object_id, - unsigned char *digest) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto digest_string = fbb.CreateString((char *) digest, DIGEST_SIZE); - auto message = - CreatePlasmaSealRequest(fbb, to_flatbuf(fbb, object_id), digest_string); +Status SendSealRequest(int sock, ObjectID object_id, unsigned char *digest) { + flatbuffers::FlatBufferBuilder fbb; + auto digest_string = fbb.CreateString((char *) digest, kDigestSize); + auto message = CreatePlasmaSealRequest( + fbb, fbb.CreateString(object_id.binary()), digest_string); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaSealRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaSealRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_SealRequest(uint8_t *data, - ObjectID *object_id, - unsigned char *digest) { - CHECK(data); +Status ReadSealRequest(uint8_t *data, + ObjectID *object_id, + unsigned char *digest) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); - CHECK(message->digest()->size() == DIGEST_SIZE); - memcpy(digest, message->digest()->data(), DIGEST_SIZE); + *object_id = ObjectID::from_binary(message->object_id()->str()); + ARROW_CHECK(message->digest()->size() == kDigestSize); + memcpy(digest, message->digest()->data(), kDigestSize); + return Status::OK(); } -int plasma_send_SealReply(int sock, - protocol_builder *B, - ObjectID object_id, - int error) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = CreatePlasmaSealReply(fbb, to_flatbuf(fbb, object_id), - (PlasmaError) error); +Status SendSealReply(int sock, ObjectID object_id, int error) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaSealReply( + fbb, fbb.CreateString(object_id.binary()), (PlasmaError) error); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaSealReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaSealReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_SealReply(uint8_t *data, ObjectID *object_id, int *error) { - CHECK(data); +Status ReadSealReply(uint8_t *data, ObjectID *object_id) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); - *error = message->error(); + *object_id = ObjectID::from_binary(message->object_id()->str()); + return plasma_error_status(message->error()); } /* Release messages. */ -int plasma_send_ReleaseRequest(int sock, - protocol_builder *B, - ObjectID object_id) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = CreatePlasmaSealRequest(fbb, to_flatbuf(fbb, object_id)); +Status SendReleaseRequest(int sock, ObjectID object_id) { + flatbuffers::FlatBufferBuilder fbb; + auto message = + CreatePlasmaSealRequest(fbb, fbb.CreateString(object_id.binary())); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaReleaseRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaReleaseRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_ReleaseRequest(uint8_t *data, ObjectID *object_id) { - CHECK(data); +Status ReadReleaseRequest(uint8_t *data, ObjectID *object_id) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); + *object_id = ObjectID::from_binary(message->object_id()->str()); + return Status::OK(); } -int plasma_send_ReleaseReply(int sock, - protocol_builder *B, - ObjectID object_id, - int error) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = CreatePlasmaReleaseReply(fbb, to_flatbuf(fbb, object_id), - (PlasmaError) error); +Status SendReleaseReply(int sock, ObjectID object_id, int error) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaReleaseReply( + fbb, fbb.CreateString(object_id.binary()), (PlasmaError) error); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaReleaseReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaReleaseReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_ReleaseReply(uint8_t *data, ObjectID *object_id, int *error) { - CHECK(data); +Status ReadReleaseReply(uint8_t *data, ObjectID *object_id) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); - *error = message->error(); + *object_id = ObjectID::from_binary(message->object_id()->str()); + return plasma_error_status(message->error()); } /* Delete messages. */ -int plasma_send_DeleteRequest(int sock, - protocol_builder *B, - ObjectID object_id) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = CreatePlasmaDeleteRequest(fbb, to_flatbuf(fbb, object_id)); +Status SendDeleteRequest(int sock, ObjectID object_id) { + flatbuffers::FlatBufferBuilder fbb; + auto message = + CreatePlasmaDeleteRequest(fbb, fbb.CreateString(object_id.binary())); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaDeleteRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaDeleteRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_DeleteRequest(uint8_t *data, ObjectID *object_id) { - CHECK(data); +Status ReadDeleteRequest(uint8_t *data, ObjectID *object_id) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); + *object_id = ObjectID::from_binary(message->object_id()->str()); + return Status::OK(); } -int plasma_send_DeleteReply(int sock, - protocol_builder *B, - ObjectID object_id, - int error) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = CreatePlasmaDeleteReply(fbb, to_flatbuf(fbb, object_id), - (PlasmaError) error); +Status SendDeleteReply(int sock, ObjectID object_id, int error) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaDeleteReply( + fbb, fbb.CreateString(object_id.binary()), (PlasmaError) error); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaDeleteReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaDeleteReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_DeleteReply(uint8_t *data, ObjectID *object_id, int *error) { - CHECK(data); +Status ReadDeleteReply(uint8_t *data, ObjectID *object_id) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); - *error = message->error(); + *object_id = ObjectID::from_binary(message->object_id()->str()); + return plasma_error_status(message->error()); } /* Satus messages. */ -int plasma_send_StatusRequest(int sock, - protocol_builder *B, - ObjectID object_ids[], - int64_t num_objects) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = - CreatePlasmaStatusRequest(fbb, to_flatbuf(fbb, object_ids, num_objects)); +Status SendStatusRequest(int sock, ObjectID object_ids[], int64_t num_objects) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaStatusRequest( + fbb, to_flatbuffer(fbb, object_ids, num_objects)); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaStatusRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaStatusRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -int64_t plasma_read_StatusRequest_num_objects(uint8_t *data) { - DCHECK(data); - auto message = flatbuffers::GetRoot(data); - return message->object_ids()->size(); -} - -void plasma_read_StatusRequest(uint8_t *data, - ObjectID object_ids[], - int64_t num_objects) { +Status ReadStatusRequest(uint8_t *data, + ObjectID object_ids[], + int64_t num_objects) { DCHECK(data); auto message = flatbuffers::GetRoot(data); for (int64_t i = 0; i < num_objects; ++i) { - object_ids[i] = from_flatbuf(message->object_ids()->Get(i)); + object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str()); } + return Status::OK(); } -int plasma_send_StatusReply(int sock, - protocol_builder *B, - ObjectID object_ids[], - int object_status[], - int64_t num_objects) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendStatusReply(int sock, + ObjectID object_ids[], + int object_status[], + int64_t num_objects) { + flatbuffers::FlatBufferBuilder fbb; auto message = - CreatePlasmaStatusReply(fbb, to_flatbuf(fbb, object_ids, num_objects), + CreatePlasmaStatusReply(fbb, to_flatbuffer(fbb, object_ids, num_objects), fbb.CreateVector(object_status, num_objects)); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaStatusReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaStatusReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -int64_t plasma_read_StatusReply_num_objects(uint8_t *data) { +int64_t ReadStatusReply_num_objects(uint8_t *data) { DCHECK(data); auto message = flatbuffers::GetRoot(data); return message->object_ids()->size(); } -void plasma_read_StatusReply(uint8_t *data, - ObjectID object_ids[], - int object_status[], - int64_t num_objects) { +Status ReadStatusReply(uint8_t *data, + ObjectID object_ids[], + int object_status[], + int64_t num_objects) { DCHECK(data); auto message = flatbuffers::GetRoot(data); for (int64_t i = 0; i < num_objects; ++i) { - object_ids[i] = from_flatbuf(message->object_ids()->Get(i)); + object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str()); } for (int64_t i = 0; i < num_objects; ++i) { object_status[i] = message->status()->data()[i]; } + return Status::OK(); } /* Contains messages. */ -int plasma_send_ContainsRequest(int sock, - protocol_builder *B, - ObjectID object_id) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = CreatePlasmaContainsRequest(fbb, to_flatbuf(fbb, object_id)); - fbb.Finish(message); - return write_message(sock, MessageType_PlasmaContainsRequest, fbb.GetSize(), - fbb.GetBufferPointer()); -} - -void plasma_read_ContainsRequest(uint8_t *data, ObjectID *object_id) { - CHECK(data); - auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); -} - -int plasma_send_ContainsReply(int sock, - protocol_builder *B, - ObjectID object_id, - int has_object) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendContainsRequest(int sock, ObjectID object_id) { + flatbuffers::FlatBufferBuilder fbb; auto message = - CreatePlasmaContainsReply(fbb, to_flatbuf(fbb, object_id), has_object); + CreatePlasmaContainsRequest(fbb, fbb.CreateString(object_id.binary())); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaContainsReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaContainsRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_ContainsReply(uint8_t *data, - ObjectID *object_id, - int *has_object) { - CHECK(data); +Status ReadContainsRequest(uint8_t *data, ObjectID *object_id) { + DCHECK(data); + auto message = flatbuffers::GetRoot(data); + *object_id = ObjectID::from_binary(message->object_id()->str()); + return Status::OK(); +} + +Status SendContainsReply(int sock, ObjectID object_id, int has_object) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaContainsReply( + fbb, fbb.CreateString(object_id.binary()), has_object); + fbb.Finish(message); + return WriteMessage(sock, MessageType_PlasmaContainsReply, fbb.GetSize(), + fbb.GetBufferPointer()); +} + +Status ReadContainsReply(uint8_t *data, ObjectID *object_id, int *has_object) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); + *object_id = ObjectID::from_binary(message->object_id()->str()); *has_object = message->has_object(); + return Status::OK(); } /* Connect messages. */ -int plasma_send_ConnectRequest(int sock, protocol_builder *B) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendConnectRequest(int sock) { + flatbuffers::FlatBufferBuilder fbb; auto message = CreatePlasmaConnectRequest(fbb); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaConnectRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaConnectRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_ConnectRequest(uint8_t *data) {} +Status ReadConnectRequest(uint8_t *data) { + return Status::OK(); +} -int plasma_send_ConnectReply(int sock, - protocol_builder *B, - int64_t memory_capacity) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendConnectReply(int sock, int64_t memory_capacity) { + flatbuffers::FlatBufferBuilder fbb; auto message = CreatePlasmaConnectReply(fbb, memory_capacity); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaConnectReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaConnectReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_ConnectReply(uint8_t *data, int64_t *memory_capacity) { +Status ReadConnectReply(uint8_t *data, int64_t *memory_capacity) { DCHECK(data); auto message = flatbuffers::GetRoot(data); *memory_capacity = message->memory_capacity(); + return Status::OK(); } /* Evict messages. */ -int plasma_send_EvictRequest(int sock, protocol_builder *B, int64_t num_bytes) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendEvictRequest(int sock, int64_t num_bytes) { + flatbuffers::FlatBufferBuilder fbb; auto message = CreatePlasmaEvictRequest(fbb, num_bytes); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaEvictRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaEvictRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_EvictRequest(uint8_t *data, int64_t *num_bytes) { - CHECK(data); +Status ReadEvictRequest(uint8_t *data, int64_t *num_bytes) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); *num_bytes = message->num_bytes(); + return Status::OK(); } -int plasma_send_EvictReply(int sock, protocol_builder *B, int64_t num_bytes) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendEvictReply(int sock, int64_t num_bytes) { + flatbuffers::FlatBufferBuilder fbb; auto message = CreatePlasmaEvictReply(fbb, num_bytes); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaEvictReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaEvictReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_EvictReply(uint8_t *data, int64_t *num_bytes) { +Status ReadEvictReply(uint8_t *data, int64_t &num_bytes) { DCHECK(data); auto message = flatbuffers::GetRoot(data); - *num_bytes = message->num_bytes(); + num_bytes = message->num_bytes(); + return Status::OK(); } /* Get messages. */ -int plasma_send_GetRequest(int sock, - protocol_builder *B, - ObjectID object_ids[], - int64_t num_objects, - int64_t timeout_ms) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendGetRequest(int sock, + ObjectID object_ids[], + int64_t num_objects, + int64_t timeout_ms) { + flatbuffers::FlatBufferBuilder fbb; auto message = CreatePlasmaGetRequest( - fbb, to_flatbuf(fbb, object_ids, num_objects), timeout_ms); + fbb, to_flatbuffer(fbb, object_ids, num_objects), timeout_ms); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaGetRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaGetRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -int64_t plasma_read_GetRequest_num_objects(uint8_t *data) { +Status ReadGetRequest(uint8_t *data, + std::vector &object_ids, + int64_t *timeout_ms) { DCHECK(data); auto message = flatbuffers::GetRoot(data); - return message->object_ids()->size(); -} - -void plasma_read_GetRequest(uint8_t *data, - ObjectID object_ids[], - int64_t *timeout_ms, - int64_t num_objects) { - DCHECK(data); - auto message = flatbuffers::GetRoot(data); - for (int64_t i = 0; i < num_objects; ++i) { - object_ids[i] = from_flatbuf(message->object_ids()->Get(i)); + for (int64_t i = 0; i < message->object_ids()->size(); ++i) { + auto object_id = message->object_ids()->Get(i)->str(); + object_ids.push_back(ObjectID::from_binary(object_id)); } *timeout_ms = message->timeout_ms(); + return Status::OK(); } -int plasma_send_GetReply( +Status SendGetReply( int sock, - protocol_builder *B, ObjectID object_ids[], std::unordered_map &plasma_objects, int64_t num_objects) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); + flatbuffers::FlatBufferBuilder fbb; std::vector objects; for (int i = 0; i < num_objects; ++i) { @@ -410,21 +385,21 @@ int plasma_send_GetReply( object.data_size, object.metadata_offset, object.metadata_size)); } auto message = CreatePlasmaGetReply( - fbb, to_flatbuf(fbb, object_ids, num_objects), + fbb, to_flatbuffer(fbb, object_ids, num_objects), fbb.CreateVectorOfStructs(objects.data(), num_objects)); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaGetReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaGetReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_GetReply(uint8_t *data, - ObjectID object_ids[], - PlasmaObject plasma_objects[], - int64_t num_objects) { - CHECK(data); +Status ReadGetReply(uint8_t *data, + ObjectID object_ids[], + PlasmaObject plasma_objects[], + int64_t num_objects) { + DCHECK(data); auto message = flatbuffers::GetRoot(data); for (int64_t i = 0; i < num_objects; ++i) { - object_ids[i] = from_flatbuf(message->object_ids()->Get(i)); + object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str()); } for (int64_t i = 0; i < num_objects; ++i) { const PlasmaObjectSpec *object = message->plasma_objects()->Get(i); @@ -435,52 +410,43 @@ void plasma_read_GetReply(uint8_t *data, plasma_objects[i].metadata_offset = object->metadata_offset(); plasma_objects[i].metadata_size = object->metadata_size(); } + return Status::OK(); } /* Fetch messages. */ -int plasma_send_FetchRequest(int sock, - protocol_builder *B, - ObjectID object_ids[], - int64_t num_objects) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = - CreatePlasmaFetchRequest(fbb, to_flatbuf(fbb, object_ids, num_objects)); +Status SendFetchRequest(int sock, ObjectID object_ids[], int64_t num_objects) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaFetchRequest( + fbb, to_flatbuffer(fbb, object_ids, num_objects)); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaFetchRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaFetchRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -int64_t plasma_read_FetchRequest_num_objects(uint8_t *data) { +Status ReadFetchRequest(uint8_t *data, std::vector &object_ids) { DCHECK(data); auto message = flatbuffers::GetRoot(data); - return message->object_ids()->size(); -} - -void plasma_read_FetchRequest(uint8_t *data, - ObjectID object_ids[], - int64_t num_objects) { - CHECK(data); - auto message = flatbuffers::GetRoot(data); - for (int64_t i = 0; i < num_objects; ++i) { - object_ids[i] = from_flatbuf(message->object_ids()->Get(i)); + for (int64_t i = 0; i < message->object_ids()->size(); ++i) { + object_ids.push_back( + ObjectID::from_binary(message->object_ids()->Get(i)->str())); } + return Status::OK(); } /* Wait messages. */ -int plasma_send_WaitRequest(int sock, - protocol_builder *B, - ObjectRequest object_requests[], - int num_requests, - int num_ready_objects, - int64_t timeout_ms) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendWaitRequest(int sock, + ObjectRequest object_requests[], + int num_requests, + int num_ready_objects, + int64_t timeout_ms) { + flatbuffers::FlatBufferBuilder fbb; std::vector> object_request_specs; for (int i = 0; i < num_requests; i++) { object_request_specs.push_back(CreateObjectRequestSpec( - fbb, to_flatbuf(fbb, object_requests[i].object_id), + fbb, fbb.CreateString(object_requests[i].object_id.binary()), object_requests[i].type)); } @@ -488,130 +454,124 @@ int plasma_send_WaitRequest(int sock, CreatePlasmaWaitRequest(fbb, fbb.CreateVector(object_request_specs), num_ready_objects, timeout_ms); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaWaitRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaWaitRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -int plasma_read_WaitRequest_num_object_ids(uint8_t *data) { - DCHECK(data); - auto message = flatbuffers::GetRoot(data); - return message->object_requests()->size(); -} - -void plasma_read_WaitRequest(uint8_t *data, - ObjectRequestMap &object_requests, - int num_object_ids, - int64_t *timeout_ms, - int *num_ready_objects) { +Status ReadWaitRequest(uint8_t *data, + ObjectRequestMap &object_requests, + int64_t *timeout_ms, + int *num_ready_objects) { DCHECK(data); auto message = flatbuffers::GetRoot(data); *num_ready_objects = message->num_ready_objects(); *timeout_ms = message->timeout(); - CHECK(num_object_ids == message->object_requests()->size()); - for (int i = 0; i < num_object_ids; i++) { - ObjectID object_id = - from_flatbuf(message->object_requests()->Get(i)->object_id()); + for (int i = 0; i < message->object_requests()->size(); i++) { + ObjectID object_id = ObjectID::from_binary( + message->object_requests()->Get(i)->object_id()->str()); ObjectRequest object_request({object_id, message->object_requests()->Get(i)->type(), ObjectStatus_Nonexistent}); object_requests[object_id] = object_request; } + return Status::OK(); } -int plasma_send_WaitReply(int sock, - protocol_builder *B, - const ObjectRequestMap &object_requests, - int num_ready_objects) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendWaitReply(int sock, + const ObjectRequestMap &object_requests, + int num_ready_objects) { + flatbuffers::FlatBufferBuilder fbb; std::vector> object_replies; for (const auto &entry : object_requests) { const auto &object_request = entry.second; object_replies.push_back(CreateObjectReply( - fbb, to_flatbuf(fbb, object_request.object_id), object_request.status)); + fbb, fbb.CreateString(object_request.object_id.binary()), + object_request.status)); } auto message = CreatePlasmaWaitReply( fbb, fbb.CreateVector(object_replies.data(), num_ready_objects), num_ready_objects); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaWaitReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaWaitReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_WaitReply(uint8_t *data, - ObjectRequest object_requests[], - int *num_ready_objects) { +Status ReadWaitReply(uint8_t *data, + ObjectRequest object_requests[], + int *num_ready_objects) { DCHECK(data); auto message = flatbuffers::GetRoot(data); *num_ready_objects = message->num_ready_objects(); for (int i = 0; i < *num_ready_objects; i++) { - object_requests[i].object_id = - from_flatbuf(message->object_requests()->Get(i)->object_id()); + object_requests[i].object_id = ObjectID::from_binary( + message->object_requests()->Get(i)->object_id()->str()); object_requests[i].status = message->object_requests()->Get(i)->status(); } + return Status::OK(); } /* Subscribe messages. */ -int plasma_send_SubscribeRequest(int sock, protocol_builder *B) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendSubscribeRequest(int sock) { + flatbuffers::FlatBufferBuilder fbb; auto message = CreatePlasmaSubscribeRequest(fbb); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaSubscribeRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaSubscribeRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } /* Data messages. */ -int plasma_send_DataRequest(int sock, - protocol_builder *B, - ObjectID object_id, - const char *address, - int port) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); +Status SendDataRequest(int sock, + ObjectID object_id, + const char *address, + int port) { + flatbuffers::FlatBufferBuilder fbb; auto addr = fbb.CreateString((char *) address, strlen(address)); - auto message = - CreatePlasmaDataRequest(fbb, to_flatbuf(fbb, object_id), addr, port); + auto message = CreatePlasmaDataRequest( + fbb, fbb.CreateString(object_id.binary()), addr, port); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaDataRequest, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaDataRequest, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_DataRequest(uint8_t *data, - ObjectID *object_id, - char **address, - int *port) { +Status ReadDataRequest(uint8_t *data, + ObjectID *object_id, + char **address, + int *port) { DCHECK(data); auto message = flatbuffers::GetRoot(data); - DCHECK(message->object_id()->size() == sizeof(object_id->id)); - *object_id = from_flatbuf(message->object_id()); + DCHECK(message->object_id()->size() == sizeof(ObjectID)); + *object_id = ObjectID::from_binary(message->object_id()->str()); *address = strdup(message->address()->c_str()); *port = message->port(); + return Status::OK(); } -int plasma_send_DataReply(int sock, - protocol_builder *B, - ObjectID object_id, - int64_t object_size, - int64_t metadata_size) { - flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE); - auto message = CreatePlasmaDataReply(fbb, to_flatbuf(fbb, object_id), - object_size, metadata_size); +Status SendDataReply(int sock, + ObjectID object_id, + int64_t object_size, + int64_t metadata_size) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaDataReply( + fbb, fbb.CreateString(object_id.binary()), object_size, metadata_size); fbb.Finish(message); - return write_message(sock, MessageType_PlasmaDataReply, fbb.GetSize(), - fbb.GetBufferPointer()); + return WriteMessage(sock, MessageType_PlasmaDataReply, fbb.GetSize(), + fbb.GetBufferPointer()); } -void plasma_read_DataReply(uint8_t *data, - ObjectID *object_id, - int64_t *object_size, - int64_t *metadata_size) { +Status ReadDataReply(uint8_t *data, + ObjectID *object_id, + int64_t *object_size, + int64_t *metadata_size) { DCHECK(data); auto message = flatbuffers::GetRoot(data); - *object_id = from_flatbuf(message->object_id()); + *object_id = ObjectID::from_binary(message->object_id()->str()); *object_size = (int64_t) message->object_size(); *metadata_size = (int64_t) message->metadata_size(); + return Status::OK(); } diff --git a/src/plasma/plasma_protocol.h b/src/plasma/plasma_protocol.h index 9ea450fe1..3d525282b 100644 --- a/src/plasma/plasma_protocol.h +++ b/src/plasma/plasma_protocol.h @@ -1,253 +1,194 @@ #ifndef PLASMA_PROTOCOL_H #define PLASMA_PROTOCOL_H +#include "status.h" #include "format/plasma_generated.h" - -#include "common.h" #include "plasma.h" -typedef void protocol_builder; - -/* An argument to a function that a return value gets written to. */ -#define OUT - -protocol_builder *make_protocol_builder(void); - -void free_protocol_builder(protocol_builder *builder); +using arrow::Status; /* Plasma receive message. */ -uint8_t *plasma_receive(int sock, int64_t message_type); +Status PlasmaReceive(int sock, + int64_t message_type, + std::vector &buffer); /* Plasma Create message functions. */ -int plasma_send_CreateRequest(int sock, - protocol_builder *B, - ObjectID object_id, - int64_t data_size, - int64_t metadata_size); +Status SendCreateRequest(int sock, + ObjectID object_id, + int64_t data_size, + int64_t metadata_size); -void plasma_read_CreateRequest(uint8_t *data, - ObjectID *object_id, - int64_t *data_size, - int64_t *metadata_size); +Status ReadCreateRequest(uint8_t *data, + ObjectID *object_id, + int64_t *data_size, + int64_t *metadata_size); -int plasma_send_CreateReply(int sock, - protocol_builder *B, - ObjectID object_id, - PlasmaObject *object, - int error); +Status SendCreateReply(int sock, + ObjectID object_id, + PlasmaObject *object, + int error); -void plasma_read_CreateReply(uint8_t *data, - ObjectID *object_id, - PlasmaObject *object, - int *error); +Status ReadCreateReply(uint8_t *data, + ObjectID *object_id, + PlasmaObject *object); /* Plasma Seal message functions. */ -int plasma_send_SealRequest(int sock, - protocol_builder *B, - ObjectID object_id, - unsigned char *digest); +Status SendSealRequest(int sock, ObjectID object_id, unsigned char *digest); -void plasma_read_SealRequest(uint8_t *data, - OUT ObjectID *object_id, - OUT unsigned char *digest); +Status ReadSealRequest(uint8_t *data, + ObjectID *object_id, + unsigned char *digest); -int plasma_send_SealReply(int sock, - protocol_builder *B, - ObjectID object_id, - int error); +Status SendSealReply(int sock, ObjectID object_id, int error); -void plasma_read_SealReply(uint8_t *data, ObjectID *object_id, int *error); +Status ReadSealReply(uint8_t *data, ObjectID *object_id); /* Plasma Get message functions. */ -int plasma_send_GetRequest(int sock, - protocol_builder *B, - ObjectID object_ids[], - int64_t num_objects, - int64_t timeout_ms); +Status SendGetRequest(int sock, + ObjectID object_ids[], + int64_t num_objects, + int64_t timeout_ms); -int64_t plasma_read_GetRequest_num_objects(uint8_t *data); +Status ReadGetRequest(uint8_t *data, + std::vector &object_ids, + int64_t *timeout_ms); -void plasma_read_GetRequest(uint8_t *data, - ObjectID object_ids[], - int64_t *timeout_ms, - int64_t num_objects); - -int plasma_send_GetReply( +Status SendGetReply( int sock, - protocol_builder *B, ObjectID object_ids[], std::unordered_map &plasma_objects, int64_t num_objects); -void plasma_read_GetReply(uint8_t *data, - ObjectID object_ids[], - PlasmaObject plasma_objects[], - int64_t num_objects); +Status ReadGetReply(uint8_t *data, + ObjectID object_ids[], + PlasmaObject plasma_objects[], + int64_t num_objects); /* Plasma Release message functions. */ -int plasma_send_ReleaseRequest(int sock, - protocol_builder *B, - ObjectID object_id); +Status SendReleaseRequest(int sock, ObjectID object_id); -void plasma_read_ReleaseRequest(uint8_t *data, ObjectID *object_id); +Status ReadReleaseRequest(uint8_t *data, ObjectID *object_id); -int plasma_send_ReleaseReply(int sock, - protocol_builder *B, - ObjectID object_id, - int error); +Status SendReleaseReply(int sock, ObjectID object_id, int error); -void plasma_read_ReleaseReply(uint8_t *data, ObjectID *object_id, int *error); +Status ReadReleaseReply(uint8_t *data, ObjectID *object_id); /* Plasma Delete message functions. */ -int plasma_send_DeleteRequest(int sock, - protocol_builder *B, - ObjectID object_id); +Status SendDeleteRequest(int sock, ObjectID object_id); -void plasma_read_DeleteRequest(uint8_t *data, ObjectID *object_id); +Status ReadDeleteRequest(uint8_t *data, ObjectID *object_id); -int plasma_send_DeleteReply(int sock, - protocol_builder *B, - ObjectID object_id, - int error); +Status SendDeleteReply(int sock, ObjectID object_id, int error); -void plasma_read_DeleteReply(uint8_t *data, ObjectID *object_id, int *error); +Status ReadDeleteReply(uint8_t *data, ObjectID *object_id); -/* Plasma Status message functions. */ +/* Satus messages. */ -int plasma_send_StatusRequest(int sock, - protocol_builder *B, - ObjectID object_ids[], - int64_t num_objects); +Status SendStatusRequest(int sock, ObjectID object_ids[], int64_t num_objects); -int64_t plasma_read_StatusRequest_num_objects(uint8_t *data); +Status ReadStatusRequest(uint8_t *data, + ObjectID object_ids[], + int64_t num_objects); -void plasma_read_StatusRequest(uint8_t *data, - ObjectID object_ids[], - int64_t num_objects); +Status SendStatusReply(int sock, + ObjectID object_ids[], + int object_status[], + int64_t num_objects); -int plasma_send_StatusReply(int sock, - protocol_builder *B, - ObjectID object_ids[], - int object_status[], - int64_t num_objects); +int64_t ReadStatusReply_num_objects(uint8_t *data); -int64_t plasma_read_StatusReply_num_objects(uint8_t *data); - -void plasma_read_StatusReply(uint8_t *data, - ObjectID object_ids[], - int object_status[], - int64_t num_objects); +Status ReadStatusReply(uint8_t *data, + ObjectID object_ids[], + int object_status[], + int64_t num_objects); /* Plasma Constains message functions. */ -int plasma_send_ContainsRequest(int sock, - protocol_builder *B, - ObjectID object_id); +Status SendContainsRequest(int sock, ObjectID object_id); -void plasma_read_ContainsRequest(uint8_t *data, ObjectID *object_id); +Status ReadContainsRequest(uint8_t *data, ObjectID *object_id); -int plasma_send_ContainsReply(int sock, - protocol_builder *B, - ObjectID object_id, - int has_object); +Status SendContainsReply(int sock, ObjectID object_id, int has_object); -void plasma_read_ContainsReply(uint8_t *data, - ObjectID *object_id, - int *has_object); +Status ReadContainsReply(uint8_t *data, ObjectID *object_id, int *has_object); /* Plasma Connect message functions. */ -int plasma_send_ConnectRequest(int sock, protocol_builder *B); +Status SendConnectRequest(int sock); -void plasma_read_ConnectRequest(uint8_t *data); +Status ReadConnectRequest(uint8_t *data); -int plasma_send_ConnectReply(int sock, - protocol_builder *B, - int64_t memory_capacity); +Status SendConnectReply(int sock, int64_t memory_capacity); -void plasma_read_ConnectReply(uint8_t *data, int64_t *memory_capacity); +Status ReadConnectReply(uint8_t *data, int64_t *memory_capacity); /* Plasma Evict message functions (no reply so far). */ -int plasma_send_EvictRequest(int sock, protocol_builder *B, int64_t num_bytes); +Status SendEvictRequest(int sock, int64_t num_bytes); -void plasma_read_EvictRequest(uint8_t *data, int64_t *num_bytes); +Status ReadEvictRequest(uint8_t *data, int64_t *num_bytes); -int plasma_send_EvictReply(int sock, protocol_builder *B, int64_t num_bytes); +Status SendEvictReply(int sock, int64_t num_bytes); -void plasma_read_EvictReply(uint8_t *data, int64_t *num_bytes); +Status ReadEvictReply(uint8_t *data, int64_t &num_bytes); /* Plasma Fetch Remote message functions. */ -int plasma_send_FetchRequest(int sock, - protocol_builder *B, - ObjectID object_ids[], - int64_t num_objects); +Status SendFetchRequest(int sock, ObjectID object_ids[], int64_t num_objects); -int64_t plasma_read_FetchRequest_num_objects(uint8_t *data); - -void plasma_read_FetchRequest(uint8_t *data, - ObjectID object_ids[], - int64_t num_objects); +Status ReadFetchRequest(uint8_t *data, std::vector &object_ids); /* Plasma Wait message functions. */ -int plasma_send_WaitRequest(int sock, - protocol_builder *B, - ObjectRequest object_requests[], - int num_requests, - int num_ready_objects, - int64_t timeout_ms); +Status SendWaitRequest(int sock, + ObjectRequest object_requests[], + int num_requests, + int num_ready_objects, + int64_t timeout_ms); -int plasma_read_WaitRequest_num_object_ids(uint8_t *data); +Status ReadWaitRequest(uint8_t *data, + ObjectRequestMap &object_requests, + int64_t *timeout_ms, + int *num_ready_objects); -void plasma_read_WaitRequest(uint8_t *data, - ObjectRequestMap &object_requests, - int num_object_ids, - int64_t *timeout_ms, - int *num_ready_objects); +Status SendWaitReply(int sock, + const ObjectRequestMap &object_requests, + int num_ready_objects); -int plasma_send_WaitReply(int sock, - protocol_builder *B, - const ObjectRequestMap &object_requests, - int num_ready_objects); - -void plasma_read_WaitReply(uint8_t *data, - ObjectRequest object_requests[], - int *num_ready_objects); +Status ReadWaitReply(uint8_t *data, + ObjectRequest object_requests[], + int *num_ready_objects); /* Plasma Subscribe message functions. */ -int plasma_send_SubscribeRequest(int sock, protocol_builder *B); +Status SendSubscribeRequest(int sock); -/* Plasma Data message functions. */ +/* Data messages. */ -int plasma_send_DataRequest(int sock, - protocol_builder *B, - ObjectID object_id, - const char *address, - int port); +Status SendDataRequest(int sock, + ObjectID object_id, + const char *address, + int port); -void plasma_read_DataRequest(uint8_t *data, - ObjectID *object_id, - char **address, - int *port); +Status ReadDataRequest(uint8_t *data, + ObjectID *object_id, + char **address, + int *port); -int plasma_send_DataReply(int sock, - protocol_builder *B, - ObjectID object_id, - int64_t object_size, - int64_t metadata_size); +Status SendDataReply(int sock, + ObjectID object_id, + int64_t object_size, + int64_t metadata_size); -void plasma_read_DataReply(uint8_t *data, - ObjectID *object_id, - int64_t *object_size, - int64_t *metadata_size); +Status ReadDataReply(uint8_t *data, + ObjectID *object_id, + int64_t *object_size, + int64_t *metadata_size); #endif /* PLASMA_PROTOCOL */ diff --git a/src/plasma/plasma_store.cc b/src/plasma/plasma_store.cc index 62fa93583..39e632a03 100644 --- a/src/plasma/plasma_store.cc +++ b/src/plasma/plasma_store.cc @@ -1,13 +1,13 @@ -/* PLASMA STORE: This is a simple object store server process - * - * It accepts incoming client connections on a unix domain socket - * (name passed in via the -s option of the executable) and uses a - * single thread to serve the clients. Each client establishes a - * connection and can create objects, wait for objects and seal - * objects through that connection. - * - * It keeps a hash table that maps object_ids (which are 20 byte long, - * just enough to store and SHA1 hash) to memory mapped files. */ +// PLASMA STORE: This is a simple object store server process +// +// It accepts incoming client connections on a unix domain socket +// (name passed in via the -s option of the executable) and uses a +// single thread to serve the clients. Each client establishes a +// connection and can create objects, wait for objects and seal +// objects through that connection. +// +// It keeps a hash table that maps object_ids (which are 20 byte long, +// just enough to store and SHA1 hash) to memory mapped files. #include #include @@ -23,17 +23,16 @@ #include #include #include -#include #include #include #include #include +#include "plasma_common.h" #include "plasma_store.h" - #include "format/common_generated.h" -#include "io.h" +#include "plasma_io.h" #include "malloc.h" extern "C" { @@ -47,20 +46,20 @@ size_t dlmalloc_set_footprint_limit(size_t bytes); struct GetRequest { GetRequest(Client *client, const std::vector &object_ids); - /** The client that called get. */ + /// The client that called get. Client *client; - /** The ID of the timer that will time out and cause this wait to return to - * the client if it hasn't already returned. */ + /// The ID of the timer that will time out and cause this wait to return to + /// the client if it hasn't already returned. int64_t timer; - /** The object IDs involved in this request. This is used in the reply. */ + /// The object IDs involved in this request. This is used in the reply. std::vector object_ids; - /** The object information for the objects in this request. This is used in - * the reply. */ + /// The object information for the objects in this request. This is used in + /// the reply. std::unordered_map objects; - /** The minimum number of objects to wait for in this request. */ + /// The minimum number of objects to wait for in this request. int64_t num_objects_to_wait_for; - /** The number of object requests in this wait request that are already - * satisfied. */ + /// The number of object requests in this wait request that are already + /// satisfied. int64_t num_satisfied; }; @@ -78,9 +77,7 @@ GetRequest::GetRequest(Client *client, const std::vector &object_ids) Client::Client(int fd) : fd(fd) {} PlasmaStore::PlasmaStore(EventLoop *loop, int64_t system_memory) - : loop_(loop), - eviction_policy_(&store_info_), - builder_(make_protocol_builder()) { + : loop_(loop), eviction_policy_(&store_info_) { store_info_.memory_capacity = system_memory; } @@ -93,61 +90,59 @@ PlasmaStore::~PlasmaStore() { free(data); } } - free_protocol_builder(builder_); } -/* If this client is not already using the object, add the client to the - * object's list of clients, otherwise do nothing. */ +// If this client is not already using the object, add the client to the +// object's list of clients, otherwise do nothing. void PlasmaStore::add_client_to_object_clients(ObjectTableEntry *entry, Client *client) { - /* Check if this client is already using the object. */ + // Check if this client is already using the object. if (entry->clients.find(client) != entry->clients.end()) { return; } - /* If there are no other clients using this object, notify the eviction policy - * that the object is being used. */ + // If there are no other clients using this object, notify the eviction policy + // that the object is being used. if (entry->clients.size() == 0) { - /* Tell the eviction policy that this object is being used. */ + // Tell the eviction policy that this object is being used. std::vector objects_to_evict; eviction_policy_.begin_object_access(entry->object_id, objects_to_evict); delete_objects(objects_to_evict); } - /* Add the client pointer to the list of clients using this object. */ + // Add the client pointer to the list of clients using this object. entry->clients.insert(client); } -/* Create a new object buffer in the hash table. */ +// Create a new object buffer in the hash table. int PlasmaStore::create_object(ObjectID object_id, int64_t data_size, int64_t metadata_size, Client *client, PlasmaObject *result) { - LOG_DEBUG("creating object"); /* TODO(pcm): add ObjectID here */ + ARROW_LOG(DEBUG) << "creating object " << object_id.hex(); if (store_info_.objects.count(object_id) != 0) { - /* There is already an object with the same ID in the Plasma Store, so - * ignore this requst. */ + // There is already an object with the same ID in the Plasma Store, so + // ignore this requst. return PlasmaError_ObjectExists; } - /* Try to evict objects until there is enough space. */ + // Try to evict objects until there is enough space. uint8_t *pointer; do { - /* Allocate space for the new object. We use dlmemalign instead of dlmalloc - * in order to align the allocated region to a 64-byte boundary. This is not - * strictly necessary, but it is an optimization that could speed up the - * computation of a hash of the data (see compute_object_hash_parallel in - * plasma_client.cc). Note that even though this pointer is 64-byte aligned, - * it is not guaranteed that the corresponding pointer in the client will be - * 64-byte aligned, but in practice it often will be. */ + // Allocate space for the new object. We use dlmemalign instead of dlmalloc + // in order to align the allocated region to a 64-byte boundary. This is not + // strictly necessary, but it is an optimization that could speed up the + // computation of a hash of the data (see compute_object_hash_parallel in + // plasma_client.cc). Note that even though this pointer is 64-byte aligned, + // it is not guaranteed that the corresponding pointer in the client will be + // 64-byte aligned, but in practice it often will be. pointer = (uint8_t *) dlmemalign(BLOCK_SIZE, data_size + metadata_size); if (pointer == NULL) { - /* Tell the eviction policy how much space we need to create this object. - */ + // Tell the eviction policy how much space we need to create this object. std::vector objects_to_evict; bool success = eviction_policy_.require_space(data_size + metadata_size, objects_to_evict); delete_objects(objects_to_evict); - /* Return an error to the client if not enough space could be freed to - * create the object. */ + // Return an error to the client if not enough space could be freed to + // create the object. if (!success) { return PlasmaError_OutOfMemory; } @@ -161,12 +156,11 @@ int PlasmaStore::create_object(ObjectID object_id, auto entry = std::unique_ptr(new ObjectTableEntry()); entry->object_id = object_id; - entry->info.object_id = - std::string((char *) &object_id.id[0], sizeof(object_id)); + entry->info.object_id = object_id.binary(); entry->info.data_size = data_size; entry->info.metadata_size = metadata_size; entry->pointer = pointer; - /* TODO(pcm): set the other fields */ + // TODO(pcm): Set the other fields. entry->fd = fd; entry->map_size = map_size; entry->offset = offset; @@ -179,11 +173,11 @@ int PlasmaStore::create_object(ObjectID object_id, result->metadata_offset = offset + data_size; result->data_size = data_size; result->metadata_size = metadata_size; - /* Notify the eviction policy that this object was created. This must be done - * immediately before the call to add_client_to_object_clients so that the - * eviction policy does not have an opportunity to evict the object. */ + // Notify the eviction policy that this object was created. This must be done + // immediately before the call to add_client_to_object_clients so that the + // eviction policy does not have an opportunity to evict the object. eviction_policy_.object_created(object_id); - /* Record that this client is using this object. */ + // Record that this client is using this object. add_client_to_object_clients(store_info_.objects[object_id].get(), client); return PlasmaError_OK; } @@ -201,30 +195,28 @@ void PlasmaObject_init(PlasmaObject *object, ObjectTableEntry *entry) { } void PlasmaStore::return_from_get(GetRequest *get_req) { - /* Send the get reply to the client. */ - int status = plasma_send_GetReply(get_req->client->fd, builder_, - &get_req->object_ids[0], get_req->objects, - get_req->object_ids.size()); - warn_if_sigpipe(status, get_req->client->fd); - /* If we successfully sent the get reply message to the client, then also send - * the file descriptors. */ - if (status >= 0) { - /* Send all of the file descriptors for the present objects. */ + // Send the get reply to the client. + Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0], + get_req->objects, get_req->object_ids.size()); + warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd); + // If we successfully sent the get reply message to the client, then also send + // the file descriptors. + if (s.ok()) { + // Send all of the file descriptors for the present objects. for (const auto &object_id : get_req->object_ids) { PlasmaObject &object = get_req->objects[object_id]; - /* We use the data size to indicate whether the object is present or not. - */ + // We use the data size to indicate whether the object is present or not. if (object.data_size != -1) { int error_code = send_fd(get_req->client->fd, object.handle.store_fd); - /* If we failed to send the file descriptor, loop until we have sent it - * successfully. TODO(rkn): This is problematic for two reasons. First - * of all, sending the file descriptor should just succeed without any - * errors, but sometimes I see a "Message too long" error number. - * Second, looping like this allows a client to potentially block the - * plasma store event loop which should never happen. */ + // If we failed to send the file descriptor, loop until we have sent it + // successfully. TODO(rkn): This is problematic for two reasons. First + // of all, sending the file descriptor should just succeed without any + // errors, but sometimes I see a "Message too long" error number. + // Second, looping like this allows a client to potentially block the + // plasma store event loop which should never happen. while (error_code < 0) { if (errno == EMSGSIZE) { - LOG_WARN("Failed to send file descriptor, retrying."); + ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying."; error_code = send_fd(get_req->client->fd, object.handle.store_fd); continue; } @@ -235,20 +227,20 @@ void PlasmaStore::return_from_get(GetRequest *get_req) { } } - /* Remove the get request from each of the relevant object_get_requests hash - * tables if it is present there. It should only be present there if the get - * request timed out. */ + // Remove the get request from each of the relevant object_get_requests hash + // tables if it is present there. It should only be present there if the get + // request timed out. for (ObjectID &object_id : get_req->object_ids) { auto &get_requests = object_get_requests_[object_id]; - /* Erase get_req from the vector. */ + // Erase get_req from the vector. auto it = std::find(get_requests.begin(), get_requests.end(), get_req); if (it != get_requests.end()) { get_requests.erase(it); } } - /* Remove the get request. */ + // Remove the get request. if (get_req->timer != -1) { - CHECK(loop_->remove_timer(get_req->timer) == AE_OK); + ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK); } delete get_req; } @@ -260,65 +252,65 @@ void PlasmaStore::update_object_get_requests(ObjectID object_id) { for (int i = 0; i < num_requests; ++i) { GetRequest *get_req = get_requests[index]; auto entry = get_object_table_entry(&store_info_, object_id); - CHECK(entry != NULL); + ARROW_CHECK(entry != NULL); PlasmaObject_init(&get_req->objects[object_id], entry); get_req->num_satisfied += 1; - /* Record the fact that this client will be using this object and will - * be responsible for releasing this object. */ + // Record the fact that this client will be using this object and will + // be responsible for releasing this object. add_client_to_object_clients(entry, get_req->client); - /* If this get request is done, reply to the client. */ + // If this get request is done, reply to the client. if (get_req->num_satisfied == get_req->num_objects_to_wait_for) { return_from_get(get_req); } else { - /* The call to return_from_get will remove the current element in the - * array, so we only increment the counter in the else branch. */ + // The call to return_from_get will remove the current element in the + // array, so we only increment the counter in the else branch. index += 1; } } DCHECK(index == get_requests.size()); - /* Remove the array of get requests for this object, since no one should be - * waiting for this object anymore. */ + // Remove the array of get requests for this object, since no one should be + // waiting for this object anymore. object_get_requests_.erase(object_id); } void PlasmaStore::process_get_request(Client *client, const std::vector &object_ids, uint64_t timeout_ms) { - /* Create a get request for this object. */ + // Create a get request for this object. GetRequest *get_req = new GetRequest(client, object_ids); for (auto object_id : object_ids) { - /* Check if this object is already present locally. If so, record that the - * object is being used and mark it as accounted for. */ + // Check if this object is already present locally. If so, record that the + // object is being used and mark it as accounted for. auto entry = get_object_table_entry(&store_info_, object_id); if (entry && entry->state == PLASMA_SEALED) { - /* Update the get request to take into account the present object. */ + // Update the get request to take into account the present object. PlasmaObject_init(&get_req->objects[object_id], entry); get_req->num_satisfied += 1; - /* If necessary, record that this client is using this object. In the case - * where entry == NULL, this will be called from seal_object. */ + // If necessary, record that this client is using this object. In the case + // where entry == NULL, this will be called from seal_object. add_client_to_object_clients(entry, client); } else { - /* Add a placeholder plasma object to the get request to indicate that the - * object is not present. This will be parsed by the client. We set the - * data size to -1 to indicate that the object is not present. */ + // Add a placeholder plasma object to the get request to indicate that the + // object is not present. This will be parsed by the client. We set the + // data size to -1 to indicate that the object is not present. get_req->objects[object_id].data_size = -1; - /* Add the get request to the relevant data structures. */ + // Add the get request to the relevant data structures. object_get_requests_[object_id].push_back(get_req); } } - /* If all of the objects are present already or if the timeout is 0, return to - * the client. */ + // If all of the objects are present already or if the timeout is 0, return to + // the client. if (get_req->num_satisfied == get_req->num_objects_to_wait_for || timeout_ms == 0) { return_from_get(get_req); } else if (timeout_ms != -1) { - /* Set a timer that will cause the get request to return to the client. Note - * that a timeout of -1 is used to indicate that no timer should be set. */ + // Set a timer that will cause the get request to return to the client. Note + // that a timeout of -1 is used to indicate that no timer should be set. get_req->timer = loop_->add_timer(timeout_ms, [this, get_req](int64_t timer_id) { return_from_get(get_req); @@ -332,166 +324,165 @@ int PlasmaStore::remove_client_from_object_clients(ObjectTableEntry *entry, auto it = entry->clients.find(client); if (it != entry->clients.end()) { entry->clients.erase(it); - /* If no more clients are using this object, notify the eviction policy - * that the object is no longer being used. */ + // If no more clients are using this object, notify the eviction policy + // that the object is no longer being used. if (entry->clients.size() == 0) { - /* Tell the eviction policy that this object is no longer being used. */ + // Tell the eviction policy that this object is no longer being used. std::vector objects_to_evict; eviction_policy_.end_object_access(entry->object_id, objects_to_evict); delete_objects(objects_to_evict); } - /* Return 1 to indicate that the client was removed. */ + // Return 1 to indicate that the client was removed. return 1; } else { - /* Return 0 to indicate that the client was not removed. */ + // Return 0 to indicate that the client was not removed. return 0; } } void PlasmaStore::release_object(ObjectID object_id, Client *client) { auto entry = get_object_table_entry(&store_info_, object_id); - CHECK(entry != NULL); - /* Remove the client from the object's array of clients. */ - CHECK(remove_client_from_object_clients(entry, client) == 1); + ARROW_CHECK(entry != NULL); + // Remove the client from the object's array of clients. + ARROW_CHECK(remove_client_from_object_clients(entry, client) == 1); } -/* Check if an object is present. */ +// Check if an object is present. int PlasmaStore::contains_object(ObjectID object_id) { auto entry = get_object_table_entry(&store_info_, object_id); return entry && (entry->state == PLASMA_SEALED) ? OBJECT_FOUND : OBJECT_NOT_FOUND; } -/* Seal an object that has been created in the hash table. */ +// Seal an object that has been created in the hash table. void PlasmaStore::seal_object(ObjectID object_id, unsigned char digest[]) { - LOG_DEBUG("sealing object"); // TODO(pcm): add ObjectID here + ARROW_LOG(DEBUG) << "sealing object " << object_id.hex(); auto entry = get_object_table_entry(&store_info_, object_id); - CHECK(entry != NULL); - CHECK(entry->state == PLASMA_CREATED); - /* Set the state of object to SEALED. */ + ARROW_CHECK(entry != NULL); + ARROW_CHECK(entry->state == PLASMA_CREATED); + // Set the state of object to SEALED. entry->state = PLASMA_SEALED; - /* Set the object digest. */ - entry->info.digest = std::string((char *) &digest[0], DIGEST_SIZE); - /* Inform all subscribers that a new object has been sealed. */ + // Set the object digest. + entry->info.digest = std::string((char *) &digest[0], kDigestSize); + // Inform all subscribers that a new object has been sealed. push_notification(&entry->info); - /* Update all get requests that involve this object. */ + // Update all get requests that involve this object. update_object_get_requests(object_id); } void PlasmaStore::delete_objects(const std::vector &object_ids) { for (const auto &object_id : object_ids) { - LOG_DEBUG("deleting object"); + ARROW_LOG(DEBUG) << "deleting object " << object_id.hex(); auto entry = get_object_table_entry(&store_info_, object_id); - /* TODO(rkn): This should probably not fail, but should instead throw an - * error. Maybe we should also support deleting objects that have been - * created but not sealed. */ - CHECKM(entry != NULL, - "To delete an object it must be in the object table."); - CHECKM(entry->state == PLASMA_SEALED, - "To delete an object it must have been sealed."); - CHECKM(entry->clients.size() == 0, - "To delete an object, there must be no clients currently using it."); + // TODO(rkn): This should probably not fail, but should instead throw an + // error. Maybe we should also support deleting objects that have been + // created but not sealed. + ARROW_CHECK(entry != NULL) + << "To delete an object it must be in the object table."; + ARROW_CHECK(entry->state == PLASMA_SEALED) + << "To delete an object it must have been sealed."; + ARROW_CHECK(entry->clients.size() == 0) + << "To delete an object, there must be no clients currently using it."; dlfree(entry->pointer); store_info_.objects.erase(object_id); - /* Inform all subscribers that the object has been deleted. */ + // Inform all subscribers that the object has been deleted. ObjectInfoT notification; - notification.object_id = - std::string((char *) &object_id.id[0], sizeof(object_id)); + notification.object_id = object_id.binary(); notification.is_deletion = true; push_notification(¬ification); } } void PlasmaStore::connect_client(int listener_sock) { - int client_fd = accept_client(listener_sock); - /* This is freed in disconnect_client. */ + int client_fd = AcceptClient(listener_sock); + // This is freed in disconnect_client. Client *client = new Client(client_fd); - /* Add a callback to handle events on this socket. - * TODO(pcm): Check return value. */ + // Add a callback to handle events on this socket. + // TODO(pcm): Check return value. loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) { process_message(client); }); - LOG_DEBUG("New connection with fd %d", client_fd); + ARROW_LOG(DEBUG) << "New connection with fd " << client_fd; } void PlasmaStore::disconnect_client(Client *client) { loop_->remove_file_event(client->fd); - /* If this client was using any objects, remove it from the appropriate - * lists. */ + // If this client was using any objects, remove it from the appropriate + // lists. for (const auto &entry : store_info_.objects) { remove_client_from_object_clients(entry.second.get(), client); } - /* Note, the store may still attempt to send a message to the disconnected - * client (for example, when an object ID that the client was waiting for - * is ready). In these cases, the attempt to send the message will fail, but - * the store should just ignore the failure. */ + // Note, the store may still attempt to send a message to the disconnected + // client (for example, when an object ID that the client was waiting for + // is ready). In these cases, the attempt to send the message will fail, but + // the store should just ignore the failure. delete client; } -/** - * Send notifications about sealed objects to the subscribers. This is called - * in seal_object. If the socket's send buffer is full, the notification will be - * buffered, and this will be called again when the send buffer has room. - * - * @param client The client to send the notification to. - * @return Void. - */ +/// Send notifications about sealed objects to the subscribers. This is called +/// in seal_object. If the socket's send buffer is full, the notification will +/// be +/// buffered, and this will be called again when the send buffer has room. +/// +/// @param client The client to send the notification to. +/// @return Void. void PlasmaStore::send_notifications(int client_fd) { auto it = pending_notifications_.find(client_fd); int num_processed = 0; bool closed = false; - /* Loop over the array of pending notifications and send as many of them as - * possible. */ + // Loop over the array of pending notifications and send as many of them as + // possible. for (int i = 0; i < it->second.object_notifications.size(); ++i) { uint8_t *notification = (uint8_t *) it->second.object_notifications.at(i); - /* Decode the length, which is the first bytes of the message. */ + // Decode the length, which is the first bytes of the message. int64_t size = *((int64_t *) notification); - /* Attempt to send a notification about this object ID. */ + // Attempt to send a notification about this object ID. int nbytes = send(client_fd, notification, sizeof(int64_t) + size, 0); if (nbytes >= 0) { - CHECK(nbytes == sizeof(int64_t) + size); + ARROW_CHECK(nbytes == sizeof(int64_t) + size); } else if (nbytes == -1 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) { - LOG_DEBUG( - "The socket's send buffer is full, so we are caching this " - "notification and will send it later."); - /* Add a callback to the event loop to send queued notifications whenever - * there is room in the socket's send buffer. Callbacks can be added - * more than once here and will be overwritten. The callback is removed - * at the end of the method. - * TODO(pcm): Introduce status codes and check in case the file descriptor - * is added twice.*/ + ARROW_LOG(DEBUG) + << "The socket's send buffer is full, so we are caching this " + "notification and will send it later."; + // Add a callback to the event loop to send queued notifications whenever + // there is room in the socket's send buffer. Callbacks can be added + // more than once here and will be overwritten. The callback is removed + // at the end of the method. + // TODO(pcm): Introduce status codes and check in case the file descriptor + // is added twice. loop_->add_file_event( client_fd, kEventLoopWrite, [this, client_fd](int events) { send_notifications(client_fd); }); break; } else { - LOG_WARN("Failed to send notification to client on fd %d", client_fd); + ARROW_LOG(WARNING) << "Failed to send notification to client on fd " + << client_fd; if (errno == EPIPE) { closed = true; break; } } num_processed += 1; - /* The corresponding malloc happened in create_object_info_buffer - * within push_notification. */ + // The corresponding malloc happened in create_object_info_buffer + // within push_notification. free(notification); } - /* Remove the sent notifications from the array. */ + // Remove the sent notifications from the array. it->second.object_notifications.erase( it->second.object_notifications.begin(), it->second.object_notifications.begin() + num_processed); - /* Stop sending notifications if the pipe was broken. */ + // Stop sending notifications if the pipe was broken. if (closed) { close(client_fd); pending_notifications_.erase(client_fd); } - /* If we have sent all notifications, remove the fd from the event loop. */ + // If we have sent all notifications, remove the fd from the event loop. if (it->second.object_notifications.empty()) { loop_->remove_file_event(client_fd); } @@ -502,121 +493,114 @@ void PlasmaStore::push_notification(ObjectInfoT *object_info) { uint8_t *notification = create_object_info_buffer(object_info); element.second.object_notifications.push_back(notification); send_notifications(element.first); - /* The notification gets freed in send_notifications when the notification - * is sent over the socket. */ + // The notification gets freed in send_notifications when the notification + // is sent over the socket. } } -/* Subscribe to notifications about sealed objects. */ +// Subscribe to notifications about sealed objects. void PlasmaStore::subscribe_to_updates(Client *client) { - LOG_DEBUG("subscribing to updates"); - /* TODO(rkn): The store could block here if the client doesn't send a file - * descriptor. */ + ARROW_LOG(DEBUG) << "subscribing to updates on fd " << client->fd; + // TODO(rkn): The store could block here if the client doesn't send a file + // descriptor. int fd = recv_fd(client->fd); if (fd < 0) { - /* This may mean that the client died before sending the file descriptor. */ - LOG_WARN("Failed to receive file descriptor from client on fd %d.", - client->fd); + // This may mean that the client died before sending the file descriptor. + ARROW_LOG(WARNING) << "Failed to receive file descriptor from client on fd " + << client->fd << "."; return; } - /* Create a new array to buffer notifications that can't be sent to the - * subscriber yet because the socket send buffer is full. TODO(rkn): the queue - * never gets freed. */ + // Create a new array to buffer notifications that can't be sent to the + // subscriber yet because the socket send buffer is full. TODO(rkn): the queue + // never gets freed. NotificationQueue &queue = pending_notifications_[fd]; - /* Push notifications to the new subscriber about existing objects. */ + // Push notifications to the new subscriber about existing objects. for (const auto &entry : store_info_.objects) { push_notification(&entry.second->info); } send_notifications(fd); } -void PlasmaStore::process_message(Client *client) { +Status PlasmaStore::process_message(Client *client) { int64_t type; - read_vector(client->fd, &type, input_buffer_); + Status s = ReadMessage(client->fd, &type, input_buffer_); + ARROW_CHECK(s.ok() || s.IsIOError()); uint8_t *input = input_buffer_.data(); ObjectID object_id; PlasmaObject object; - /* TODO(pcm): Get rid of the following. */ + // TODO(pcm): Get rid of the following. memset(&object, 0, sizeof(object)); - /* Process the different types of requests. */ + // Process the different types of requests. switch (type) { case MessageType_PlasmaCreateRequest: { int64_t data_size; int64_t metadata_size; - plasma_read_CreateRequest(input, &object_id, &data_size, &metadata_size); + RETURN_NOT_OK( + ReadCreateRequest(input, &object_id, &data_size, &metadata_size)); int error_code = create_object(object_id, data_size, metadata_size, client, &object); - warn_if_sigpipe(plasma_send_CreateReply(client->fd, builder_, object_id, - &object, error_code), - client->fd); + HANDLE_SIGPIPE(SendCreateReply(client->fd, object_id, &object, error_code), + client->fd); if (error_code == PlasmaError_OK) { warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd); } } break; case MessageType_PlasmaGetRequest: { - int num_objects = plasma_read_GetRequest_num_objects(input); - std::vector object_ids_to_get(num_objects); + std::vector object_ids_to_get; int64_t timeout_ms; - plasma_read_GetRequest(input, object_ids_to_get.data(), &timeout_ms, - num_objects); + RETURN_NOT_OK(ReadGetRequest(input, object_ids_to_get, &timeout_ms)); process_get_request(client, object_ids_to_get, timeout_ms); } break; case MessageType_PlasmaReleaseRequest: - plasma_read_ReleaseRequest(input, &object_id); + RETURN_NOT_OK(ReadReleaseRequest(input, &object_id)); release_object(object_id, client); break; case MessageType_PlasmaContainsRequest: - plasma_read_ContainsRequest(input, &object_id); + RETURN_NOT_OK(ReadContainsRequest(input, &object_id)); if (contains_object(object_id) == OBJECT_FOUND) { - warn_if_sigpipe( - plasma_send_ContainsReply(client->fd, builder_, object_id, 1), - client->fd); + HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd); } else { - warn_if_sigpipe( - plasma_send_ContainsReply(client->fd, builder_, object_id, 0), - client->fd); + HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd); } break; case MessageType_PlasmaSealRequest: { - unsigned char digest[DIGEST_SIZE]; - plasma_read_SealRequest(input, &object_id, &digest[0]); + unsigned char digest[kDigestSize]; + RETURN_NOT_OK(ReadSealRequest(input, &object_id, &digest[0])); seal_object(object_id, &digest[0]); } break; case MessageType_PlasmaEvictRequest: { - /* This code path should only be used for testing. */ + // This code path should only be used for testing. int64_t num_bytes; - plasma_read_EvictRequest(input, &num_bytes); + RETURN_NOT_OK(ReadEvictRequest(input, &num_bytes)); std::vector objects_to_evict; int64_t num_bytes_evicted = eviction_policy_.choose_objects_to_evict(num_bytes, objects_to_evict); delete_objects(objects_to_evict); - warn_if_sigpipe( - plasma_send_EvictReply(client->fd, builder_, num_bytes_evicted), - client->fd); + HANDLE_SIGPIPE(SendEvictReply(client->fd, num_bytes_evicted), client->fd); } break; case MessageType_PlasmaSubscribeRequest: subscribe_to_updates(client); break; case MessageType_PlasmaConnectRequest: { - warn_if_sigpipe(plasma_send_ConnectReply(client->fd, builder_, - store_info_.memory_capacity), - client->fd); + HANDLE_SIGPIPE(SendConnectReply(client->fd, store_info_.memory_capacity), + client->fd); } break; case DISCONNECT_CLIENT: - LOG_INFO("Disconnecting client on fd %d", client->fd); + ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd; disconnect_client(client); break; default: - /* This code should be unreachable. */ - CHECK(0); + // This code should be unreachable. + ARROW_CHECK(0); } + return Status::OK(); } -/* Report "success" to valgrind. */ +// Report "success" to valgrind. void signal_handler(int signal) { if (signal == SIGTERM) { exit(0); @@ -624,15 +608,15 @@ void signal_handler(int signal) { } void start_server(char *socket_name, int64_t system_memory) { - /* Ignore SIGPIPE signals. If we don't do this, then when we attempt to write - * to a client that has already died, the store could die. */ + // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write + // to a client that has already died, the store could die. signal(SIGPIPE, SIG_IGN); - /* Create the event loop. */ + // Create the event loop. EventLoop loop; PlasmaStore store(&loop, system_memory); int socket = bind_ipc_sock(socket_name, true); - CHECK(socket >= 0); - /* TODO(pcm): Check return value. */ + ARROW_CHECK(socket >= 0); + // TODO(pcm): Check return value. loop.add_file_event(socket, kEventLoopRead, [&store, socket](int events) { store.connect_client(socket); }); @@ -652,9 +636,10 @@ int main(int argc, char *argv[]) { case 'm': { char extra; int scanned = sscanf(optarg, "%" SCNd64 "%c", &system_memory, &extra); - CHECK(scanned == 1); - LOG_INFO("Allowing the Plasma store to use up to %.2fGB of memory.", - ((double) system_memory) / 1000000000); + ARROW_CHECK(scanned == 1); + ARROW_LOG(INFO) << "Allowing the Plasma store to use up to " + << ((double) system_memory) / 1000000000 + << "GB of memory."; break; } default: @@ -662,34 +647,38 @@ int main(int argc, char *argv[]) { } } if (!socket_name) { - LOG_FATAL("please specify socket for incoming connections with -s switch"); + ARROW_LOG(FATAL) + << "please specify socket for incoming connections with -s switch"; } if (system_memory == -1) { - LOG_FATAL("please specify the amount of system memory with -m switch"); + ARROW_LOG(FATAL) + << "please specify the amount of system memory with -m switch"; } #ifdef __linux__ - /* On Linux, check that the amount of memory available in /dev/shm is large - * enough to accommodate the request. If it isn't, then fail. */ + // On Linux, check that the amount of memory available in /dev/shm is large + // enough to accommodate the request. If it isn't, then fail. int shm_fd = open("/dev/shm", O_RDONLY); struct statvfs shm_vfs_stats; fstatvfs(shm_fd, &shm_vfs_stats); - /* The value shm_vfs_stats.f_bsize is the block size, and the value - * shm_vfs_stats.f_bavail is the number of available blocks. */ + // The value shm_vfs_stats.f_bsize is the block size, and the value + // shm_vfs_stats.f_bavail is the number of available blocks. int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail; close(shm_fd); if (system_memory > shm_mem_avail) { - LOG_FATAL( - "System memory request exceeds memory available in /dev/shm. The " - "request is for %" PRId64 " bytes, and the amount available is %" PRId64 - " bytes. You may be able to free up space by deleting files in " - "/dev/shm. If you are inside a Docker container, you may need to pass " - "an argument with the flag '--shm-size' to 'docker run'.", - system_memory, shm_mem_avail); + ARROW_LOG(FATAL) + << "System memory request exceeds memory available in /dev/shm. The " + "request is for " + << system_memory << " bytes, and the amount available is " + << shm_mem_avail + << " bytes. You may be able to free up space by deleting files in " + "/dev/shm. If you are inside a Docker container, you may need to " + "pass " + "an argument with the flag '--shm-size' to 'docker run'."; } #endif - /* Make it so dlmalloc fails if we try to request more memory than is - * available. */ + // Make it so dlmalloc fails if we try to request more memory than is + // available. dlmalloc_set_footprint_limit((size_t) system_memory); - LOG_DEBUG("starting server listening on %s", socket_name); + ARROW_LOG(DEBUG) << "starting server listening on " << socket_name; start_server(socket_name, system_memory); } diff --git a/src/plasma/plasma_store.h b/src/plasma/plasma_store.h index a74873121..c63fb43b6 100644 --- a/src/plasma/plasma_store.h +++ b/src/plasma/plasma_store.h @@ -1,25 +1,25 @@ #ifndef PLASMA_STORE_H #define PLASMA_STORE_H -#include "common.h" #include "eviction_policy.h" #include "plasma.h" +#include "plasma_common.h" #include "plasma_events.h" #include "plasma_protocol.h" class GetRequest; struct NotificationQueue { - /** The object notifications for clients. We notify the client about the - * objects in the order that the objects were sealed or deleted. */ + /// The object notifications for clients. We notify the client about the + /// objects in the order that the objects were sealed or deleted. std::deque object_notifications; }; -/** Contains all information that is associated with a Plasma store client. */ +/// Contains all information that is associated with a Plasma store client. struct Client { Client(int fd); - /** The file descriptor used to communicate with the client. */ + /// The file descriptor used to communicate with the client. int fd; }; @@ -29,109 +29,93 @@ class PlasmaStore { ~PlasmaStore(); - /** - * Create a new object. The client must do a call to release_object to tell - * the store when it is done with the object. - * - * @param object_id Object ID of the object to be created. - * @param data_size Size in bytes of the object to be created. - * @param metadata_size Size in bytes of the object metadata. - * @return One of the following error codes: - * - PlasmaError_OK, if the object was created successfully. - * - PlasmaError_ObjectExists, if an object with this ID is already - * present in the store. In this case, the client should not call - * plasma_release. - * - PlasmaError_OutOfMemory, if the store is out of memory and cannot - * create the object. In this case, the client should not call - * plasma_release. - */ + /// Create a new object. The client must do a call to release_object to tell + /// the store when it is done with the object. + /// + /// @param object_id Object ID of the object to be created. + /// @param data_size Size in bytes of the object to be created. + /// @param metadata_size Size in bytes of the object metadata. + /// @return One of the following error codes: + /// - PlasmaError_OK, if the object was created successfully. + /// - PlasmaError_ObjectExists, if an object with this ID is already + /// present in the store. In this case, the client should not call + /// plasma_release. + /// - PlasmaError_OutOfMemory, if the store is out of memory and + /// cannot create the object. In this case, the client should not call + /// plasma_release. int create_object(ObjectID object_id, int64_t data_size, int64_t metadata_size, Client *client, PlasmaObject *result); - /** - * Delete objects that have been created in the hash table. This should only - * be called on objects that are returned by the eviction policy to evict. - * - * @param object_ids Object IDs of the objects to be deleted. - * @return Void. - */ + /// Delete objects that have been created in the hash table. This should only + /// be called on objects that are returned by the eviction policy to evict. + /// + /// @param object_ids Object IDs of the objects to be deleted. + /// @return Void. void delete_objects(const std::vector &object_ids); - /** - * Process a get request from a client. This method assumes that we will - * eventually have these objects sealed. If one of the objects has not yet - * been sealed, the client that requested the object will be notified when it - * is sealed. - * - * For each object, the client must do a call to release_object to tell the - * store when it is done with the object. - * - * @param client The client making this request. - * @param object_ids Object IDs of the objects to be gotten. - * @param timeout_ms The timeout for the get request in milliseconds. - * @return Void. - */ + /// Process a get request from a client. This method assumes that we will + /// eventually have these objects sealed. If one of the objects has not yet + /// been sealed, the client that requested the object will be notified when it + /// is sealed. + /// + /// For each object, the client must do a call to release_object to tell the + /// store when it is done with the object. + /// + /// @param client The client making this request. + /// @param object_ids Object IDs of the objects to be gotten. + /// @param timeout_ms The timeout for the get request in milliseconds. + /// @return Void. void process_get_request(Client *client, const std::vector &object_ids, uint64_t timeout_ms); - /** - * Seal an object. The object is now immutable and can be accessed with get. - * - * @param object_id Object ID of the object to be sealed. - * @param digest The digest of the object. This is used to tell if two objects - * with the same object ID are the same. - * @return Void. - */ + /// Seal an object. The object is now immutable and can be accessed with get. + /// + /// @param object_id Object ID of the object to be sealed. + /// @param digest The digest of the object. This is used to tell if two + /// objects + /// with the same object ID are the same. + /// @return Void. void seal_object(ObjectID object_id, unsigned char digest[]); - /** - * Check if the plasma store contains an object: - * - * @param object_id Object ID that will be checked. - * @return OBJECT_FOUND if the object is in the store, OBJECT_NOT_FOUND if not - */ + /// Check if the plasma store contains an object: + /// + /// @param object_id Object ID that will be checked. + /// @return OBJECT_FOUND if the object is in the store, OBJECT_NOT_FOUND if + /// not int contains_object(ObjectID object_id); - /** - * Record the fact that a particular client is no longer using an object. - * - * @param object_id The object ID of the object that is being released. - * @param client The client making this request. - * @param Void. - */ + /// Record the fact that a particular client is no longer using an object. + /// + /// @param object_id The object ID of the object that is being released. + /// @param client The client making this request. + /// @param Void. void release_object(ObjectID object_id, Client *client); - /** - * Subscribe a file descriptor to updates about new sealed objects. - * - * @param client The client making this request. - * @return Void. - */ + /// Subscribe a file descriptor to updates about new sealed objects. + /// + /// @param client The client making this request. + /// @return Void. void subscribe_to_updates(Client *client); - /** - * Connect a new client to the PlasmaStore. - * - * @param listener_sock The socket that is listening to incoming connections. - * @return Void. - */ + /// Connect a new client to the PlasmaStore. + /// + /// @param listener_sock The socket that is listening to incoming connections. + /// @return Void. void connect_client(int listener_sock); - /** - * Disconnect a client from the PlasmaStore. - * - * @param client The client that is disconnected. - * @return Void. - */ + /// Disconnect a client from the PlasmaStore. + /// + /// @param client The client that is disconnected. + /// @return Void. void disconnect_client(Client *client); void send_notifications(int client_fd); - void process_message(Client *client); + Status process_message(Client *client); private: void push_notification(ObjectInfoT *object_notification); @@ -145,28 +129,26 @@ class PlasmaStore { int remove_client_from_object_clients(ObjectTableEntry *entry, Client *client); - /* Event loop of the plasma store. */ + /// Event loop of the plasma store. EventLoop *loop_; - /** The plasma store information, including the object tables, that is exposed - * to the eviction policy. */ + /// The plasma store information, including the object tables, that is exposed + /// to the eviction policy. PlasmaStoreInfo store_info_; - /** The state that is managed by the eviction policy. */ + /// The state that is managed by the eviction policy. EvictionPolicy eviction_policy_; - /** Input buffer. This is allocated only once to avoid mallocs for every - * call to process_message. */ + /// Input buffer. This is allocated only once to avoid mallocs for every + /// call to process_message. std::vector input_buffer_; - /** Buffer that holds memory for serializing plasma protocol messages. */ - protocol_builder *builder_; - /** A hash table mapping object IDs to a vector of the get requests that are - * waiting for the object to arrive. */ + /// A hash table mapping object IDs to a vector of the get requests that are + /// waiting for the object to arrive. std::unordered_map, UniqueIDHasher> object_get_requests_; - /** The pending notifications that have not been sent to subscribers because - * the socket send buffers were full. This is a hash table from client file - * descriptor to an array of object_ids to send to that client. - * TODO(pcm): Consider putting this into the Client data structure and - * reorganize the code slightly. */ + /// The pending notifications that have not been sent to subscribers because + /// the socket send buffers were full. This is a hash table from client file + /// descriptor to an array of object_ids to send to that client. + /// TODO(pcm): Consider putting this into the Client data structure and + /// reorganize the code slightly. std::unordered_map pending_notifications_; }; -#endif /* PLASMA_STORE_H */ +#endif // PLASMA_STORE_H diff --git a/src/plasma/status.cc b/src/plasma/status.cc new file mode 100644 index 000000000..11082d6cd --- /dev/null +++ b/src/plasma/status.cc @@ -0,0 +1,90 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A Status encapsulates the result of an operation. It may indicate success, +// or it may indicate an error with an associated error message. +// +// Multiple threads can invoke const methods on a Status without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Status must use +// external synchronization. + +#include "status.h" + +#include + +namespace arrow { + +Status::Status(StatusCode code, const std::string &msg, int16_t posix_code) { + assert(code != StatusCode::OK); + const uint32_t size = static_cast(msg.size()); + char *result = new char[size + 7]; + memcpy(result, &size, sizeof(size)); + result[4] = static_cast(code); + memcpy(result + 5, &posix_code, sizeof(posix_code)); + memcpy(result + 7, msg.c_str(), msg.size()); + state_ = result; +} + +const char *Status::CopyState(const char *state) { + uint32_t size; + memcpy(&size, state, sizeof(size)); + char *result = new char[size + 7]; + memcpy(result, state, size + 7); + return result; +} + +std::string Status::CodeAsString() const { + if (state_ == NULL) { + return "OK"; + } + + const char *type; + switch (code()) { + case StatusCode::OK: + type = "OK"; + break; + case StatusCode::OutOfMemory: + type = "Out of memory"; + break; + case StatusCode::KeyError: + type = "Key error"; + break; + case StatusCode::TypeError: + type = "Type error"; + break; + case StatusCode::Invalid: + type = "Invalid"; + break; + case StatusCode::IOError: + type = "IOError"; + break; + case StatusCode::UnknownError: + type = "Unknown error"; + break; + case StatusCode::NotImplemented: + type = "NotImplemented"; + break; + default: + type = "Unknown"; + break; + } + return std::string(type); +} + +std::string Status::ToString() const { + std::string result(CodeAsString()); + if (state_ == NULL) { + return result; + } + + result.append(": "); + + uint32_t length; + memcpy(&length, state_, sizeof(length)); + result.append(reinterpret_cast(state_ + 7), length); + return result; +} + +} // namespace arrow diff --git a/src/plasma/status.h b/src/plasma/status.h new file mode 100644 index 000000000..30d3a1e2f --- /dev/null +++ b/src/plasma/status.h @@ -0,0 +1,226 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A Status encapsulates the result of an operation. It may indicate success, +// or it may indicate an error with an associated error message. +// +// Multiple threads can invoke const methods on a Status without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Status must use +// external synchronization. + +// Adapted from Kudu github.com/cloudera/kudu + +#ifndef ARROW_STATUS_H_ +#define ARROW_STATUS_H_ + +#include +#include +#include + +// Return the given status if it is not OK. +#define ARROW_RETURN_NOT_OK(s) \ + do { \ + ::arrow::Status _s = (s); \ + if (!_s.ok()) { \ + return _s; \ + } \ + } while (0); + +// Return the given status if it is not OK, but first clone it and +// prepend the given message. +#define ARROW_RETURN_NOT_OK_PREPEND(s, msg) \ + do { \ + ::arrow::Status _s = (s); \ + if (::gutil::PREDICT_FALSE(!_s.ok())) \ + return _s.CloneAndPrepend(msg); \ + } while (0); + +// Return 'to_return' if 'to_call' returns a bad status. +// The substitution for 'to_return' may reference the variable +// 's' for the bad status. +#define ARROW_RETURN_NOT_OK_RET(to_call, to_return) \ + do { \ + ::arrow::Status s = (to_call); \ + if (::gutil::PREDICT_FALSE(!s.ok())) \ + return (to_return); \ + } while (0); + +// If 'to_call' returns a bad status, CHECK immediately with a logged message +// of 'msg' followed by the status. +#define ARROW_CHECK_OK_PREPEND(to_call, msg) \ + do { \ + ::arrow::Status _s = (to_call); \ + ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString(); \ + } while (0); + +// If the status is bad, CHECK immediately, appending the status to the +// logged message. +#define ARROW_CHECK_OK(s) ARROW_CHECK_OK_PREPEND(s, "Bad status") + +namespace arrow { + +#define RETURN_NOT_OK(s) \ + do { \ + Status _s = (s); \ + if (!_s.ok()) { \ + return _s; \ + } \ + } while (0); + +#define RETURN_NOT_OK_ELSE(s, else_) \ + do { \ + Status _s = (s); \ + if (!_s.ok()) { \ + else_; \ + return _s; \ + } \ + } while (0); + +enum class StatusCode : char { + OK = 0, + OutOfMemory = 1, + KeyError = 2, + TypeError = 3, + Invalid = 4, + IOError = 5, + UnknownError = 9, + NotImplemented = 10, + PlasmaObjectExists = 20, + PlasmaObjectNonexistent = 21, + PlasmaStoreFull = 22 +}; + +class Status { + public: + // Create a success status. + Status() : state_(NULL) {} + ~Status() { delete[] state_; } + + Status(StatusCode code, const std::string &msg) : Status(code, msg, -1) {} + + // Copy the specified status. + Status(const Status &s); + void operator=(const Status &s); + + // Return a success status. + static Status OK() { return Status(); } + + // Return error status of an appropriate type. + static Status OutOfMemory(const std::string &msg, int16_t posix_code = -1) { + return Status(StatusCode::OutOfMemory, msg, posix_code); + } + + static Status KeyError(const std::string &msg) { + return Status(StatusCode::KeyError, msg, -1); + } + + static Status TypeError(const std::string &msg) { + return Status(StatusCode::TypeError, msg, -1); + } + + static Status UnknownError(const std::string &msg) { + return Status(StatusCode::UnknownError, msg, -1); + } + + static Status NotImplemented(const std::string &msg) { + return Status(StatusCode::NotImplemented, msg, -1); + } + + static Status Invalid(const std::string &msg) { + return Status(StatusCode::Invalid, msg, -1); + } + + static Status IOError(const std::string &msg) { + return Status(StatusCode::IOError, msg, -1); + } + + static Status PlasmaObjectExists(const std::string &msg) { + return Status(StatusCode::PlasmaObjectExists, msg, -1); + } + + static Status PlasmaObjectNonexistent(const std::string &msg) { + return Status(StatusCode::PlasmaObjectNonexistent, msg, -1); + } + + static Status PlasmaStoreFull(const std::string &msg) { + return Status(StatusCode::PlasmaStoreFull, msg, -1); + } + + // Returns true iff the status indicates success. + bool ok() const { return (state_ == NULL); } + + bool IsOutOfMemory() const { return code() == StatusCode::OutOfMemory; } + bool IsKeyError() const { return code() == StatusCode::KeyError; } + bool IsInvalid() const { return code() == StatusCode::Invalid; } + bool IsIOError() const { return code() == StatusCode::IOError; } + bool IsTypeError() const { return code() == StatusCode::TypeError; } + bool IsUnknownError() const { return code() == StatusCode::UnknownError; } + bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; } + // An object with this object ID already exists in the plasma store. + bool IsPlasmaObjectExists() const { + return code() == StatusCode::PlasmaObjectExists; + } + // An object was requested that doesn't exist in the plasma store. + bool IsPlasmaObjectNonexistent() const { + return code() == StatusCode::PlasmaObjectNonexistent; + } + // An object is too large to fit into the plasma store. + bool IsPlasmaStoreFull() const { + return code() == StatusCode::PlasmaStoreFull; + } + + // Return a string representation of this status suitable for printing. + // Returns the string "OK" for success. + std::string ToString() const; + + // Return a string representation of the status code, without the message + // text or posix code information. + std::string CodeAsString() const; + + // Get the POSIX code associated with this Status, or -1 if there is none. + int16_t posix_code() const; + + StatusCode code() const { + return ((state_ == NULL) ? StatusCode::OK + : static_cast(state_[4])); + } + + std::string message() const { + uint32_t length; + memcpy(&length, state_, sizeof(length)); + std::string msg; + msg.append((state_ + 7), length); + return msg; + } + + private: + // OK status has a NULL state_. Otherwise, state_ is a new[] array + // of the following form: + // state_[0..3] == length of message + // state_[4] == code + // state_[5..6] == posix_code + // state_[7..] == message + const char *state_; + + Status(StatusCode code, const std::string &msg, int16_t posix_code); + static const char *CopyState(const char *s); +}; + +inline Status::Status(const Status &s) { + state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); +} + +inline void Status::operator=(const Status &s) { + // The following condition catches both aliasing (when this == &s), + // and the common case where both s and *this are ok. + if (state_ != s.state_) { + delete[] state_; + state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); + } +} + +} // namespace arrow + +#endif // ARROW_STATUS_H_ diff --git a/src/plasma/test/client_tests.cc b/src/plasma/test/client_tests.cc index dd939f248..f255734cb 100644 --- a/src/plasma/test/client_tests.cc +++ b/src/plasma/test/client_tests.cc @@ -4,6 +4,7 @@ #include #include +#include "plasma_common.h" #include "plasma.h" #include "plasma_protocol.h" #include "plasma_client.h" @@ -11,14 +12,17 @@ SUITE(plasma_client_tests); TEST plasma_status_tests(void) { - PlasmaConnection *plasma_conn1 = plasma_connect( - "/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY); - PlasmaConnection *plasma_conn2 = plasma_connect( - "/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY); - ObjectID oid1 = globally_unique_id(); + PlasmaClient client1; + ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1", + PLASMA_DEFAULT_RELEASE_DELAY)); + PlasmaClient client2; + ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2", + PLASMA_DEFAULT_RELEASE_DELAY)); + ObjectID oid1 = ObjectID::from_random(); /* Test for object non-existence. */ - int status = plasma_status(plasma_conn1, oid1); + int status; + ARROW_CHECK_OK(client1.Info(oid1, &status)); ASSERT(status == ObjectStatus_Nonexistent); /* Test for the object being in local Plasma store. */ @@ -27,36 +31,39 @@ TEST plasma_status_tests(void) { uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); uint8_t *data; - plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data); - plasma_seal(plasma_conn1, oid1); + ARROW_CHECK_OK( + client1.Create(oid1, data_size, metadata, metadata_size, &data)); + ARROW_CHECK_OK(client1.Seal(oid1)); /* Sleep to avoid race condition of Plasma Manager waiting for notification. */ sleep(1); - status = plasma_status(plasma_conn1, oid1); + ARROW_CHECK_OK(client1.Info(oid1, &status)); ASSERT(status == ObjectStatus_Local); /* Test for object being remote. */ - status = plasma_status(plasma_conn2, oid1); + ARROW_CHECK_OK(client2.Info(oid1, &status)); ASSERT(status == ObjectStatus_Remote); - plasma_disconnect(plasma_conn1); - plasma_disconnect(plasma_conn2); + ARROW_CHECK_OK(client1.Disconnect()); + ARROW_CHECK_OK(client2.Disconnect()); PASS(); } TEST plasma_fetch_tests(void) { - PlasmaConnection *plasma_conn1 = plasma_connect( - "/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY); - PlasmaConnection *plasma_conn2 = plasma_connect( - "/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY); - ObjectID oid1 = globally_unique_id(); + PlasmaClient client1; + ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1", + PLASMA_DEFAULT_RELEASE_DELAY)); + PlasmaClient client2; + ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2", + PLASMA_DEFAULT_RELEASE_DELAY)); + ObjectID oid1 = ObjectID::from_random(); /* Test for object non-existence. */ int status; /* No object in the system */ - status = plasma_status(plasma_conn1, oid1); + ARROW_CHECK_OK(client1.Info(oid1, &status)); ASSERT(status == ObjectStatus_Nonexistent); /* Test for the object being in local Plasma store. */ @@ -65,37 +72,38 @@ TEST plasma_fetch_tests(void) { uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); uint8_t *data; - plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data); - plasma_seal(plasma_conn1, oid1); + ARROW_CHECK_OK( + client1.Create(oid1, data_size, metadata, metadata_size, &data)); + ARROW_CHECK_OK(client1.Seal(oid1)); /* Object with ID oid1 has been just inserted. On the next fetch we might * either find the object or not, depending on whether the Plasma Manager has * received the notification from the Plasma Store or not. */ ObjectID oid_array1[1] = {oid1}; - plasma_fetch(plasma_conn1, 1, oid_array1); - status = plasma_status(plasma_conn1, oid1); + ARROW_CHECK_OK(client1.Fetch(1, oid_array1)); + ARROW_CHECK_OK(client1.Info(oid1, &status)); ASSERT((status == ObjectStatus_Local) || (status == ObjectStatus_Nonexistent)); /* Sleep to make sure Plasma Manager got the notification. */ sleep(1); - status = plasma_status(plasma_conn1, oid1); + ARROW_CHECK_OK(client1.Info(oid1, &status)); ASSERT(status == ObjectStatus_Local); /* Test for object being remote. */ - status = plasma_status(plasma_conn2, oid1); + ARROW_CHECK_OK(client2.Info(oid1, &status)); ASSERT(status == ObjectStatus_Remote); /* Sleep to make sure the object has been fetched and it is now stored in the * local Plasma Store. */ - plasma_fetch(plasma_conn2, 1, oid_array1); + ARROW_CHECK_OK(client2.Fetch(1, oid_array1)); sleep(1); - status = plasma_status(plasma_conn2, oid1); + ARROW_CHECK_OK(client2.Info(oid1, &status)); ASSERT(status == ObjectStatus_Local); sleep(1); - plasma_disconnect(plasma_conn1); - plasma_disconnect(plasma_conn2); + ARROW_CHECK_OK(client1.Disconnect()); + ARROW_CHECK_OK(client2.Disconnect()); PASS(); } @@ -116,14 +124,15 @@ bool is_equal_data_123(uint8_t *data1, uint8_t *data2, uint64_t size) { } TEST plasma_nonblocking_get_tests(void) { - PlasmaConnection *plasma_conn = plasma_connect("/tmp/store1", "/tmp/manager1", - PLASMA_DEFAULT_RELEASE_DELAY); - ObjectID oid = globally_unique_id(); + PlasmaClient client; + ARROW_CHECK_OK(client.Connect("/tmp/store1", "/tmp/manager1", + PLASMA_DEFAULT_RELEASE_DELAY)); + ObjectID oid = ObjectID::from_random(); ObjectID oid_array[1] = {oid}; ObjectBuffer obj_buffer; /* Test for object non-existence. */ - plasma_get(plasma_conn, oid_array, 1, 0, &obj_buffer); + ARROW_CHECK_OK(client.Get(oid_array, 1, 0, &obj_buffer)); ASSERT(obj_buffer.data_size == -1); /* Test for the object being in local Plasma store. */ @@ -132,27 +141,29 @@ TEST plasma_nonblocking_get_tests(void) { uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); uint8_t *data; - plasma_create(plasma_conn, oid, data_size, metadata, metadata_size, &data); + ARROW_CHECK_OK(client.Create(oid, data_size, metadata, metadata_size, &data)); init_data_123(data, data_size, 0); - plasma_seal(plasma_conn, oid); + ARROW_CHECK_OK(client.Seal(oid)); sleep(1); - plasma_get(plasma_conn, oid_array, 1, 0, &obj_buffer); + ARROW_CHECK_OK(client.Get(oid_array, 1, 0, &obj_buffer)); ASSERT(is_equal_data_123(data, obj_buffer.data, data_size) == true); sleep(1); - plasma_disconnect(plasma_conn); + ARROW_CHECK_OK(client.Disconnect()); PASS(); } TEST plasma_wait_for_objects_tests(void) { - PlasmaConnection *plasma_conn1 = plasma_connect( - "/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY); - PlasmaConnection *plasma_conn2 = plasma_connect( - "/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY); - ObjectID oid1 = globally_unique_id(); - ObjectID oid2 = globally_unique_id(); + PlasmaClient client1; + ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1", + PLASMA_DEFAULT_RELEASE_DELAY)); + PlasmaClient client2; + ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2", + PLASMA_DEFAULT_RELEASE_DELAY)); + ObjectID oid1 = ObjectID::from_random(); + ObjectID oid2 = ObjectID::from_random(); #define NUM_OBJ_REQUEST 2 #define WAIT_TIMEOUT_MS 1000 ObjectRequest obj_requests[NUM_OBJ_REQUEST]; @@ -164,8 +175,9 @@ TEST plasma_wait_for_objects_tests(void) { struct timeval start, end; gettimeofday(&start, NULL); - int n = plasma_wait(plasma_conn1, NUM_OBJ_REQUEST, obj_requests, - NUM_OBJ_REQUEST, WAIT_TIMEOUT_MS); + int n; + ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, + WAIT_TIMEOUT_MS, n)); ASSERT(n == 0); gettimeofday(&end, NULL); float diff_ms = (end.tv_sec - start.tv_sec); @@ -178,48 +190,51 @@ TEST plasma_wait_for_objects_tests(void) { uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); uint8_t *data; - plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data); - plasma_seal(plasma_conn1, oid1); + ARROW_CHECK_OK( + client1.Create(oid1, data_size, metadata, metadata_size, &data)); + ARROW_CHECK_OK(client1.Seal(oid1)); - n = plasma_wait(plasma_conn1, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS); + ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, + WAIT_TIMEOUT_MS, n)); ASSERT(n == 1); - /* Create and insert an object in plasma_conn2. */ - plasma_create(plasma_conn2, oid2, data_size, metadata, metadata_size, &data); - plasma_seal(plasma_conn2, oid2); + /* Create and insert an object in client2. */ + ARROW_CHECK_OK( + client2.Create(oid2, data_size, metadata, metadata_size, &data)); + ARROW_CHECK_OK(client2.Seal(oid2)); - n = plasma_wait(plasma_conn1, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS); + ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, + WAIT_TIMEOUT_MS, n)); ASSERT(n == 2); - n = plasma_wait(plasma_conn2, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS); + ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, + WAIT_TIMEOUT_MS, n)); ASSERT(n == 2); obj_requests[0].type = PLASMA_QUERY_LOCAL; obj_requests[1].type = PLASMA_QUERY_LOCAL; - n = plasma_wait(plasma_conn1, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS); + ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, + WAIT_TIMEOUT_MS, n)); ASSERT(n == 1); - n = plasma_wait(plasma_conn2, NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS); + ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, + WAIT_TIMEOUT_MS, n)); ASSERT(n == 1); - plasma_disconnect(plasma_conn1); - plasma_disconnect(plasma_conn2); + ARROW_CHECK_OK(client1.Disconnect()); + ARROW_CHECK_OK(client2.Disconnect()); PASS(); } TEST plasma_get_tests(void) { - PlasmaConnection *plasma_conn1 = plasma_connect( - "/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY); - PlasmaConnection *plasma_conn2 = plasma_connect( - "/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY); - ObjectID oid1 = globally_unique_id(); - ObjectID oid2 = globally_unique_id(); + PlasmaClient client1, client2; + ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1", + PLASMA_DEFAULT_RELEASE_DELAY)); + ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2", + PLASMA_DEFAULT_RELEASE_DELAY)); + ObjectID oid1 = ObjectID::from_random(); + ObjectID oid2 = ObjectID::from_random(); ObjectBuffer obj_buffer; ObjectID oid_array1[1] = {oid1}; @@ -229,35 +244,38 @@ TEST plasma_get_tests(void) { uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); uint8_t *data; - plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data); + ARROW_CHECK_OK( + client1.Create(oid1, data_size, metadata, metadata_size, &data)); init_data_123(data, data_size, 1); - plasma_seal(plasma_conn1, oid1); + ARROW_CHECK_OK(client1.Seal(oid1)); - plasma_get(plasma_conn1, oid_array1, 1, -1, &obj_buffer); + ARROW_CHECK_OK(client1.Get(oid_array1, 1, -1, &obj_buffer)); ASSERT(data[0] == obj_buffer.data[0]); - plasma_create(plasma_conn2, oid2, data_size, metadata, metadata_size, &data); + ARROW_CHECK_OK( + client2.Create(oid2, data_size, metadata, metadata_size, &data)); init_data_123(data, data_size, 2); - plasma_seal(plasma_conn2, oid2); + ARROW_CHECK_OK(client2.Seal(oid2)); - plasma_fetch(plasma_conn1, 1, oid_array2); - plasma_get(plasma_conn1, oid_array2, 1, -1, &obj_buffer); + ARROW_CHECK_OK(client1.Fetch(1, oid_array2)); + ARROW_CHECK_OK(client1.Get(oid_array2, 1, -1, &obj_buffer)); ASSERT(data[0] == obj_buffer.data[0]); sleep(1); - plasma_disconnect(plasma_conn1); - plasma_disconnect(plasma_conn2); + ARROW_CHECK_OK(client1.Disconnect()); + ARROW_CHECK_OK(client2.Disconnect()); PASS(); } TEST plasma_get_multiple_tests(void) { - PlasmaConnection *plasma_conn1 = plasma_connect( - "/tmp/store1", "/tmp/manager1", PLASMA_DEFAULT_RELEASE_DELAY); - PlasmaConnection *plasma_conn2 = plasma_connect( - "/tmp/store2", "/tmp/manager2", PLASMA_DEFAULT_RELEASE_DELAY); - ObjectID oid1 = globally_unique_id(); - ObjectID oid2 = globally_unique_id(); + PlasmaClient client1, client2; + ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1", + PLASMA_DEFAULT_RELEASE_DELAY)); + ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2", + PLASMA_DEFAULT_RELEASE_DELAY)); + ObjectID oid1 = ObjectID::from_random(); + ObjectID oid2 = ObjectID::from_random(); ObjectID obj_ids[NUM_OBJ_REQUEST]; ObjectBuffer obj_buffer[NUM_OBJ_REQUEST]; int obj1_first = 1, obj2_first = 2; @@ -269,26 +287,28 @@ TEST plasma_get_multiple_tests(void) { uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); uint8_t *data; - plasma_create(plasma_conn1, oid1, data_size, metadata, metadata_size, &data); + ARROW_CHECK_OK( + client1.Create(oid1, data_size, metadata, metadata_size, &data)); init_data_123(data, data_size, obj1_first); - plasma_seal(plasma_conn1, oid1); + ARROW_CHECK_OK(client1.Seal(oid1)); /* This only waits for oid1. */ - plasma_get(plasma_conn1, obj_ids, 1, -1, obj_buffer); + ARROW_CHECK_OK(client1.Get(obj_ids, 1, -1, obj_buffer)); ASSERT(data[0] == obj_buffer[0].data[0]); - plasma_create(plasma_conn2, oid2, data_size, metadata, metadata_size, &data); + ARROW_CHECK_OK( + client2.Create(oid2, data_size, metadata, metadata_size, &data)); init_data_123(data, data_size, obj2_first); - plasma_seal(plasma_conn2, oid2); + ARROW_CHECK_OK(client2.Seal(oid2)); - plasma_fetch(plasma_conn1, 2, obj_ids); - plasma_get(plasma_conn1, obj_ids, 2, -1, obj_buffer); + ARROW_CHECK_OK(client1.Fetch(2, obj_ids)); + ARROW_CHECK_OK(client1.Get(obj_ids, 2, -1, obj_buffer)); ASSERT(obj1_first == obj_buffer[0].data[0]); ASSERT(obj2_first == obj_buffer[1].data[0]); sleep(1); - plasma_disconnect(plasma_conn1); - plasma_disconnect(plasma_conn2); + ARROW_CHECK_OK(client1.Disconnect()); + ARROW_CHECK_OK(client2.Disconnect()); PASS(); } diff --git a/src/plasma/test/manager_tests.cc b/src/plasma/test/manager_tests.cc index 3514b8e17..de500c28b 100644 --- a/src/plasma/test/manager_tests.cc +++ b/src/plasma/test/manager_tests.cc @@ -53,7 +53,7 @@ typedef struct { ClientConnection *read_conn; /* Connect a new client to the local plasma manager and mock a request to an * object. */ - PlasmaConnection *plasma_conn; + PlasmaClient *plasma_client; ClientConnection *client_conn; } plasma_mock; @@ -85,8 +85,9 @@ plasma_mock *init_plasma_mock(plasma_mock *remote_mock) { } /* Connect a new client to the local plasma manager and mock a request to an * object. */ - mock->plasma_conn = plasma_connect(plasma_store_socket_name, - utstring_body(manager_socket_name), 0); + mock->plasma_client = new PlasmaClient(); + ARROW_CHECK_OK(mock->plasma_client->Connect( + plasma_store_socket_name, utstring_body(manager_socket_name), 0)); wait_for_pollin(mock->manager_local_fd); mock->client_conn = ClientConnection_listen( mock->loop, mock->manager_local_fd, mock->state, 0); @@ -96,7 +97,8 @@ plasma_mock *init_plasma_mock(plasma_mock *remote_mock) { void destroy_plasma_mock(plasma_mock *mock) { PlasmaManagerState_free(mock->state); - plasma_disconnect(mock->plasma_conn); + ARROW_CHECK_OK(mock->plasma_client->Disconnect()); + delete mock->plasma_client; close(mock->local_store); close(mock->manager_local_fd); close(mock->manager_remote_fd); @@ -127,17 +129,18 @@ TEST request_transfer_test(void) { local_mock->state); event_loop_run(local_mock->loop); int read_fd = get_client_sock(remote_mock->read_conn); - uint8_t *request_data = - plasma_receive(read_fd, MessageType_PlasmaDataRequest); + std::vector request_data; + ARROW_CHECK_OK( + PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, request_data)); ObjectID object_id2; char *address; int port; - plasma_read_DataRequest(request_data, &object_id2, &address, &port); + ARROW_CHECK_OK( + ReadDataRequest(request_data.data(), &object_id2, &address, &port)); ASSERT(ObjectID_equal(object_id, object_id2)); free(address); /* Clean up. */ utstring_free(addr); - free(request_data); destroy_plasma_mock(remote_mock); destroy_plasma_mock(local_mock); PASS(); @@ -180,18 +183,19 @@ TEST request_transfer_retry_test(void) { event_loop_run(local_mock->loop); int read_fd = get_client_sock(remote_mock2->read_conn); - uint8_t *request_data = - plasma_receive(read_fd, MessageType_PlasmaDataRequest); + std::vector request_data; + ARROW_CHECK_OK( + PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, request_data)); ObjectID object_id2; char *address; int port; - plasma_read_DataRequest(request_data, &object_id2, &address, &port); + ARROW_CHECK_OK( + ReadDataRequest(request_data.data(), &object_id2, &address, &port)); free(address); ASSERT(ObjectID_equal(object_id, object_id2)); /* Clean up. */ utstring_free(addr0); utstring_free(addr1); - free(request_data); destroy_plasma_mock(remote_mock2); destroy_plasma_mock(remote_mock1); destroy_plasma_mock(local_mock); diff --git a/src/plasma/test/serialization_tests.cc b/src/plasma/test/serialization_tests.cc index 59002bebd..b39094535 100644 --- a/src/plasma/test/serialization_tests.cc +++ b/src/plasma/test/serialization_tests.cc @@ -3,16 +3,13 @@ #include #include +#include "plasma_common.h" #include "plasma.h" +#include "plasma_io.h" #include "plasma_protocol.h" -#include "common.h" -#include "io.h" -#include "../plasma.h" SUITE(plasma_serialization_tests); -protocol_builder *g_B; - /** * Create a temporary file. Needs to be closed by the caller. * @@ -34,14 +31,13 @@ int create_temp_file(void) { * @return Pointer to the content of the message. Needs to be freed by the * caller. */ -uint8_t *read_message_from_file(int fd, int message_type) { +std::vector read_message_from_file(int fd, int message_type) { /* Go to the beginning of the file. */ lseek(fd, 0, SEEK_SET); int64_t type; - int64_t length; - uint8_t *data; - read_message(fd, &type, &length, &data); - CHECK(type == message_type); + std::vector data; + ARROW_CHECK_OK(ReadMessage(fd, &type, data)); + ARROW_CHECK(type == message_type); return data; } @@ -60,70 +56,68 @@ PlasmaObject random_plasma_object(void) { TEST plasma_create_request_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); + ObjectID object_id1 = ObjectID::from_random(); int64_t data_size1 = 42; int64_t metadata_size1 = 11; - plasma_send_CreateRequest(fd, g_B, object_id1, data_size1, metadata_size1); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaCreateRequest); + ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaCreateRequest); ObjectID object_id2; int64_t data_size2; int64_t metadata_size2; - plasma_read_CreateRequest(data, &object_id2, &data_size2, &metadata_size2); + ARROW_CHECK_OK(ReadCreateRequest(data.data(), &object_id2, &data_size2, + &metadata_size2)); ASSERT_EQ(data_size1, data_size2); ASSERT_EQ(metadata_size1, metadata_size2); - ASSERT(ObjectID_equal(object_id1, object_id2)); - free(data); + ASSERT(object_id1 == object_id2); close(fd); PASS(); } TEST plasma_create_reply_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); + ObjectID object_id1 = ObjectID::from_random(); PlasmaObject object1 = random_plasma_object(); - plasma_send_CreateReply(fd, g_B, object_id1, &object1, 0); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaCreateReply); + ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaCreateReply); ObjectID object_id2; PlasmaObject object2; memset(&object2, 0, sizeof(object2)); - int error_code; - plasma_read_CreateReply(data, &object_id2, &object2, &error_code); - ASSERT(ObjectID_equal(object_id1, object_id2)); + ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2)); + ASSERT(object_id1 == object_id2); ASSERT(memcmp(&object1, &object2, sizeof(object1)) == 0); - free(data); close(fd); PASS(); } TEST plasma_seal_request_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); - unsigned char digest1[DIGEST_SIZE]; - memset(&digest1[0], 7, DIGEST_SIZE); - plasma_send_SealRequest(fd, g_B, object_id1, &digest1[0]); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaSealRequest); + ObjectID object_id1 = ObjectID::from_random(); + unsigned char digest1[kDigestSize]; + memset(&digest1[0], 7, kDigestSize); + ARROW_CHECK_OK(SendSealRequest(fd, object_id1, &digest1[0])); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaSealRequest); ObjectID object_id2; - unsigned char digest2[DIGEST_SIZE]; - plasma_read_SealRequest(data, &object_id2, &digest2[0]); - ASSERT(ObjectID_equal(object_id1, object_id2)); - ASSERT(memcmp(&digest1[0], &digest2[0], DIGEST_SIZE) == 0); - free(data); + unsigned char digest2[kDigestSize]; + ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0])); + ASSERT(object_id1 == object_id2); + ASSERT(memcmp(&digest1[0], &digest2[0], kDigestSize) == 0); close(fd); PASS(); } TEST plasma_seal_reply_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); - int error1 = 5; - plasma_send_SealReply(fd, g_B, object_id1, error1); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaSealReply); + ObjectID object_id1 = ObjectID::from_random(); + ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaSealReply); ObjectID object_id2; - int error2; - plasma_read_SealReply(data, &object_id2, &error2); - ASSERT(ObjectID_equal(object_id1, object_id2)); - ASSERT(error1 == error2); - free(data); + Status s = ReadSealReply(data.data(), &object_id2); + ASSERT(object_id1 == object_id2); + ASSERT(s.IsPlasmaObjectExists()); close(fd); PASS(); } @@ -131,18 +125,19 @@ TEST plasma_seal_reply_test(void) { TEST plasma_get_request_test(void) { int fd = create_temp_file(); ObjectID object_ids[2]; - object_ids[0] = globally_unique_id(); - object_ids[1] = globally_unique_id(); + object_ids[0] = ObjectID::from_random(); + object_ids[1] = ObjectID::from_random(); int64_t timeout_ms = 1234; - plasma_send_GetRequest(fd, g_B, object_ids, 2, timeout_ms); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaGetRequest); - ObjectID object_ids_return[2]; + ARROW_CHECK_OK(SendGetRequest(fd, object_ids, 2, timeout_ms)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaGetRequest); + std::vector object_ids_return; int64_t timeout_ms_return; - plasma_read_GetRequest(data, &object_ids_return[0], &timeout_ms_return, 2); - ASSERT(ObjectID_equal(object_ids[0], object_ids_return[0])); - ASSERT(ObjectID_equal(object_ids[1], object_ids_return[1])); + ARROW_CHECK_OK( + ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return)); + ASSERT(object_ids[0] == object_ids_return[0]); + ASSERT(object_ids[1] == object_ids_return[1]); ASSERT(timeout_ms == timeout_ms_return); - free(data); close(fd); PASS(); } @@ -150,101 +145,97 @@ TEST plasma_get_request_test(void) { TEST plasma_get_reply_test(void) { int fd = create_temp_file(); ObjectID object_ids[2]; - object_ids[0] = globally_unique_id(); - object_ids[1] = globally_unique_id(); + object_ids[0] = ObjectID::from_random(); + object_ids[1] = ObjectID::from_random(); std::unordered_map plasma_objects; plasma_objects[object_ids[0]] = random_plasma_object(); plasma_objects[object_ids[1]] = random_plasma_object(); - plasma_send_GetReply(fd, g_B, object_ids, plasma_objects, 2); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaGetReply); - int64_t num_objects = plasma_read_GetRequest_num_objects(data); - ObjectID object_ids_return[num_objects]; + ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaGetReply); + ObjectID object_ids_return[2]; PlasmaObject plasma_objects_return[2]; memset(&plasma_objects_return, 0, sizeof(plasma_objects_return)); - plasma_read_GetReply(data, object_ids_return, &plasma_objects_return[0], - num_objects); - ASSERT(ObjectID_equal(object_ids[0], object_ids_return[0])); - ASSERT(ObjectID_equal(object_ids[1], object_ids_return[1])); + ARROW_CHECK_OK(ReadGetReply(data.data(), object_ids_return, + &plasma_objects_return[0], 2)); + ASSERT(object_ids[0] == object_ids_return[0]); + ASSERT(object_ids[1] == object_ids_return[1]); ASSERT(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0], sizeof(PlasmaObject)) == 0); ASSERT(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1], sizeof(PlasmaObject)) == 0); - free(data); close(fd); PASS(); } TEST plasma_release_request_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); - plasma_send_ReleaseRequest(fd, g_B, object_id1); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaReleaseRequest); + ObjectID object_id1 = ObjectID::from_random(); + ARROW_CHECK_OK(SendReleaseRequest(fd, object_id1)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaReleaseRequest); ObjectID object_id2; - plasma_read_ReleaseRequest(data, &object_id2); - ASSERT(ObjectID_equal(object_id1, object_id2)); - free(data); + ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2)); + ASSERT(object_id1 == object_id2); close(fd); PASS(); } TEST plasma_release_reply_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); - int error1 = 5; - plasma_send_ReleaseReply(fd, g_B, object_id1, error1); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaReleaseReply); + ObjectID object_id1 = ObjectID::from_random(); + ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaReleaseReply); ObjectID object_id2; - int error2; - plasma_read_ReleaseReply(data, &object_id2, &error2); - ASSERT(ObjectID_equal(object_id1, object_id2)); - ASSERT(error1 == error2); - free(data); + Status s = ReadReleaseReply(data.data(), &object_id2); + ASSERT(object_id1 == object_id2); + ASSERT(s.IsPlasmaObjectExists()); close(fd); PASS(); } TEST plasma_delete_request_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); - plasma_send_DeleteRequest(fd, g_B, object_id1); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaDeleteRequest); + ObjectID object_id1 = ObjectID::from_random(); + ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaDeleteRequest); ObjectID object_id2; - plasma_read_DeleteRequest(data, &object_id2); - ASSERT(ObjectID_equal(object_id1, object_id2)); - free(data); + ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2)); + ASSERT(object_id1 == object_id2); close(fd); PASS(); } TEST plasma_delete_reply_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); - int error1 = 5; - plasma_send_DeleteReply(fd, g_B, object_id1, error1); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaDeleteReply); + ObjectID object_id1 = ObjectID::from_random(); + int error1 = PlasmaError_ObjectExists; + ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaDeleteReply); ObjectID object_id2; - int error2; - plasma_read_DeleteReply(data, &object_id2, &error2); - ASSERT(ObjectID_equal(object_id1, object_id2)); - ASSERT(error1 == error2); - free(data); + Status s = ReadDeleteReply(data.data(), &object_id2); + ASSERT(object_id1 == object_id2); + ASSERT(s.IsPlasmaObjectExists()); close(fd); PASS(); } TEST plasma_status_request_test(void) { int fd = create_temp_file(); - ObjectID object_ids[2]; - object_ids[0] = globally_unique_id(); - object_ids[1] = globally_unique_id(); - plasma_send_StatusRequest(fd, g_B, object_ids, 2); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaStatusRequest); - int64_t num_objects = plasma_read_StatusRequest_num_objects(data); + int64_t num_objects = 2; + ObjectID object_ids[num_objects]; + object_ids[0] = ObjectID::from_random(); + object_ids[1] = ObjectID::from_random(); + ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaStatusRequest); ObjectID object_ids_read[num_objects]; - plasma_read_StatusRequest(data, object_ids_read, num_objects); - ASSERT(ObjectID_equal(object_ids[0], object_ids_read[0])); - ASSERT(ObjectID_equal(object_ids[1], object_ids_read[1])); - free(data); + ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects)); + ASSERT(object_ids[0] == object_ids_read[0]); + ASSERT(object_ids[1] == object_ids_read[1]); close(fd); PASS(); } @@ -252,21 +243,21 @@ TEST plasma_status_request_test(void) { TEST plasma_status_reply_test(void) { int fd = create_temp_file(); ObjectID object_ids[2]; - object_ids[0] = globally_unique_id(); - object_ids[1] = globally_unique_id(); + object_ids[0] = ObjectID::from_random(); + object_ids[1] = ObjectID::from_random(); int object_statuses[2] = {42, 43}; - plasma_send_StatusReply(fd, g_B, object_ids, object_statuses, 2); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaStatusReply); - int64_t num_objects = plasma_read_StatusReply_num_objects(data); + ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaStatusReply); + int64_t num_objects = ReadStatusReply_num_objects(data.data()); ObjectID object_ids_read[num_objects]; int object_statuses_read[num_objects]; - plasma_read_StatusReply(data, object_ids_read, object_statuses_read, - num_objects); - ASSERT(ObjectID_equal(object_ids[0], object_ids_read[0])); - ASSERT(ObjectID_equal(object_ids[1], object_ids_read[1])); + ARROW_CHECK_OK(ReadStatusReply(data.data(), object_ids_read, + object_statuses_read, num_objects)); + ASSERT(object_ids[0] == object_ids_read[0]); + ASSERT(object_ids[1] == object_ids_read[1]); ASSERT_EQ(object_statuses[0], object_statuses_read[0]); ASSERT_EQ(object_statuses[1], object_statuses_read[1]); - free(data); close(fd); PASS(); } @@ -274,12 +265,12 @@ TEST plasma_status_reply_test(void) { TEST plasma_evict_request_test(void) { int fd = create_temp_file(); int64_t num_bytes = 111; - plasma_send_EvictRequest(fd, g_B, num_bytes); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaEvictRequest); + ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaEvictRequest); int64_t num_bytes_received; - plasma_read_EvictRequest(data, &num_bytes_received); + ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received)); ASSERT_EQ(num_bytes, num_bytes_received); - free(data); close(fd); PASS(); } @@ -287,12 +278,12 @@ TEST plasma_evict_request_test(void) { TEST plasma_evict_reply_test(void) { int fd = create_temp_file(); int64_t num_bytes = 111; - plasma_send_EvictReply(fd, g_B, num_bytes); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaEvictReply); + ARROW_CHECK_OK(SendEvictReply(fd, num_bytes)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaEvictReply); int64_t num_bytes_received; - plasma_read_EvictReply(data, &num_bytes_received); + ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received)); ASSERT_EQ(num_bytes, num_bytes_received); - free(data); close(fd); PASS(); } @@ -300,15 +291,15 @@ TEST plasma_evict_reply_test(void) { TEST plasma_fetch_request_test(void) { int fd = create_temp_file(); ObjectID object_ids[2]; - object_ids[0] = globally_unique_id(); - object_ids[1] = globally_unique_id(); - plasma_send_FetchRequest(fd, g_B, object_ids, 2); - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaFetchRequest); - ObjectID object_ids_read[2]; - plasma_read_FetchRequest(data, &object_ids_read[0], 2); - ASSERT(ObjectID_equal(object_ids[0], object_ids_read[0])); - ASSERT(ObjectID_equal(object_ids[1], object_ids_read[1])); - free(data); + object_ids[0] = ObjectID::from_random(); + object_ids[1] = ObjectID::from_random(); + ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2)); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaFetchRequest); + std::vector object_ids_read; + ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read)); + ASSERT(object_ids[0] == object_ids_read[0]); + ASSERT(object_ids[1] == object_ids_read[1]); close(fd); PASS(); } @@ -317,22 +308,21 @@ TEST plasma_wait_request_test(void) { int fd = create_temp_file(); const int num_objects_in = 2; ObjectRequest object_requests_in[num_objects_in] = { - ObjectRequest({globally_unique_id(), PLASMA_QUERY_ANYWHERE, 0}), - ObjectRequest({globally_unique_id(), PLASMA_QUERY_LOCAL, 0})}; + ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_ANYWHERE, 0}), + ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_LOCAL, 0})}; const int num_ready_objects_in = 1; int64_t timeout_ms = 1000; - plasma_send_WaitRequest(fd, g_B, &object_requests_in[0], num_objects_in, - num_ready_objects_in, timeout_ms); + ARROW_CHECK_OK(SendWaitRequest(fd, &object_requests_in[0], num_objects_in, + num_ready_objects_in, timeout_ms)); /* Read message back. */ - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaWaitRequest); - int num_object_ids_out = plasma_read_WaitRequest_num_object_ids(data); - ASSERT_EQ(num_object_ids_out, num_objects_in); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaWaitRequest); int num_ready_objects_out; int64_t timeout_ms_read; ObjectRequestMap object_requests_out; - plasma_read_WaitRequest(data, object_requests_out, num_object_ids_out, - &timeout_ms_read, &num_ready_objects_out); + ARROW_CHECK_OK(ReadWaitRequest(data.data(), object_requests_out, + &timeout_ms_read, &num_ready_objects_out)); ASSERT_EQ(num_objects_in, object_requests_out.size()); ASSERT_EQ(num_ready_objects_out, num_ready_objects_in); for (int i = 0; i < num_objects_in; i++) { @@ -340,11 +330,9 @@ TEST plasma_wait_request_test(void) { ASSERT_EQ(1, object_requests_out.count(object_id)); const auto &entry = object_requests_out.find(object_id); ASSERT(entry != object_requests_out.end()); - ASSERT(ObjectID_equal(entry->second.object_id, - object_requests_in[i].object_id)); + ASSERT(entry->second.object_id == object_requests_in[i].object_id); ASSERT_EQ(entry->second.type, object_requests_in[i].type); } - free(data); close(fd); PASS(); } @@ -354,68 +342,69 @@ TEST plasma_wait_reply_test(void) { const int num_objects_in = 2; /* Create a map with two ObjectRequests in it. */ ObjectRequestMap objects_in(num_objects_in); - ObjectID id1 = globally_unique_id(); + ObjectID id1 = ObjectID::from_random(); objects_in[id1] = ObjectRequest({id1, 0, ObjectStatus_Local}); - ObjectID id2 = globally_unique_id(); + ObjectID id2 = ObjectID::from_random(); objects_in[id2] = ObjectRequest({id2, 0, ObjectStatus_Nonexistent}); - plasma_send_WaitReply(fd, g_B, objects_in, num_objects_in); + ARROW_CHECK_OK(SendWaitReply(fd, objects_in, num_objects_in)); /* Read message back. */ - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaWaitReply); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaWaitReply); ObjectRequest objects_out[2]; int num_objects_out; - plasma_read_WaitReply(data, &objects_out[0], &num_objects_out); + ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out)); ASSERT(num_objects_in == num_objects_out); for (int i = 0; i < num_objects_out; i++) { /* Each object request must appear exactly once. */ ASSERT(1 == objects_in.count(objects_out[i].object_id)); const auto &entry = objects_in.find(objects_out[i].object_id); ASSERT(entry != objects_in.end()); - ASSERT(ObjectID_equal(entry->second.object_id, objects_out[i].object_id)); + ASSERT(entry->second.object_id == objects_out[i].object_id); ASSERT(entry->second.status == objects_out[i].status); } - free(data); close(fd); PASS(); } TEST plasma_data_request_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); + ObjectID object_id1 = ObjectID::from_random(); const char *address1 = "address1"; int port1 = 12345; - plasma_send_DataRequest(fd, g_B, object_id1, address1, port1); + ARROW_CHECK_OK(SendDataRequest(fd, object_id1, address1, port1)); /* Reading message back. */ - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaDataRequest); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaDataRequest); ObjectID object_id2; char *address2; int port2; - plasma_read_DataRequest(data, &object_id2, &address2, &port2); - ASSERT(ObjectID_equal(object_id1, object_id2)); + ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2)); + ASSERT(object_id1 == object_id2); ASSERT(strcmp(address1, address2) == 0); ASSERT(port1 == port2); free(address2); - free(data); close(fd); PASS(); } TEST plasma_data_reply_test(void) { int fd = create_temp_file(); - ObjectID object_id1 = globally_unique_id(); + ObjectID object_id1 = ObjectID::from_random(); int64_t object_size1 = 146; int64_t metadata_size1 = 198; - plasma_send_DataReply(fd, g_B, object_id1, object_size1, metadata_size1); + ARROW_CHECK_OK(SendDataReply(fd, object_id1, object_size1, metadata_size1)); /* Reading message back. */ - uint8_t *data = read_message_from_file(fd, MessageType_PlasmaDataReply); + std::vector data = + read_message_from_file(fd, MessageType_PlasmaDataReply); ObjectID object_id2; int64_t object_size2; int64_t metadata_size2; - plasma_read_DataReply(data, &object_id2, &object_size2, &metadata_size2); - ASSERT(ObjectID_equal(object_id1, object_id2)); + ARROW_CHECK_OK( + ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2)); + ASSERT(object_id1 == object_id2); ASSERT(object_size1 == object_size2); ASSERT(metadata_size1 == metadata_size2); - free(data); PASS(); } @@ -444,9 +433,7 @@ SUITE(plasma_serialization_tests) { GREATEST_MAIN_DEFS(); int main(int argc, char **argv) { - g_B = make_protocol_builder(); GREATEST_MAIN_BEGIN(); RUN_SUITE(plasma_serialization_tests); GREATEST_MAIN_END(); - free_protocol_builder(g_B); } diff --git a/src/plasma/thirdparty/ae/ae.c b/src/plasma/thirdparty/ae/ae.c new file mode 100644 index 000000000..e66808a81 --- /dev/null +++ b/src/plasma/thirdparty/ae/ae.c @@ -0,0 +1,465 @@ +/* A simple event-driven programming library. Originally I wrote this code + * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated + * it in form of a library for easy reuse. + * + * Copyright (c) 2006-2010, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ae.h" +#include "zmalloc.h" +#include "config.h" + +/* Include the best multiplexing layer supported by this system. + * The following should be ordered by performances, descending. */ +#ifdef HAVE_EVPORT +#include "ae_evport.c" +#else + #ifdef HAVE_EPOLL + #include "ae_epoll.c" + #else + #ifdef HAVE_KQUEUE + #include "ae_kqueue.c" + #else + #include "ae_select.c" + #endif + #endif +#endif + +aeEventLoop *aeCreateEventLoop(int setsize) { + aeEventLoop *eventLoop; + int i; + + if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err; + eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize); + eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize); + if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err; + eventLoop->setsize = setsize; + eventLoop->lastTime = time(NULL); + eventLoop->timeEventHead = NULL; + eventLoop->timeEventNextId = 0; + eventLoop->stop = 0; + eventLoop->maxfd = -1; + eventLoop->beforesleep = NULL; + if (aeApiCreate(eventLoop) == -1) goto err; + /* Events with mask == AE_NONE are not set. So let's initialize the + * vector with it. */ + for (i = 0; i < setsize; i++) + eventLoop->events[i].mask = AE_NONE; + return eventLoop; + +err: + if (eventLoop) { + zfree(eventLoop->events); + zfree(eventLoop->fired); + zfree(eventLoop); + } + return NULL; +} + +/* Return the current set size. */ +int aeGetSetSize(aeEventLoop *eventLoop) { + return eventLoop->setsize; +} + +/* Resize the maximum set size of the event loop. + * If the requested set size is smaller than the current set size, but + * there is already a file descriptor in use that is >= the requested + * set size minus one, AE_ERR is returned and the operation is not + * performed at all. + * + * Otherwise AE_OK is returned and the operation is successful. */ +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { + int i; + + if (setsize == eventLoop->setsize) return AE_OK; + if (eventLoop->maxfd >= setsize) return AE_ERR; + if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR; + + eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize); + eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize); + eventLoop->setsize = setsize; + + /* Make sure that if we created new slots, they are initialized with + * an AE_NONE mask. */ + for (i = eventLoop->maxfd+1; i < setsize; i++) + eventLoop->events[i].mask = AE_NONE; + return AE_OK; +} + +void aeDeleteEventLoop(aeEventLoop *eventLoop) { + aeApiFree(eventLoop); + zfree(eventLoop->events); + zfree(eventLoop->fired); + zfree(eventLoop); +} + +void aeStop(aeEventLoop *eventLoop) { + eventLoop->stop = 1; +} + +int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, + aeFileProc *proc, void *clientData) +{ + if (fd >= eventLoop->setsize) { + errno = ERANGE; + return AE_ERR; + } + aeFileEvent *fe = &eventLoop->events[fd]; + + if (aeApiAddEvent(eventLoop, fd, mask) == -1) + return AE_ERR; + fe->mask |= mask; + if (mask & AE_READABLE) fe->rfileProc = proc; + if (mask & AE_WRITABLE) fe->wfileProc = proc; + fe->clientData = clientData; + if (fd > eventLoop->maxfd) + eventLoop->maxfd = fd; + return AE_OK; +} + +void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) +{ + if (fd >= eventLoop->setsize) return; + aeFileEvent *fe = &eventLoop->events[fd]; + if (fe->mask == AE_NONE) return; + + aeApiDelEvent(eventLoop, fd, mask); + fe->mask = fe->mask & (~mask); + if (fd == eventLoop->maxfd && fe->mask == AE_NONE) { + /* Update the max fd */ + int j; + + for (j = eventLoop->maxfd-1; j >= 0; j--) + if (eventLoop->events[j].mask != AE_NONE) break; + eventLoop->maxfd = j; + } +} + +int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { + if (fd >= eventLoop->setsize) return 0; + aeFileEvent *fe = &eventLoop->events[fd]; + + return fe->mask; +} + +static void aeGetTime(long *seconds, long *milliseconds) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + *seconds = tv.tv_sec; + *milliseconds = tv.tv_usec/1000; +} + +static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) { + long cur_sec, cur_ms, when_sec, when_ms; + + aeGetTime(&cur_sec, &cur_ms); + when_sec = cur_sec + milliseconds/1000; + when_ms = cur_ms + milliseconds%1000; + if (when_ms >= 1000) { + when_sec ++; + when_ms -= 1000; + } + *sec = when_sec; + *ms = when_ms; +} + +long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, + aeTimeProc *proc, void *clientData, + aeEventFinalizerProc *finalizerProc) +{ + long long id = eventLoop->timeEventNextId++; + aeTimeEvent *te; + + te = zmalloc(sizeof(*te)); + if (te == NULL) return AE_ERR; + te->id = id; + aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms); + te->timeProc = proc; + te->finalizerProc = finalizerProc; + te->clientData = clientData; + te->next = eventLoop->timeEventHead; + eventLoop->timeEventHead = te; + return id; +} + +int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) +{ + aeTimeEvent *te = eventLoop->timeEventHead; + while(te) { + if (te->id == id) { + te->id = AE_DELETED_EVENT_ID; + return AE_OK; + } + te = te->next; + } + return AE_ERR; /* NO event with the specified ID found */ +} + +/* Search the first timer to fire. + * This operation is useful to know how many time the select can be + * put in sleep without to delay any event. + * If there are no timers NULL is returned. + * + * Note that's O(N) since time events are unsorted. + * Possible optimizations (not needed by Redis so far, but...): + * 1) Insert the event in order, so that the nearest is just the head. + * Much better but still insertion or deletion of timers is O(N). + * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)). + */ +static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop) +{ + aeTimeEvent *te = eventLoop->timeEventHead; + aeTimeEvent *nearest = NULL; + + while(te) { + if (!nearest || te->when_sec < nearest->when_sec || + (te->when_sec == nearest->when_sec && + te->when_ms < nearest->when_ms)) + nearest = te; + te = te->next; + } + return nearest; +} + +/* Process time events */ +static int processTimeEvents(aeEventLoop *eventLoop) { + int processed = 0; + aeTimeEvent *te, *prev; + long long maxId; + time_t now = time(NULL); + + /* If the system clock is moved to the future, and then set back to the + * right value, time events may be delayed in a random way. Often this + * means that scheduled operations will not be performed soon enough. + * + * Here we try to detect system clock skews, and force all the time + * events to be processed ASAP when this happens: the idea is that + * processing events earlier is less dangerous than delaying them + * indefinitely, and practice suggests it is. */ + if (now < eventLoop->lastTime) { + te = eventLoop->timeEventHead; + while(te) { + te->when_sec = 0; + te = te->next; + } + } + eventLoop->lastTime = now; + + prev = NULL; + te = eventLoop->timeEventHead; + maxId = eventLoop->timeEventNextId-1; + while(te) { + long now_sec, now_ms; + long long id; + + /* Remove events scheduled for deletion. */ + if (te->id == AE_DELETED_EVENT_ID) { + aeTimeEvent *next = te->next; + if (prev == NULL) + eventLoop->timeEventHead = te->next; + else + prev->next = te->next; + if (te->finalizerProc) + te->finalizerProc(eventLoop, te->clientData); + zfree(te); + te = next; + continue; + } + + /* Make sure we don't process time events created by time events in + * this iteration. Note that this check is currently useless: we always + * add new timers on the head, however if we change the implementation + * detail, this check may be useful again: we keep it here for future + * defense. */ + if (te->id > maxId) { + te = te->next; + continue; + } + aeGetTime(&now_sec, &now_ms); + if (now_sec > te->when_sec || + (now_sec == te->when_sec && now_ms >= te->when_ms)) + { + int retval; + + id = te->id; + retval = te->timeProc(eventLoop, id, te->clientData); + processed++; + if (retval != AE_NOMORE) { + aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms); + } else { + te->id = AE_DELETED_EVENT_ID; + } + } + prev = te; + te = te->next; + } + return processed; +} + +/* Process every pending time event, then every pending file event + * (that may be registered by time event callbacks just processed). + * Without special flags the function sleeps until some file event + * fires, or when the next time event occurs (if any). + * + * If flags is 0, the function does nothing and returns. + * if flags has AE_ALL_EVENTS set, all the kind of events are processed. + * if flags has AE_FILE_EVENTS set, file events are processed. + * if flags has AE_TIME_EVENTS set, time events are processed. + * if flags has AE_DONT_WAIT set the function returns ASAP until all + * the events that's possible to process without to wait are processed. + * + * The function returns the number of events processed. */ +int aeProcessEvents(aeEventLoop *eventLoop, int flags) +{ + int processed = 0, numevents; + + /* Nothing to do? return ASAP */ + if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0; + + /* Note that we want call select() even if there are no + * file events to process as long as we want to process time + * events, in order to sleep until the next time event is ready + * to fire. */ + if (eventLoop->maxfd != -1 || + ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) { + int j; + aeTimeEvent *shortest = NULL; + struct timeval tv, *tvp; + + if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT)) + shortest = aeSearchNearestTimer(eventLoop); + if (shortest) { + long now_sec, now_ms; + + aeGetTime(&now_sec, &now_ms); + tvp = &tv; + + /* How many milliseconds we need to wait for the next + * time event to fire? */ + long long ms = + (shortest->when_sec - now_sec)*1000 + + shortest->when_ms - now_ms; + + if (ms > 0) { + tvp->tv_sec = ms/1000; + tvp->tv_usec = (ms % 1000)*1000; + } else { + tvp->tv_sec = 0; + tvp->tv_usec = 0; + } + } else { + /* If we have to check for events but need to return + * ASAP because of AE_DONT_WAIT we need to set the timeout + * to zero */ + if (flags & AE_DONT_WAIT) { + tv.tv_sec = tv.tv_usec = 0; + tvp = &tv; + } else { + /* Otherwise we can block */ + tvp = NULL; /* wait forever */ + } + } + + numevents = aeApiPoll(eventLoop, tvp); + for (j = 0; j < numevents; j++) { + aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd]; + int mask = eventLoop->fired[j].mask; + int fd = eventLoop->fired[j].fd; + int rfired = 0; + + /* note the fe->mask & mask & ... code: maybe an already processed + * event removed an element that fired and we still didn't + * processed, so we check if the event is still valid. */ + if (fe->mask & mask & AE_READABLE) { + rfired = 1; + fe->rfileProc(eventLoop,fd,fe->clientData,mask); + } + if (fe->mask & mask & AE_WRITABLE) { + if (!rfired || fe->wfileProc != fe->rfileProc) + fe->wfileProc(eventLoop,fd,fe->clientData,mask); + } + processed++; + } + } + /* Check time events */ + if (flags & AE_TIME_EVENTS) + processed += processTimeEvents(eventLoop); + + return processed; /* return the number of processed file/time events */ +} + +/* Wait for milliseconds until the given file descriptor becomes + * writable/readable/exception */ +int aeWait(int fd, int mask, long long milliseconds) { + struct pollfd pfd; + int retmask = 0, retval; + + memset(&pfd, 0, sizeof(pfd)); + pfd.fd = fd; + if (mask & AE_READABLE) pfd.events |= POLLIN; + if (mask & AE_WRITABLE) pfd.events |= POLLOUT; + + if ((retval = poll(&pfd, 1, milliseconds))== 1) { + if (pfd.revents & POLLIN) retmask |= AE_READABLE; + if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE; + if (pfd.revents & POLLERR) retmask |= AE_WRITABLE; + if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE; + return retmask; + } else { + return retval; + } +} + +void aeMain(aeEventLoop *eventLoop) { + eventLoop->stop = 0; + while (!eventLoop->stop) { + if (eventLoop->beforesleep != NULL) + eventLoop->beforesleep(eventLoop); + aeProcessEvents(eventLoop, AE_ALL_EVENTS); + } +} + +char *aeGetApiName(void) { + return aeApiName(); +} + +void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) { + eventLoop->beforesleep = beforesleep; +} diff --git a/src/plasma/thirdparty/ae/ae.h b/src/plasma/thirdparty/ae/ae.h new file mode 100644 index 000000000..827c4c9e4 --- /dev/null +++ b/src/plasma/thirdparty/ae/ae.h @@ -0,0 +1,123 @@ +/* A simple event-driven programming library. Originally I wrote this code + * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated + * it in form of a library for easy reuse. + * + * Copyright (c) 2006-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __AE_H__ +#define __AE_H__ + +#include + +#define AE_OK 0 +#define AE_ERR -1 + +#define AE_NONE 0 +#define AE_READABLE 1 +#define AE_WRITABLE 2 + +#define AE_FILE_EVENTS 1 +#define AE_TIME_EVENTS 2 +#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS) +#define AE_DONT_WAIT 4 + +#define AE_NOMORE -1 +#define AE_DELETED_EVENT_ID -1 + +/* Macros */ +#define AE_NOTUSED(V) ((void) V) + +struct aeEventLoop; + +/* Types and data structures */ +typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask); +typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData); +typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData); +typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop); + +/* File event structure */ +typedef struct aeFileEvent { + int mask; /* one of AE_(READABLE|WRITABLE) */ + aeFileProc *rfileProc; + aeFileProc *wfileProc; + void *clientData; +} aeFileEvent; + +/* Time event structure */ +typedef struct aeTimeEvent { + long long id; /* time event identifier. */ + long when_sec; /* seconds */ + long when_ms; /* milliseconds */ + aeTimeProc *timeProc; + aeEventFinalizerProc *finalizerProc; + void *clientData; + struct aeTimeEvent *next; +} aeTimeEvent; + +/* A fired event */ +typedef struct aeFiredEvent { + int fd; + int mask; +} aeFiredEvent; + +/* State of an event based program */ +typedef struct aeEventLoop { + int maxfd; /* highest file descriptor currently registered */ + int setsize; /* max number of file descriptors tracked */ + long long timeEventNextId; + time_t lastTime; /* Used to detect system clock skew */ + aeFileEvent *events; /* Registered events */ + aeFiredEvent *fired; /* Fired events */ + aeTimeEvent *timeEventHead; + int stop; + void *apidata; /* This is used for polling API specific data */ + aeBeforeSleepProc *beforesleep; +} aeEventLoop; + +/* Prototypes */ +aeEventLoop *aeCreateEventLoop(int setsize); +void aeDeleteEventLoop(aeEventLoop *eventLoop); +void aeStop(aeEventLoop *eventLoop); +int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, + aeFileProc *proc, void *clientData); +void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask); +int aeGetFileEvents(aeEventLoop *eventLoop, int fd); +long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, + aeTimeProc *proc, void *clientData, + aeEventFinalizerProc *finalizerProc); +int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id); +int aeProcessEvents(aeEventLoop *eventLoop, int flags); +int aeWait(int fd, int mask, long long milliseconds); +void aeMain(aeEventLoop *eventLoop); +char *aeGetApiName(void); +void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep); +int aeGetSetSize(aeEventLoop *eventLoop); +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); + +#endif diff --git a/src/plasma/thirdparty/ae/ae_epoll.c b/src/plasma/thirdparty/ae/ae_epoll.c new file mode 100644 index 000000000..410aac70d --- /dev/null +++ b/src/plasma/thirdparty/ae/ae_epoll.c @@ -0,0 +1,135 @@ +/* Linux epoll(2) based ae.c module + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include + +typedef struct aeApiState { + int epfd; + struct epoll_event *events; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize); + if (!state->events) { + zfree(state); + return -1; + } + state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */ + if (state->epfd == -1) { + zfree(state->events); + zfree(state); + return -1; + } + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize); + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->epfd); + zfree(state->events); + zfree(state); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct epoll_event ee = {0}; /* avoid valgrind warning */ + /* If the fd was already monitored for some event, we need a MOD + * operation. Otherwise we need an ADD operation. */ + int op = eventLoop->events[fd].mask == AE_NONE ? + EPOLL_CTL_ADD : EPOLL_CTL_MOD; + + ee.events = 0; + mask |= eventLoop->events[fd].mask; /* Merge old events */ + if (mask & AE_READABLE) ee.events |= EPOLLIN; + if (mask & AE_WRITABLE) ee.events |= EPOLLOUT; + ee.data.fd = fd; + if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1; + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) { + aeApiState *state = eventLoop->apidata; + struct epoll_event ee = {0}; /* avoid valgrind warning */ + int mask = eventLoop->events[fd].mask & (~delmask); + + ee.events = 0; + if (mask & AE_READABLE) ee.events |= EPOLLIN; + if (mask & AE_WRITABLE) ee.events |= EPOLLOUT; + ee.data.fd = fd; + if (mask != AE_NONE) { + epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee); + } else { + /* Note, Kernel < 2.6.9 requires a non null event pointer even for + * EPOLL_CTL_DEL. */ + epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee); + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, numevents = 0; + + retval = epoll_wait(state->epfd,state->events,eventLoop->setsize, + tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1); + if (retval > 0) { + int j; + + numevents = retval; + for (j = 0; j < numevents; j++) { + int mask = 0; + struct epoll_event *e = state->events+j; + + if (e->events & EPOLLIN) mask |= AE_READABLE; + if (e->events & EPOLLOUT) mask |= AE_WRITABLE; + if (e->events & EPOLLERR) mask |= AE_WRITABLE; + if (e->events & EPOLLHUP) mask |= AE_WRITABLE; + eventLoop->fired[j].fd = e->data.fd; + eventLoop->fired[j].mask = mask; + } + } + return numevents; +} + +static char *aeApiName(void) { + return "epoll"; +} diff --git a/src/plasma/thirdparty/ae/ae_evport.c b/src/plasma/thirdparty/ae/ae_evport.c new file mode 100644 index 000000000..5c317becb --- /dev/null +++ b/src/plasma/thirdparty/ae/ae_evport.c @@ -0,0 +1,320 @@ +/* ae.c module for illumos event ports. + * + * Copyright (c) 2012, Joyent, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include +#include +#include + +#include +#include + +#include + +static int evport_debug = 0; + +/* + * This file implements the ae API using event ports, present on Solaris-based + * systems since Solaris 10. Using the event port interface, we associate file + * descriptors with the port. Each association also includes the set of poll(2) + * events that the consumer is interested in (e.g., POLLIN and POLLOUT). + * + * There's one tricky piece to this implementation: when we return events via + * aeApiPoll, the corresponding file descriptors become dissociated from the + * port. This is necessary because poll events are level-triggered, so if the + * fd didn't become dissociated, it would immediately fire another event since + * the underlying state hasn't changed yet. We must re-associate the file + * descriptor, but only after we know that our caller has actually read from it. + * The ae API does not tell us exactly when that happens, but we do know that + * it must happen by the time aeApiPoll is called again. Our solution is to + * keep track of the last fds returned by aeApiPoll and re-associate them next + * time aeApiPoll is invoked. + * + * To summarize, in this module, each fd association is EITHER (a) represented + * only via the in-kernel association OR (b) represented by pending_fds and + * pending_masks. (b) is only true for the last fds we returned from aeApiPoll, + * and only until we enter aeApiPoll again (at which point we restore the + * in-kernel association). + */ +#define MAX_EVENT_BATCHSZ 512 + +typedef struct aeApiState { + int portfd; /* event port */ + int npending; /* # of pending fds */ + int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */ + int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */ +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + int i; + aeApiState *state = zmalloc(sizeof(aeApiState)); + if (!state) return -1; + + state->portfd = port_create(); + if (state->portfd == -1) { + zfree(state); + return -1; + } + + state->npending = 0; + + for (i = 0; i < MAX_EVENT_BATCHSZ; i++) { + state->pending_fds[i] = -1; + state->pending_masks[i] = AE_NONE; + } + + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Nothing to resize here. */ + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->portfd); + zfree(state); +} + +static int aeApiLookupPending(aeApiState *state, int fd) { + int i; + + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == fd) + return (i); + } + + return (-1); +} + +/* + * Helper function to invoke port_associate for the given fd and mask. + */ +static int aeApiAssociate(const char *where, int portfd, int fd, int mask) { + int events = 0; + int rv, err; + + if (mask & AE_READABLE) + events |= POLLIN; + if (mask & AE_WRITABLE) + events |= POLLOUT; + + if (evport_debug) + fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events); + + rv = port_associate(portfd, PORT_SOURCE_FD, fd, events, + (void *)(uintptr_t)mask); + err = errno; + + if (evport_debug) + fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err)); + + if (rv == -1) { + fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err)); + + if (err == EAGAIN) + fprintf(stderr, "aeApiAssociate: event port limit exceeded."); + } + + return rv; +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask, pfd; + + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask); + + /* + * Since port_associate's "events" argument replaces any existing events, we + * must be sure to include whatever events are already associated when + * we call port_associate() again. + */ + fullmask = mask | eventLoop->events[fd].mask; + pfd = aeApiLookupPending(state, fd); + + if (pfd != -1) { + /* + * This fd was recently returned from aeApiPoll. It should be safe to + * assume that the consumer has processed that poll event, but we play + * it safer by simply updating pending_mask. The fd will be + * re-associated as usual when aeApiPoll is called again. + */ + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd); + state->pending_masks[pfd] |= fullmask; + return 0; + } + + return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask)); +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask, pfd; + + if (evport_debug) + fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask); + + pfd = aeApiLookupPending(state, fd); + + if (pfd != -1) { + if (evport_debug) + fprintf(stderr, "deleting event from pending fd %d\n", fd); + + /* + * This fd was just returned from aeApiPoll, so it's not currently + * associated with the port. All we need to do is update + * pending_mask appropriately. + */ + state->pending_masks[pfd] &= ~mask; + + if (state->pending_masks[pfd] == AE_NONE) + state->pending_fds[pfd] = -1; + + return; + } + + /* + * The fd is currently associated with the port. Like with the add case + * above, we must look at the full mask for the file descriptor before + * updating that association. We don't have a good way of knowing what the + * events are without looking into the eventLoop state directly. We rely on + * the fact that our caller has already updated the mask in the eventLoop. + */ + + fullmask = eventLoop->events[fd].mask; + if (fullmask == AE_NONE) { + /* + * We're removing *all* events, so use port_dissociate to remove the + * association completely. Failure here indicates a bug. + */ + if (evport_debug) + fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd); + + if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) { + perror("aeApiDelEvent: port_dissociate"); + abort(); /* will not return */ + } + } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd, + fullmask) != 0) { + /* + * ENOMEM is a potentially transient condition, but the kernel won't + * generally return it unless things are really bad. EAGAIN indicates + * we've reached an resource limit, for which it doesn't make sense to + * retry (counter-intuitively). All other errors indicate a bug. In any + * of these cases, the best we can do is to abort. + */ + abort(); /* will not return */ + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + struct timespec timeout, *tsp; + int mask, i; + uint_t nevents; + port_event_t event[MAX_EVENT_BATCHSZ]; + + /* + * If we've returned fd events before, we must re-associate them with the + * port now, before calling port_get(). See the block comment at the top of + * this file for an explanation of why. + */ + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == -1) + /* This fd has since been deleted. */ + continue; + + if (aeApiAssociate("aeApiPoll", state->portfd, + state->pending_fds[i], state->pending_masks[i]) != 0) { + /* See aeApiDelEvent for why this case is fatal. */ + abort(); + } + + state->pending_masks[i] = AE_NONE; + state->pending_fds[i] = -1; + } + + state->npending = 0; + + if (tvp != NULL) { + timeout.tv_sec = tvp->tv_sec; + timeout.tv_nsec = tvp->tv_usec * 1000; + tsp = &timeout; + } else { + tsp = NULL; + } + + /* + * port_getn can return with errno == ETIME having returned some events (!). + * So if we get ETIME, we check nevents, too. + */ + nevents = 1; + if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents, + tsp) == -1 && (errno != ETIME || nevents == 0)) { + if (errno == ETIME || errno == EINTR) + return 0; + + /* Any other error indicates a bug. */ + perror("aeApiPoll: port_get"); + abort(); + } + + state->npending = nevents; + + for (i = 0; i < nevents; i++) { + mask = 0; + if (event[i].portev_events & POLLIN) + mask |= AE_READABLE; + if (event[i].portev_events & POLLOUT) + mask |= AE_WRITABLE; + + eventLoop->fired[i].fd = event[i].portev_object; + eventLoop->fired[i].mask = mask; + + if (evport_debug) + fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n", + (int)event[i].portev_object, mask); + + state->pending_fds[i] = event[i].portev_object; + state->pending_masks[i] = (uintptr_t)event[i].portev_user; + } + + return nevents; +} + +static char *aeApiName(void) { + return "evport"; +} diff --git a/src/plasma/thirdparty/ae/ae_kqueue.c b/src/plasma/thirdparty/ae/ae_kqueue.c new file mode 100644 index 000000000..6796f4ceb --- /dev/null +++ b/src/plasma/thirdparty/ae/ae_kqueue.c @@ -0,0 +1,138 @@ +/* Kqueue(2)-based ae.c module + * + * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include +#include + +typedef struct aeApiState { + int kqfd; + struct kevent *events; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize); + if (!state->events) { + zfree(state); + return -1; + } + state->kqfd = kqueue(); + if (state->kqfd == -1) { + zfree(state->events); + zfree(state); + return -1; + } + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct kevent)*setsize); + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->kqfd); + zfree(state->events); + zfree(state); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct kevent ke; + + if (mask & AE_READABLE) { + EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL); + if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; + } + if (mask & AE_WRITABLE) { + EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL); + if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; + } + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct kevent ke; + + if (mask & AE_READABLE) { + EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); + kevent(state->kqfd, &ke, 1, NULL, 0, NULL); + } + if (mask & AE_WRITABLE) { + EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL); + kevent(state->kqfd, &ke, 1, NULL, 0, NULL); + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, numevents = 0; + + if (tvp != NULL) { + struct timespec timeout; + timeout.tv_sec = tvp->tv_sec; + timeout.tv_nsec = tvp->tv_usec * 1000; + retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize, + &timeout); + } else { + retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize, + NULL); + } + + if (retval > 0) { + int j; + + numevents = retval; + for(j = 0; j < numevents; j++) { + int mask = 0; + struct kevent *e = state->events+j; + + if (e->filter == EVFILT_READ) mask |= AE_READABLE; + if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE; + eventLoop->fired[j].fd = e->ident; + eventLoop->fired[j].mask = mask; + } + } + return numevents; +} + +static char *aeApiName(void) { + return "kqueue"; +} diff --git a/src/plasma/thirdparty/ae/ae_select.c b/src/plasma/thirdparty/ae/ae_select.c new file mode 100644 index 000000000..c039a8ea3 --- /dev/null +++ b/src/plasma/thirdparty/ae/ae_select.c @@ -0,0 +1,106 @@ +/* Select()-based ae.c module. + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include + +typedef struct aeApiState { + fd_set rfds, wfds; + /* We need to have a copy of the fd sets as it's not safe to reuse + * FD sets after select(). */ + fd_set _rfds, _wfds; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + FD_ZERO(&state->rfds); + FD_ZERO(&state->wfds); + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Just ensure we have enough room in the fd_set type. */ + if (setsize >= FD_SETSIZE) return -1; + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + zfree(eventLoop->apidata); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + + if (mask & AE_READABLE) FD_SET(fd,&state->rfds); + if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds); + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + + if (mask & AE_READABLE) FD_CLR(fd,&state->rfds); + if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds); +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, j, numevents = 0; + + memcpy(&state->_rfds,&state->rfds,sizeof(fd_set)); + memcpy(&state->_wfds,&state->wfds,sizeof(fd_set)); + + retval = select(eventLoop->maxfd+1, + &state->_rfds,&state->_wfds,NULL,tvp); + if (retval > 0) { + for (j = 0; j <= eventLoop->maxfd; j++) { + int mask = 0; + aeFileEvent *fe = &eventLoop->events[j]; + + if (fe->mask == AE_NONE) continue; + if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds)) + mask |= AE_READABLE; + if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds)) + mask |= AE_WRITABLE; + eventLoop->fired[numevents].fd = j; + eventLoop->fired[numevents].mask = mask; + numevents++; + } + } + return numevents; +} + +static char *aeApiName(void) { + return "select"; +} diff --git a/src/plasma/thirdparty/ae/config.h b/src/plasma/thirdparty/ae/config.h new file mode 100644 index 000000000..4f8e1ea1b --- /dev/null +++ b/src/plasma/thirdparty/ae/config.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __CONFIG_H +#define __CONFIG_H + +#ifdef __APPLE__ +#include +#endif + +/* Test for polling API */ +#ifdef __linux__ +#define HAVE_EPOLL 1 +#endif + +#if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__) +#define HAVE_KQUEUE 1 +#endif + +#ifdef __sun +#include +#ifdef _DTRACE_VERSION +#define HAVE_EVPORT 1 +#endif +#endif + + +#endif diff --git a/src/plasma/thirdparty/ae/zmalloc.h b/src/plasma/thirdparty/ae/zmalloc.h new file mode 100644 index 000000000..54c8a69cb --- /dev/null +++ b/src/plasma/thirdparty/ae/zmalloc.h @@ -0,0 +1,16 @@ +#ifndef _ZMALLOC_H +#define _ZMALLOC_H + +#ifndef zmalloc +#define zmalloc malloc +#endif + +#ifndef zfree +#define zfree free +#endif + +#ifndef zrealloc +#define zrealloc realloc +#endif + +#endif /* _ZMALLOC_H */ diff --git a/test/component_failures_test.py b/test/component_failures_test.py index fe47ea22c..4231be80b 100644 --- a/test/component_failures_test.py +++ b/test/component_failures_test.py @@ -163,7 +163,7 @@ class ComponentFailureTest(unittest.TestCase): if check_component_alive: self.assertTrue(component.poll() is None) else: - self.assertTrue(component.poll() <= 0) + self.assertTrue(not component.poll() is None) def testLocalSchedulerFailed(self): # Kill all local schedulers on worker nodes.