wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-02 13:01:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove legacy Ray code. (#3121)

Browse Source 
* Remove legacy Ray code.

* Fix cmake and simplify monitor.

* Fix linting

* Updates

* Fix

* Implement some methods.

* Remove more plasma manager references.

* Fix

* Linting

* Fix

* Fix

* Make sure class IDs are strings.

* Some path fixes

* Fix

* Path fixes and update arrow

* Fixes.

* linting

* Fixes

* Java fixes

* Some java fixes

* TaskLanguage -> Language

* Minor

* Fix python test and remove unused method signature.

* Fix java tests

* Fix jenkins tests

* Remove commented out code.
This commit is contained in:
Robert Nishihara
2018-10-26 13:36:58 -07:00
committed by Philipp Moritz
parent 055daf17a0
commit 658c14282c
289 changed files with 2460 additions and 40708 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/CMakeLists.txt
-131
View File
@@ -1,131 +0,0 @@
cmake_minimum_required(VERSION 3.4)
project(common)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${CMAKE_CURRENT_LIST_DIR}/lib/python")
endif ()
add_subdirectory(redis_module)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -g")
include_directories(thirdparty/ae)
# Compile flatbuffers
set(COMMON_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
set(COMMON_FBS_OUTPUT_FILES
"${OUTPUT_DIR}/common_generated.h")
add_custom_target(gen_common_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
add_custom_command(
OUTPUT ${COMMON_FBS_OUTPUT_FILES}
# The --gen-object-api flag generates a C++ class MessageT for each
# flatbuffers message Message, which can be used to store deserialized
# messages in data structures. This is currently used for ObjectInfo for
# example.
COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${COMMON_FBS_SRC} --gen-object-api --scoped-enums
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
add_custom_target(gen_common_python_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
# Generate Python bindings for the flatbuffers objects.
set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../python/ray/core/generated/)
add_custom_command(
TARGET gen_common_python_fbs
COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
# Encode the fact that the ray redis module requires the autogenerated
# flatbuffer files to compile.
add_dependencies(ray_redis_module gen_common_python_fbs)
add_dependencies(gen_common_python_fbs flatbuffers_ep)
endif()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
add_custom_target(gen_common_java_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
# Generate Java bindings for the flatbuffers objects.
set(JAVA_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/java)
add_custom_command(
TARGET gen_common_java_fbs
COMMAND ${FLATBUFFERS_COMPILER} -j -o ${JAVA_OUTPUT_DIR} ${COMMON_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
# Encode the fact that the ray redis module requires the autogenerated
# flatbuffer files to compile.
add_dependencies(ray_redis_module gen_common_java_fbs)
add_dependencies(gen_common_java_fbs flatbuffers_ep)
endif()
add_custom_target(
hiredis
COMMAND make
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis)
add_library(common STATIC
event_loop.cc
common.cc
common_protocol.cc
task.cc
io.cc
net.cc
logging.cc
state/redis.cc
state/table.cc
state/object_table.cc
state/task_table.cc
state/db_client_table.cc
state/driver_table.cc
state/actor_notification_table.cc
state/local_scheduler_table.cc
state/error_table.cc
thirdparty/ae/ae.c
thirdparty/sha256.c)
add_dependencies(common arrow)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
add_dependencies(common gen_common_python_fbs)
endif()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
add_dependencies(common gen_common_java_fbs)
endif()
target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhiredis.a")
function(define_test test_name library)
add_executable(${test_name} test/${test_name}.cc ${ARGN})
add_dependencies(${test_name} hiredis flatbuffers_ep)
target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${library} -lpthread)
target_compile_options(${test_name} PUBLIC "-DPLASMA_TEST -DLOCAL_SCHEDULER_TEST -DCOMMON_TEST -DRAY_COMMON_LOG_LEVEL=4")
endfunction()
define_test(db_tests "")
define_test(io_tests "")
define_test(task_tests "")
define_test(redis_tests "")
define_test(task_table_tests "")
define_test(object_table_tests "")
add_custom_target(copy_redis ALL)
foreach(file "redis-cli" "redis-server")
add_custom_command(TARGET copy_redis POST_BUILD
COMMAND ${CMAKE_COMMAND} -E
copy ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/redis/src/${file}
${CMAKE_BINARY_DIR}/src/common/thirdparty/redis/src/${file})
endforeach()
src/common/common.cc
-20
View File
@@ -1,20 +0,0 @@
#include "common.h"
#include <chrono>
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include "io.h"
#include <functional>
const unsigned char NIL_DIGEST[DIGEST_SIZE] = {0};
int64_t current_time_ms() {
std::chrono::milliseconds ms_since_epoch =
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch());
return ms_since_epoch.count();
}
src/common/common.h
-75
View File
@@ -1,75 +0,0 @@
#ifndef COMMON_H
#define COMMON_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <errno.h>
#include <inttypes.h>
#ifndef _WIN32
#include <execinfo.h>
#endif
#ifdef __cplusplus
#include <functional>
extern "C" {
#endif
#include "sha256.h"
#ifdef __cplusplus
}
#endif
#include "arrow/util/macros.h"
#include "plasma/common.h"
#include "ray/id.h"
#include "ray/util/logging.h"
#include "state/ray_config.h"
/** Definitions for Ray logging levels. */
#define RAY_COMMON_DEBUG 0
#define RAY_COMMON_INFO 1
#define RAY_COMMON_WARNING 2
#define RAY_COMMON_ERROR 3
#define RAY_COMMON_FATAL 4
/**
* RAY_COMMON_LOG_LEVEL should be defined to one of the above logging level
* integer values. Any logging statement in the code with a logging level
* greater than or equal to RAY_COMMON_LOG_LEVEL will be outputted to stderr.
* The default logging level is INFO. */
#ifndef RAY_COMMON_LOG_LEVEL
#define RAY_COMMON_LOG_LEVEL RAY_COMMON_INFO
#endif
/* These are exit codes for common errors that can occur in Ray components. */
#define EXIT_COULD_NOT_BIND_PORT -2
/** This macro indicates that this pointer owns the data it is pointing to
* and is responsible for freeing it. */
#define OWNER
/** The worker ID is the ID of a worker or driver. */
typedef ray::UniqueID WorkerID;
typedef ray::UniqueID DBClientID;
#define MAX(x, y) ((x) >= (y) ? (x) : (y))
#define MIN(x, y) ((x) <= (y) ? (x) : (y))
/** Definitions for computing hash digests. */
#define DIGEST_SIZE SHA256_BLOCK_SIZE
extern const unsigned char NIL_DIGEST[DIGEST_SIZE];
/**
* Return the current time in milliseconds since the Unix epoch.
*
* @return The number of milliseconds since the Unix epoch.
*/
int64_t current_time_ms();
#endif
src/common/doc/tasks.md
-32
View File
@@ -1,32 +0,0 @@
# Task specifications, task instances and task logs
A *task specification* contains all information that is needed for computing
the results of a task:
- The ID of the task
- The function ID of the function that executes the task
- The arguments (either object IDs for pass by reference
or values for pass by value)
- The IDs of the result objects
From these, a task ID can be computed which is also stored in the task
specification.
A *task* represents the execution of a task specification.
It consists of:
- A scheduling state (WAITING, SCHEDULED, RUNNING, DONE)
- The target node where the task is scheduled or executed
- The task specification
The task data structures are defined in `common/task.h`.
The *task table* is a mapping from the task ID to the *task* information. It is
updated by various parts of the system:
1. The local scheduler writes it with status WAITING when submits a task to the global scheduler
2. The global scheduler appends an update WAITING -> SCHEDULED together with the node ID when assigning the task to a local scheduler
3. The local scheduler appends an update SCHEDULED -> RUNNING when it assigns a task to a worker
4. The local scheduler appends an update RUNNING -> DONE when the task finishes execution
The task table is defined in `common/state/task_table.h`.
src/common/event_loop.cc
-63
View File
@@ -1,63 +0,0 @@
#include "event_loop.h"
#include "common.h"
#include <errno.h>
#define INITIAL_EVENT_LOOP_SIZE 1024
event_loop *event_loop_create(void) {
return aeCreateEventLoop(INITIAL_EVENT_LOOP_SIZE);
}
void event_loop_destroy(event_loop *loop) {
/* Clean up timer events. This is to make valgrind happy. */
aeTimeEvent *te = loop->timeEventHead;
while (te) {
aeTimeEvent *next = te->next;
free(te);
te = next;
}
aeDeleteEventLoop(loop);
}
bool event_loop_add_file(event_loop *loop,
int fd,
int events,
event_loop_file_handler handler,
void *context) {
/* Try to add the file descriptor. */
int err = aeCreateFileEvent(loop, fd, events, handler, context);
/* If it cannot be added, increase the size of the event loop. */
if (err == AE_ERR && errno == ERANGE) {
err = aeResizeSetSize(loop, 3 * aeGetSetSize(loop) / 2);
if (err != AE_OK) {
return false;
}
err = aeCreateFileEvent(loop, fd, events, handler, context);
}
/* In any case, test if there were errors. */
return (err == AE_OK);
}
void event_loop_remove_file(event_loop *loop, int fd) {
aeDeleteFileEvent(loop, fd, EVENT_LOOP_READ | EVENT_LOOP_WRITE);
}
int64_t event_loop_add_timer(event_loop *loop,
int64_t timeout,
event_loop_timer_handler handler,
void *context) {
return aeCreateTimeEvent(loop, timeout, handler, context, NULL);
}
int event_loop_remove_timer(event_loop *loop, int64_t id) {
return aeDeleteTimeEvent(loop, id);
}
void event_loop_run(event_loop *loop) {
aeMain(loop);
}
void event_loop_stop(event_loop *loop) {
aeStop(loop);
}
src/common/event_loop.h
-103
View File
@@ -1,103 +0,0 @@
#ifndef EVENT_LOOP_H
#define EVENT_LOOP_H
#include <stdint.h>
extern "C" {
#ifdef _WIN32
/* Quirks mean that Windows version needs to be included differently */
#include <hiredis/hiredis.h>
#include <ae.h>
#else
#include "ae/ae.h"
#endif
}
/* Unique timer ID that will be generated when the timer is added to the
* event loop. Will not be reused later on in another call
* to event_loop_add_timer. */
typedef long long timer_id;
typedef aeEventLoop event_loop;
/* File descriptor is readable. */
#define EVENT_LOOP_READ AE_READABLE
/* File descriptor is writable. */
#define EVENT_LOOP_WRITE AE_WRITABLE
/* Constant specifying that the timer is done and it will be removed. */
#define EVENT_LOOP_TIMER_DONE AE_NOMORE
/* Signature of the handler that will be called when there is a new event
* on the file descriptor that this handler has been registered for. The
* context is the one that was passed into add_file by the user. The
* events parameter indicates which event is available on the file,
* it can be EVENT_LOOP_READ or EVENT_LOOP_WRITE. */
typedef void (*event_loop_file_handler)(event_loop *loop,
int fd,
void *context,
int events);
/* This handler will be called when a timer times out. The id of the timer
* as well as the context that was specified when registering this handler
* are passed as arguments. The return is the number of milliseconds the
* timer shall be reset to or EVENT_LOOP_TIMER_DONE if the timer shall
* not be triggered again. */
typedef int (*event_loop_timer_handler)(event_loop *loop,
timer_id timer_id,
void *context);
/* Create and return a new event loop. */
event_loop *event_loop_create(void);
/* Deallocate space associated with the event loop that was created
* with the "create" function. */
void event_loop_destroy(event_loop *loop);
/* Register a handler that will be called any time a new event happens on
* a file descriptor. Can specify a context that will be passed as an
* argument to the handler. Currently there can only be one handler per file.
* The events parameter specifies which events we listen to: EVENT_LOOP_READ
* or EVENT_LOOP_WRITE. */
bool event_loop_add_file(event_loop *loop,
int fd,
int events,
event_loop_file_handler handler,
void *context);
/* Remove a registered file event handler from the event loop. */
void event_loop_remove_file(event_loop *loop, int fd);
/** Register a handler that will be called after a time slice of
* "timeout" milliseconds.
*
* @param loop The event loop.
* @param timeout The timeout in milliseconds.
* @param handler The handler for the timeout.
* @param context User context that can be passed in and will be passed in
* as an argument for the timer handler.
* @return The ID of the timer.
*/
int64_t event_loop_add_timer(event_loop *loop,
int64_t timeout,
event_loop_timer_handler handler,
void *context);
/**
* Remove a registered time event handler from the event loop. Can be called
* multiple times on the same timer.
*
* @param loop The event loop.
* @param timer_id The ID of the timer to be removed.
* @return Returns 0 if the removal was successful.
*/
int event_loop_remove_timer(event_loop *loop, int64_t timer_id);
/* Run the event loop. */
void event_loop_run(event_loop *loop);
/* Stop the event loop. */
void event_loop_stop(event_loop *loop);
#endif
src/common/format/common.fbs
-203
View File
@@ -1,203 +0,0 @@
// Indices into resource vectors.
// A resource vector maps a resource index to the number
// of units of that resource required.
table Arg {
// Object ID for pass-by-reference arguments. Normally there is only one
// object ID in this list which represents the object that is being passed.
// However to support reducers in a MapReduce workload, we also support
// passing multiple object IDs for each argument.
object_ids: [string];
// Data for pass-by-value arguments.
data: string;
}
table ResourcePair {
// The name of the resource.
key: string;
// The quantity of the resource.
value: double;
}
// NOTE: This enum is duplicate with the `Language` enum in `gcs.fbs`,
// because we cannot include this file in `gcs.fbs` due to cyclic dependency.
// TODO(raulchen): remove it once we get rid of legacy ray.
enum TaskLanguage:int {
PYTHON = 0,
JAVA = 1
}
table TaskInfo {
// ID of the driver that created this task.
driver_id: string;
// Task ID of the task.
task_id: string;
// Task ID of the parent task.
parent_task_id: string;
// A count of the number of tasks submitted by the parent task before this one.
parent_counter: int;
// The ID of the actor to create if this is an actor creation task.
actor_creation_id: string;
// The dummy object ID of the actor creation task if this is an actor method.
actor_creation_dummy_object_id: string;
// Actor ID of the task. This is the actor that this task is executed on
// or NIL_ACTOR_ID if the task is just a normal task.
actor_id: string;
// The ID of the handle that was used to submit the task. This should be
// unique across handles with the same actor_id.
actor_handle_id: string;
// Number of tasks that have been submitted to this actor so far.
actor_counter: int;
// True if this task is an actor checkpoint task and false otherwise.
is_actor_checkpoint_method: bool;
// Function ID of the task.
function_id: string;
// Task arguments.
args: [Arg];
// Object IDs of return values.
returns: [string];
// The required_resources vector indicates the quantities of the different
// resources required by this task.
required_resources: [ResourcePair];
// The resources required for placing this task on a node. If this is empty,
// then the placement resources are equal to the required_resources.
required_placement_resources: [ResourcePair];
// The language that this task belongs to
language: TaskLanguage;
// Function descriptor, which is a list of strings that can
// uniquely describe a function.
// For a Python function, it should be: [module_name, class_name, function_name]
// For a Java function, it should be: [class_name, method_name, type_descriptor]
// TODO(hchen): after changing Python worker to use function_descriptor,
// function_id can be removed.
function_descriptor: [string];
}
// Object information data structure.
// NOTE(pcm): This structure is replicated in
// https://github.com/apache/arrow/blob/master/cpp/src/plasma/format/common.fbs,
// so if you modify it, you should also modify that one.
table ObjectInfo {
// Object ID of this object.
object_id: string;
// Number of bytes the content of this object occupies in memory.
data_size: long;
// Number of bytes the metadata of this object occupies in memory.
metadata_size: long;
// Number of clients using the objects.
ref_count: int;
// Unix epoch of when this object was created.
create_time: long;
// How long creation of this object took.
construct_duration: long;
// Hash of the object content. If the object is not sealed yet this is
// an empty string.
digest: string;
// Specifies if this object was deleted or added.
is_deletion: bool;
}
root_type TaskInfo;
table TaskExecutionDependencies {
// A list of object IDs representing this task's dependencies at execution
// time.
execution_dependencies: [string];
}
root_type TaskExecutionDependencies;
table SubscribeToNotificationsReply {
// The object ID of the object that the notification is about.
object_id: string;
// The size of the object.
object_size: long;
// The IDs of the managers that contain this object.
manager_ids: [string];
}
root_type SubscribeToNotificationsReply;
table TaskReply {
// The task ID of the task that the message is about.
task_id: string;
// The state of the task. This is encoded as a bit mask of scheduling_state
// enum values in task.h.
state: long;
// A local scheduler ID.
local_scheduler_id: string;
// A string of bytes representing the task's TaskExecutionDependencies.
execution_dependencies: string;
// A string of bytes representing the task specification.
task_spec: string;
// The number of times the task was spilled back by local schedulers.
spillback_count: long;
// A boolean representing whether the update was successful. This field
// should only be used for test-and-set operations.
updated: bool;
}
root_type TaskReply;
table SubscribeToDBClientTableReply {
// The db client ID of the client that the message is about.
db_client_id: string;
// The type of the client.
client_type: string;
// If the client is a local scheduler, this is the address of the plasma
// manager that the local scheduler is connected to. Otherwise, it is empty.
manager_address: string;
// True if the message is about the addition of a client and false if it is
// about the deletion of a client.
is_insertion: bool;
}
root_type SubscribeToDBClientTableReply;
table LocalSchedulerInfoMessage {
// The db client ID of the client that the message is about.
db_client_id: string;
// The total number of workers that are connected to this local scheduler.
total_num_workers: long;
// The number of tasks queued in this local scheduler.
task_queue_length: long;
// The number of workers that are available and waiting for tasks.
available_workers: long;
// The resources generally available to this local scheduler.
static_resources: [ResourcePair];
// The resources currently available to this local scheduler.
dynamic_resources: [ResourcePair];
// Whether the local scheduler is dead. If true, then all other fields
// besides `db_client_id` will not be set.
is_dead: bool;
}
root_type LocalSchedulerInfoMessage;
table ResultTableReply {
// The task ID of the task that created the object.
task_id: string;
// Whether the task created the object through a ray.put.
is_put: bool;
// The size of the object created.
data_size: long;
// The hash of the object created.
hash: string;
}
root_type ResultTableReply;
table DriverTableMessage {
// The driver ID of the driver that died.
driver_id: string;
}
table ActorCreationNotification {
// The ID of the actor that was created.
actor_id: string;
// The ID of the driver that created the actor.
driver_id: string;
// The ID of the local scheduler that created the actor.
local_scheduler_id: string;
}
src/common/io.cc
-416
View File
@@ -1,416 +0,0 @@
#include "io.h"
#include <stdlib.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <string.h>
#include <stdio.h>
#include <inttypes.h>
#include <stdarg.h>
#include <sys/ioctl.h>
#include <netinet/in.h>
#include <netdb.h>
#include "common.h"
#include "event_loop.h"
#ifndef _WIN32
/* This function is actually not declared in standard POSIX, so declare it. */
extern int usleep(useconds_t usec);
#endif
int bind_inet_sock(const int port, bool shall_listen) {
struct sockaddr_in name;
int socket_fd = socket(PF_INET, SOCK_STREAM, 0);
if (socket_fd < 0) {
RAY_LOG(ERROR) << "socket() failed for port " << port;
return -1;
}
name.sin_family = AF_INET;
name.sin_port = htons(port);
name.sin_addr.s_addr = htonl(INADDR_ANY);
int on = 1;
/* TODO(pcm): http://stackoverflow.com/q/1150635 */
if (ioctl(socket_fd, FIONBIO, (char *) &on) < 0) {
RAY_LOG(ERROR) << "ioctl failed";
close(socket_fd);
return -1;
}
int *const pon = (int *const) & on;
if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, pon, sizeof(on)) < 0) {
RAY_LOG(ERROR) << "setsockopt failed for port " << port;
close(socket_fd);
return -1;
}
if (bind(socket_fd, (struct sockaddr *) &name, sizeof(name)) < 0) {
RAY_LOG(ERROR) << "Bind failed for port " << port;
close(socket_fd);
return -1;
}
if (shall_listen && listen(socket_fd, 128) == -1) {
RAY_LOG(ERROR) << "Could not listen to socket " << port;
close(socket_fd);
return -1;
}
return socket_fd;
}
int bind_ipc_sock(const char *socket_pathname, bool shall_listen) {
struct sockaddr_un socket_address;
int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
if (socket_fd < 0) {
RAY_LOG(ERROR) << "socket() failed for pathname " << socket_pathname;
return -1;
}
/* Tell the system to allow the port to be reused. */
int on = 1;
if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on,
sizeof(on)) < 0) {
RAY_LOG(ERROR) << "setsockopt failed for pathname " << socket_pathname;
close(socket_fd);
return -1;
}
unlink(socket_pathname);
memset(&socket_address, 0, sizeof(socket_address));
socket_address.sun_family = AF_UNIX;
if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) {
RAY_LOG(ERROR) << "Socket pathname is too long.";
close(socket_fd);
return -1;
}
strncpy(socket_address.sun_path, socket_pathname,
strlen(socket_pathname) + 1);
if (bind(socket_fd, (struct sockaddr *) &socket_address,
sizeof(socket_address)) != 0) {
RAY_LOG(ERROR) << "Bind failed for pathname " << socket_pathname;
close(socket_fd);
return -1;
}
if (shall_listen && listen(socket_fd, 128) == -1) {
RAY_LOG(ERROR) << "Could not listen to socket " << socket_pathname;
close(socket_fd);
return -1;
}
return socket_fd;
}
int connect_ipc_sock_retry(const char *socket_pathname,
int num_retries,
int64_t timeout) {
/* Pick the default values if the user did not specify. */
if (num_retries < 0) {
num_retries = RayConfig::instance().num_connect_attempts();
}
if (timeout < 0) {
timeout = RayConfig::instance().connect_timeout_milliseconds();
}
RAY_CHECK(socket_pathname);
int fd = -1;
for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
fd = connect_ipc_sock(socket_pathname);
if (fd >= 0) {
break;
}
if (num_attempts == 0) {
RAY_LOG(ERROR) << "Connection to socket failed for pathname "
<< socket_pathname;
}
/* Sleep for timeout milliseconds. */
usleep(timeout * 1000);
}
/* If we could not connect to the socket, exit. */
if (fd == -1) {
RAY_LOG(FATAL) << "Could not connect to socket " << socket_pathname;
}
return fd;
}
int connect_ipc_sock(const char *socket_pathname) {
struct sockaddr_un socket_address;
int socket_fd;
socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
if (socket_fd < 0) {
RAY_LOG(ERROR) << "socket() failed for pathname " << socket_pathname;
return -1;
}
memset(&socket_address, 0, sizeof(socket_address));
socket_address.sun_family = AF_UNIX;
if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) {
RAY_LOG(ERROR) << "Socket pathname is too long.";
return -1;
}
strncpy(socket_address.sun_path, socket_pathname,
strlen(socket_pathname) + 1);
if (connect(socket_fd, (struct sockaddr *) &socket_address,
sizeof(socket_address)) != 0) {
close(socket_fd);
return -1;
}
return socket_fd;
}
int connect_inet_sock_retry(const char *ip_addr,
int port,
int num_retries,
int64_t timeout) {
/* Pick the default values if the user did not specify. */
if (num_retries < 0) {
num_retries = RayConfig::instance().num_connect_attempts();
}
if (timeout < 0) {
timeout = RayConfig::instance().connect_timeout_milliseconds();
}
RAY_CHECK(ip_addr);
int fd = -1;
for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
fd = connect_inet_sock(ip_addr, port);
if (fd >= 0) {
break;
}
if (num_attempts == 0) {
RAY_LOG(ERROR) << "Connection to socket failed for address " << ip_addr
<< ":" << port;
}
/* Sleep for timeout milliseconds. */
usleep(timeout * 1000);
}
/* If we could not connect to the socket, exit. */
if (fd == -1) {
RAY_LOG(FATAL) << "Could not connect to address " << ip_addr << ":" << port;
}
return fd;
}
int connect_inet_sock(const char *ip_addr, int port) {
int fd = socket(PF_INET, SOCK_STREAM, 0);
if (fd < 0) {
RAY_LOG(ERROR) << "socket() failed for address " << ip_addr << ":" << port;
return -1;
}
struct hostent *manager = gethostbyname(ip_addr); /* TODO(pcm): cache this */
if (!manager) {
RAY_LOG(ERROR) << "Failed to get hostname from address " << ip_addr << ":"
<< port;
close(fd);
return -1;
}
struct sockaddr_in addr;
addr.sin_family = AF_INET;
memcpy(&addr.sin_addr.s_addr, manager->h_addr_list[0], manager->h_length);
addr.sin_port = htons(port);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) != 0) {
close(fd);
return -1;
}
return fd;
}
int accept_client(int socket_fd) {
int client_fd = accept(socket_fd, NULL, NULL);
if (client_fd < 0) {
RAY_LOG(ERROR) << "Error reading from socket.";
return -1;
}
return client_fd;
}
int write_bytes(int fd, uint8_t *cursor, size_t length) {
ssize_t nbytes = 0;
size_t bytesleft = length;
size_t offset = 0;
while (bytesleft > 0) {
/* While we haven't written the whole message, write to the file
* descriptor, advance the cursor, and decrease the amount left to write. */
nbytes = write(fd, cursor + offset, bytesleft);
if (nbytes < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
continue;
}
return -1; /* Errno will be set. */
} else if (0 == nbytes) {
/* Encountered early EOF. */
return -1;
}
RAY_CHECK(nbytes > 0);
bytesleft -= nbytes;
offset += nbytes;
}
return 0;
}
int do_write_message(int fd, int64_t type, int64_t length, uint8_t *bytes) {
int64_t version = RayConfig::instance().ray_protocol_version();
int closed;
closed = write_bytes(fd, (uint8_t *) &version, sizeof(version));
if (closed) {
return closed;
}
closed = write_bytes(fd, (uint8_t *) &type, sizeof(type));
if (closed) {
return closed;
}
closed = write_bytes(fd, (uint8_t *) &length, sizeof(length));
if (closed) {
return closed;
}
closed = write_bytes(fd, bytes, length * sizeof(char));
if (closed) {
return closed;
}
return 0;
}
int write_message(int fd,
int64_t type,
int64_t length,
uint8_t *bytes,
std::mutex *mutex) {
if (mutex != NULL) {
std::unique_lock<std::mutex> guard(*mutex);
return do_write_message(fd, type, length, bytes);
} else {
return do_write_message(fd, type, length, bytes);
}
}
int read_bytes(int fd, uint8_t *cursor, size_t length) {
ssize_t nbytes = 0;
/* Termination condition: EOF or read 'length' bytes total. */
size_t bytesleft = length;
size_t offset = 0;
while (bytesleft > 0) {
nbytes = read(fd, cursor + offset, bytesleft);
if (nbytes < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
continue;
}
return -1; /* Errno will be set. */
} else if (0 == nbytes) {
/* Encountered early EOF. */
return -1;
}
RAY_CHECK(nbytes > 0);
bytesleft -= nbytes;
offset += nbytes;
}
return 0;
}
void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes) {
int64_t version;
int closed = read_bytes(fd, (uint8_t *) &version, sizeof(version));
if (closed) {
goto disconnected;
}
RAY_CHECK(version == RayConfig::instance().ray_protocol_version());
closed = read_bytes(fd, (uint8_t *) type, sizeof(*type));
if (closed) {
goto disconnected;
}
closed = read_bytes(fd, (uint8_t *) length, sizeof(*length));
if (closed) {
goto disconnected;
}
*bytes = (uint8_t *) malloc(*length * sizeof(uint8_t));
closed = read_bytes(fd, *bytes, *length);
if (closed) {
free(*bytes);
goto disconnected;
}
return;
disconnected:
/* Handle the case in which the socket is closed. */
*type = static_cast<int64_t>(CommonMessageType::DISCONNECT_CLIENT);
*length = 0;
*bytes = NULL;
return;
}
uint8_t *read_message_async(event_loop *loop, int sock) {
int64_t size;
int error = read_bytes(sock, (uint8_t *) &size, sizeof(int64_t));
if (error < 0) {
/* The other side has closed the socket. */
RAY_LOG(DEBUG) << "Socket has been closed, or some other error has "
<< "occurred.";
if (loop != NULL) {
event_loop_remove_file(loop, sock);
}
close(sock);
return NULL;
}
uint8_t *message = (uint8_t *) malloc(size);
error = read_bytes(sock, message, size);
if (error < 0) {
/* The other side has closed the socket. */
RAY_LOG(DEBUG) << "Socket has been closed, or some other error has "
<< "occurred.";
if (loop != NULL) {
event_loop_remove_file(loop, sock);
}
close(sock);
return NULL;
}
return message;
}
int64_t read_vector(int fd, int64_t *type, std::vector<uint8_t> &buffer) {
int64_t version;
int closed = read_bytes(fd, (uint8_t *) &version, sizeof(version));
if (closed) {
goto disconnected;
}
RAY_CHECK(version == RayConfig::instance().ray_protocol_version());
int64_t length;
closed = read_bytes(fd, (uint8_t *) type, sizeof(*type));
if (closed) {
goto disconnected;
}
closed = read_bytes(fd, (uint8_t *) &length, sizeof(length));
if (closed) {
goto disconnected;
}
if (static_cast<size_t>(length) > buffer.size()) {
buffer.resize(length);
}
closed = read_bytes(fd, buffer.data(), length);
if (closed) {
goto disconnected;
}
return length;
disconnected:
/* Handle the case in which the socket is closed. */
*type = static_cast<int64_t>(CommonMessageType::DISCONNECT_CLIENT);
return 0;
}
void write_log_message(int fd, const char *message) {
/* Account for the \0 at the end of the string. */
do_write_message(fd, static_cast<int64_t>(CommonMessageType::LOG_MESSAGE),
strlen(message) + 1, (uint8_t *) message);
}
char *read_log_message(int fd) {
uint8_t *bytes;
int64_t type;
int64_t length;
read_message(fd, &type, &length, &bytes);
RAY_CHECK(static_cast<CommonMessageType>(type) ==
CommonMessageType::LOG_MESSAGE);
return (char *) bytes;
}
src/common/io.h
-228
View File
@@ -1,228 +0,0 @@
#ifndef IO_H
#define IO_H
#include <stdint.h>
#include <stdlib.h>
#include <mutex>
#include <vector>
struct aeEventLoop;
typedef aeEventLoop event_loop;
enum class CommonMessageType : int32_t {
/** Disconnect a client. */
DISCONNECT_CLIENT,
/** Log a message from a client. */
LOG_MESSAGE,
/** Submit a task to the local scheduler. */
SUBMIT_TASK,
};
/* Helper functions for socket communication. */
/**
* Binds to an Internet socket at the given port. Removes any existing file at
* the pathname. Returns a non-blocking file descriptor for the socket, or -1
* if an error occurred.
*
* @note Since the returned file descriptor is non-blocking, it is not
* recommended to use the Linux read and write calls directly, since these
* might read or write a partial message. Instead, use the provided
* write_message and read_message methods.
*
* @param port The port to bind to.
* @param shall_listen Are we also starting to listen on the socket?
* @return A non-blocking file descriptor for the socket, or -1 if an error
* occurs.
*/
int bind_inet_sock(const int port, bool shall_listen);
/**
* Binds to a Unix domain streaming socket at the given
* pathname. Removes any existing file at the pathname.
*
* @param socket_pathname The pathname for the socket.
* @param shall_listen Are we also starting to listen on the socket?
* @return A blocking file descriptor for the socket, or -1 if an error
* occurs.
*/
int bind_ipc_sock(const char *socket_pathname, bool shall_listen);
/**
* Connect to a Unix domain streaming socket at the given
* pathname.
*
* @param socket_pathname The pathname for the socket.
* @return A file descriptor for the socket, or -1 if an error occurred.
*/
int connect_ipc_sock(const char *socket_pathname);
/**
* Connect to a Unix domain streaming socket at the given
* pathname, or fail after some number of retries.
*
* @param socket_pathname The pathname for the socket.
* @param num_retries The number of times to retry the connection
* before exiting. If -1 is provided, then this defaults to
* num_connect_attempts.
* @param timeout The number of milliseconds to wait in between
* retries. If -1 is provided, then this defaults to
* connect_timeout_milliseconds.
* @return A file descriptor for the socket, or -1 if an error occurred.
*/
int connect_ipc_sock_retry(const char *socket_pathname,
int num_retries,
int64_t timeout);
/**
* Connect to an Internet socket at the given address and port.
*
* @param ip_addr The IP address to connect to.
* @param port The port number to connect to.
*
* @param socket_pathname The pathname for the socket.
* @return A file descriptor for the socket, or -1 if an error occurred.
*/
int connect_inet_sock(const char *ip_addr, int port);
/**
* Connect to an Internet socket at the given address and port, or fail after
* some number of retries.
*
* @param ip_addr The IP address to connect to.
* @param port The port number to connect to.
* @param num_retries The number of times to retry the connection
* before exiting. If -1 is provided, then this defaults to
* num_connect_attempts.
* @param timeout The number of milliseconds to wait in between
* retries. If -1 is provided, then this defaults to
* connect_timeout_milliseconds.
* @return A file descriptor for the socket, or -1 if an error occurred.
*/
int connect_inet_sock_retry(const char *ip_addr,
int port,
int num_retries,
int64_t timeout);
/**
* Accept a new client connection on the given socket
* descriptor. Returns a descriptor for the new socket.
*/
int accept_client(int socket_fd);
/* Reading and writing data. */
/**
* Write a sequence of bytes on a file descriptor. The bytes should then be read
* by read_message.
*
* @param fd The file descriptor to write to. It can be non-blocking.
* @param version The protocol version.
* @param type The type of the message to send.
* @param length The size in bytes of the bytes parameter.
* @param bytes The address of the message to send.
* @param mutex If not NULL, the whole write operation will be locked
* with this mutex, otherwise do nothing.
* @return int Whether there was an error while writing. 0 corresponds to
* success and -1 corresponds to an error (errno will be set).
*/
int write_message(int fd,
int64_t type,
int64_t length,
uint8_t *bytes,
std::mutex *mutex = NULL);
/**
* Read a sequence of bytes written by write_message from a file descriptor.
* This allocates space for the message.
*
* @note The caller must free the memory.
*
* @param fd The file descriptor to read from. It can be non-blocking.
* @param type The type of the message that is read will be written at this
* address. If there was an error while reading, this will be
* DISCONNECT_CLIENT.
* @param length The size in bytes of the message that is read will be written
* at this address. This size does not include the bytes used to encode
* the type and length. If there was an error while reading, this will
* be 0.
* @param bytes The address at which to write the pointer to the bytes that are
* read and allocated by this function. If there was an error while
* reading, this will be NULL.
* @return Void.
*/
void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes);
/**
* Read a message from a file descriptor and remove the file descriptor from the
* event loop if there is an error. This will actually do two reads. The first
* read reads sizeof(int64_t) bytes to determine the number of bytes to read in
* the next read.
*
* @param loop: The event loop.
* @param sock: The file descriptor to read from.
* @return A byte buffer contining the message or NULL if there was an
* error. The buffer needs to be freed by the user.
*/
uint8_t *read_message_async(event_loop *loop, int sock);
/**
* Read a sequence of bytes written by write_message from a file descriptor.
* This does not allocate space for the message if the provided buffer is
* large enough and can therefore often avoid allocations.
*
* @param fd The file descriptor to read from. It can be non-blocking.
* @param type The type of the message that is read will be written at this
* address. If there was an error while reading, this will be
* DISCONNECT_CLIENT.
* @param buffer The array the message will be written to. If it is not
* large enough to hold the message, it will be enlarged by read_vector.
* @return Number of bytes of the message that were read. This size does not
* include the bytes used to encode the type and length. If there was
* an error while reading, this will be 0.
*/
int64_t read_vector(int fd, int64_t *type, std::vector<uint8_t> &buffer);
/**
* Write a null-terminated string to a file descriptor.
*/
void write_log_message(int fd, const char *message);
/**
* Reads a null-terminated string from the file descriptor that has been
* written by write_log_message. Allocates and returns a pointer to the string.
* NOTE: Caller must free the memory!
*/
char *read_log_message(int fd);
/**
* Read a sequence of bytes from a file descriptor into a buffer. This will
* block until one of the following happens: (1) there is an error (2) end of
* file, or (3) all length bytes have been written.
*
* @note The buffer pointed to by cursor must already have length number of
* bytes allocated before calling this method.
*
* @param fd The file descriptor to read from. It can be non-blocking.
* @param cursor The cursor pointing to the beginning of the buffer.
* @param length The size of the byte sequence to read.
* @return int Whether there was an error while reading. 0 corresponds to
* success and -1 corresponds to an error (errno will be set).
*/
int read_bytes(int fd, uint8_t *cursor, size_t length);
/**
* Write a sequence of bytes into a file descriptor. This will block until one
* of the following happens: (1) there is an error (2) end of file, or (3) all
* length bytes have been written.
*
* @param fd The file descriptor to write to. It can be non-blocking.
* @param cursor The cursor pointing to the beginning of the bytes to send.
* @param length The size of the bytes sequence to write.
* @return int Whether there was an error while writing. 0 corresponds to
* success and -1 corresponds to an error (errno will be set).
*/
int write_bytes(int fd, uint8_t *cursor, size_t length);
#endif /* IO_H */
src/common/logging.cc
-107
View File
@@ -1,107 +0,0 @@
#include "logging.h"
#include <inttypes.h>
#include <stdint.h>
#include <sys/time.h>
#include <hiredis/hiredis.h>
#include "state/redis.h"
#include "io.h"
#include <iostream>
#include <string>
static const char *log_levels[5] = {"DEBUG", "INFO", "WARN", "ERROR", "FATAL"};
static const char *log_fmt =
"HMSET log:%s:%s log_level %s event_type %s message %s timestamp %s";
struct RayLoggerImpl {
/* String that identifies this client type. */
const char *client_type;
/* Suppress all log messages below this level. */
int log_level;
/* Whether or not we have a direct connection to Redis. */
int is_direct;
/* Either a db_handle or a socket to a process with a db_handle,
* depending on the is_direct flag. */
void *conn;
};
RayLogger *RayLogger_init(const char *client_type,
int log_level,
int is_direct,
void *conn) {
RayLogger *logger = (RayLogger *) malloc(sizeof(RayLogger));
logger->client_type = client_type;
logger->log_level = log_level;
logger->is_direct = is_direct;
logger->conn = conn;
return logger;
}
void RayLogger_free(RayLogger *logger) {
free(logger);
}
void RayLogger_log(RayLogger *logger,
int log_level,
const char *event_type,
const char *message) {
if (log_level < logger->log_level) {
return;
}
if (log_level < RAY_LOG_DEBUG || log_level > RAY_LOG_FATAL) {
return;
}
struct timeval tv;
gettimeofday(&tv, NULL);
std::string timestamp =
std::to_string(tv.tv_sec) + "." + std::to_string(tv.tv_usec);
/* Find number of bytes that would have been written for formatted_message
* size */
size_t formatted_message_size =
std::snprintf(nullptr, 0, log_fmt, timestamp.c_str(), "%b",
log_levels[log_level], event_type, message,
timestamp.c_str()) +
1;
/* Fill out everything except the client ID, which is binary data. */
char formatted_message[formatted_message_size];
std::snprintf(formatted_message, formatted_message_size, log_fmt,
timestamp.c_str(), "%b", log_levels[log_level], event_type,
message, timestamp.c_str());
if (logger->is_direct) {
DBHandle *db = (DBHandle *) logger->conn;
/* Fill in the client ID and send the message to Redis. */
redisAsyncContext *context = get_redis_context(db, db->client);
int status =
redisAsyncCommand(context, NULL, NULL, formatted_message,
(char *) db->client.data(), sizeof(db->client));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error while logging message to log table");
}
} else {
/* If we don't own a Redis connection, we leave our client
* ID to be filled in by someone else. */
int *socket_fd = (int *) logger->conn;
write_log_message(*socket_fd, formatted_message);
}
}
void RayLogger_log_event(DBHandle *db,
uint8_t *key,
int64_t key_length,
uint8_t *value,
int64_t value_length,
double timestamp) {
std::string timestamp_string = std::to_string(timestamp);
int status = redisAsyncCommand(db->context, NULL, NULL, "ZADD %b %s %b", key,
key_length, timestamp_string.c_str(), value,
value_length);
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error while logging message to event log");
}
}
src/common/logging.h
-58
View File
@@ -1,58 +0,0 @@
#ifndef LOGGING_H
#define LOGGING_H
#define RAY_LOG_VERBOSE -1
#define RAY_LOG_DEBUG 0
#define RAY_LOG_INFO 1
#define RAY_LOG_WARNING 2
#define RAY_LOG_ERROR 3
#define RAY_LOG_FATAL 4
/* Entity types. */
#define RAY_FUNCTION "FUNCTION"
#define RAY_OBJECT "OBJECT"
#define RAY_TASK "TASK"
#include "state/db.h"
typedef struct RayLoggerImpl RayLogger;
/* Initialize a Ray logger for the given client type and logging level. If the
* is_direct flag is set, the logger will treat the given connection as a
* direct connection to the log. Otherwise, it will treat it as a socket to
* another process with a connection to the log.
* NOTE: User is responsible for freeing the returned logger. */
RayLogger *RayLogger_init(const char *client_type,
int log_level,
int is_direct,
void *conn);
/* Free the logger. This does not free the connection to the log. */
void RayLogger_free(RayLogger *logger);
/* Log an event at the given log level with the given event_type.
* NOTE: message cannot contain spaces! JSON format is recommended.
* TODO: Support spaces in messages. */
void RayLogger_log(RayLogger *logger,
int log_level,
const char *event_type,
const char *message);
/**
* Log an event to the event log.
*
* @param db The database handle.
* @param key The key in Redis to store the event in.
* @param key_length The length of the key.
* @param value The value to log.
* @param value_length The length of the value.
* @return Void.
*/
void RayLogger_log_event(DBHandle *db,
uint8_t *key,
int64_t key_length,
uint8_t *value,
int64_t value_length,
double time);
#endif /* LOGGING_H */
src/common/net.cc
-24
View File
@@ -1,24 +0,0 @@
#include "net.h"
#include <arpa/inet.h>
#include <sstream>
#include "common.h"
int parse_ip_addr_port(const char *ip_addr_port, char *ip_addr, int *port) {
char port_str[6];
int parsed = sscanf(ip_addr_port, "%15[0-9.]:%5[0-9]", ip_addr, port_str);
if (parsed != 2) {
return -1;
}
*port = atoi(port_str);
return 0;
}
/* Return true if the ip address is valid. */
bool valid_ip_address(const std::string &ip_address) {
struct sockaddr_in sa;
int result = inet_pton(AF_INET, ip_address.c_str(), &sa.sin_addr);
return result == 1;
}
src/common/net.h
-9
View File
@@ -1,9 +0,0 @@
#ifndef NET_H
#define NET_H
/* Helper function to parse a string of the form <IP address>:<port> into the
* given ip_addr and port pointers. The ip_addr buffer must already be
* allocated. Return 0 upon success and -1 upon failure. */
int parse_ip_addr_port(const char *ip_addr_port, char *ip_addr, int *port);
#endif /* NET_H */
src/common/redis_module/ray_redis_module.cc
-1886
View File
File diff suppressed because it is too large Load Diff
src/common/shims/windows/getopt.c
-69
View File
@@ -1,69 +0,0 @@
/* http://stackoverflow.com/a/17195644/541686 */
#include <string.h>
#include <stdio.h>
int opterr = 1, /* if error message should be printed */
optind = 1, /* index into parent argv vector */
optopt, /* character checked for validity */
optreset; /* reset getopt */
char *optarg; /* argument associated with option */
#define BADCH (int) '?'
#define BADARG (int) ':'
#define EMSG ""
/*
* getopt --
* Parse argc/argv argument vector.
*/
int getopt(int nargc, char *const nargv[], const char *ostr) {
static char *place = EMSG; /* option letter processing */
const char *oli; /* option letter list index */
if (optreset || !*place) { /* update scanning pointer */
optreset = 0;
if (optind >= nargc || *(place = nargv[optind]) != '-') {
place = EMSG;
return (-1);
}
if (place[1] && *++place == '-') { /* found "--" */
++optind;
place = EMSG;
return (-1);
}
} /* option letter okay? */
if ((optopt = (int) *place++) == (int) ':' || !(oli = strchr(ostr, optopt))) {
/*
* if the user didn't specify '-' as an option,
* assume it means -1.
*/
if (optopt == (int) '-')
return (-1);
if (!*place)
++optind;
if (opterr && *ostr != ':')
(void) printf("illegal option -- %c\n", optopt);
return (BADCH);
}
if (*++oli != ':') { /* don't need argument */
optarg = NULL;
if (!*place)
++optind;
} else { /* need an argument */
if (*place) /* no white space */
optarg = place;
else if (nargc <= ++optind) { /* no arg */
place = EMSG;
if (*ostr == ':')
return (BADARG);
if (opterr)
(void) printf("option requires an argument -- %c\n", optopt);
return (BADCH);
} else /* white space */
optarg = nargv[optind];
place = EMSG;
++optind;
}
return (optopt); /* dump back option letter */
}
src/common/shims/windows/getopt.h
-4
View File
@@ -1,4 +0,0 @@
#ifndef GETOPT_H
#define GETOPT_H
#endif /* GETOPT_H */
src/common/shims/windows/msg.c
-208
View File
@@ -1,208 +0,0 @@
#include <sys/socket.h>
int socketpair(int domain, int type, int protocol, int sv[2]) {
if ((domain != AF_UNIX && domain != AF_INET) || type != SOCK_STREAM) {
return INVALID_SOCKET;
}
SOCKET sockets[2];
int r = dumb_socketpair(sockets);
sv[0] = (int) sockets[0];
sv[1] = (int) sockets[1];
return r;
}
#pragma comment(lib, "IPHlpAPI.lib")
struct _MIB_TCPROW2 {
DWORD dwState, dwLocalAddr, dwLocalPort, dwRemoteAddr, dwRemotePort,
dwOwningPid;
enum _TCP_CONNECTION_OFFLOAD_STATE dwOffloadState;
};
struct _MIB_TCPTABLE2 {
DWORD dwNumEntries;
struct _MIB_TCPROW2 table[1];
};
DECLSPEC_IMPORT ULONG WINAPI GetTcpTable2(struct _MIB_TCPTABLE2 *TcpTable,
PULONG SizePointer,
BOOL Order);
static DWORD getsockpid(SOCKET client) {
/* http://stackoverflow.com/a/25431340 */
DWORD pid = 0;
struct sockaddr_in Server = {0};
int ServerSize = sizeof(Server);
struct sockaddr_in Client = {0};
int ClientSize = sizeof(Client);
if ((getsockname(client, (struct sockaddr *) &Server, &ServerSize) == 0) &&
(getpeername(client, (struct sockaddr *) &Client, &ClientSize) == 0)) {
struct _MIB_TCPTABLE2 *TcpTable = NULL;
ULONG TcpTableSize = 0;
ULONG result;
do {
result = GetTcpTable2(TcpTable, &TcpTableSize, TRUE);
if (result != ERROR_INSUFFICIENT_BUFFER) {
break;
}
free(TcpTable);
TcpTable = (struct _MIB_TCPTABLE2 *) malloc(TcpTableSize);
} while (TcpTable != NULL);
if (result == NO_ERROR) {
for (DWORD dw = 0; dw < TcpTable->dwNumEntries; ++dw) {
struct _MIB_TCPROW2 *row = &(TcpTable->table[dw]);
if ((row->dwState == 5 /* MIB_TCP_STATE_ESTAB */) &&
(row->dwLocalAddr == Client.sin_addr.s_addr) &&
((row->dwLocalPort & 0xFFFF) == Client.sin_port) &&
(row->dwRemoteAddr == Server.sin_addr.s_addr) &&
((row->dwRemotePort & 0xFFFF) == Server.sin_port)) {
pid = row->dwOwningPid;
break;
}
}
}
free(TcpTable);
}
return pid;
}
ssize_t sendmsg(int sockfd, struct msghdr *msg, int flags) {
ssize_t result = -1;
struct cmsghdr *header = CMSG_FIRSTHDR(msg);
if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) {
/* We're trying to send over a handle of some kind.
* We have to look up which process we're communicating with,
* open a handle to it, and then duplicate our handle into it.
* However, the first two steps cannot be done atomically.
* Therefore, this code HAS A RACE CONDITIONS and is therefore NOT SECURE.
* In the absense of a malicious actor, though, it is exceedingly unlikely
* that the child process closes AND that its process ID is reassigned
* to another existing process.
*/
struct msghdr const old_msg = *msg;
int *const pfd = (int *) CMSG_DATA(header);
msg->msg_control = NULL;
msg->msg_controllen = 0;
WSAPROTOCOL_INFO protocol_info = {0};
BOOL const is_socket = !!FDAPI_GetSocketStatePtr(*pfd);
DWORD const target_pid = getsockpid(sockfd);
HANDLE target_process = NULL;
if (target_pid) {
if (!is_socket) {
/* This is a regular handle... fit it into the same struct */
target_process = OpenProcess(PROCESS_DUP_HANDLE, FALSE, target_pid);
if (target_process) {
if (DuplicateHandle(GetCurrentProcess(), (HANDLE)(intptr_t) *pfd,
target_process, (HANDLE *) &protocol_info, 0,
TRUE, DUPLICATE_SAME_ACCESS)) {
result = 0;
}
}
} else {
/* This is a socket... */
result = FDAPI_WSADuplicateSocket(*pfd, target_pid, &protocol_info);
}
}
if (result == 0) {
int const nbufs = msg->dwBufferCount + 1;
WSABUF *const bufs =
(struct _WSABUF *) _alloca(sizeof(*msg->lpBuffers) * nbufs);
bufs[0].buf = (char *) &protocol_info;
bufs[0].len = sizeof(protocol_info);
memcpy(&bufs[1], msg->lpBuffers,
msg->dwBufferCount * sizeof(*msg->lpBuffers));
DWORD nb;
msg->lpBuffers = bufs;
msg->dwBufferCount = nbufs;
GUID const wsaid_WSASendMsg = {
0xa441e712,
0x754f,
0x43ca,
{0x84, 0xa7, 0x0d, 0xee, 0x44, 0xcf, 0x60, 0x6d}};
typedef INT PASCAL WSASendMsg_t(
SOCKET s, LPWSAMSG lpMsg, DWORD dwFlags, LPDWORD lpNumberOfBytesSent,
LPWSAOVERLAPPED lpOverlapped,
LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine);
WSASendMsg_t *WSASendMsg = NULL;
result = FDAPI_WSAIoctl(sockfd, SIO_GET_EXTENSION_FUNCTION_POINTER,
&wsaid_WSASendMsg, sizeof(wsaid_WSASendMsg),
&WSASendMsg, sizeof(WSASendMsg), &nb, NULL, 0);
if (result == 0) {
result = (*WSASendMsg)(sockfd, msg, flags, &nb, NULL, NULL) == 0
? (ssize_t)(nb - sizeof(protocol_info))
: 0;
}
}
if (result != 0 && target_process && !is_socket) {
/* we failed to send the handle, and it needs cleaning up! */
HANDLE duplicated_back = NULL;
if (DuplicateHandle(target_process, *(HANDLE *) &protocol_info,
GetCurrentProcess(), &duplicated_back, 0, FALSE,
DUPLICATE_CLOSE_SOURCE)) {
CloseHandle(duplicated_back);
}
}
if (target_process) {
CloseHandle(target_process);
}
*msg = old_msg;
}
return result;
}
ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags) {
int result = -1;
struct cmsghdr *header = CMSG_FIRSTHDR(msg);
if (msg->msg_controllen &&
flags == 0 /* We can't send flags on Windows... */) {
struct msghdr const old_msg = *msg;
msg->msg_control = NULL;
msg->msg_controllen = 0;
WSAPROTOCOL_INFO protocol_info = {0};
int const nbufs = msg->dwBufferCount + 1;
WSABUF *const bufs =
(struct _WSABUF *) _alloca(sizeof(*msg->lpBuffers) * nbufs);
bufs[0].buf = (char *) &protocol_info;
bufs[0].len = sizeof(protocol_info);
memcpy(&bufs[1], msg->lpBuffers,
msg->dwBufferCount * sizeof(*msg->lpBuffers));
typedef INT PASCAL WSARecvMsg_t(
SOCKET s, LPWSAMSG lpMsg, LPDWORD lpNumberOfBytesRecvd,
LPWSAOVERLAPPED lpOverlapped,
LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine);
WSARecvMsg_t *WSARecvMsg = NULL;
DWORD nb;
GUID const wsaid_WSARecvMsg = {
0xf689d7c8,
0x6f1f,
0x436b,
{0x8a, 0x53, 0xe5, 0x4f, 0xe3, 0x51, 0xc3, 0x22}};
result = FDAPI_WSAIoctl(sockfd, SIO_GET_EXTENSION_FUNCTION_POINTER,
&wsaid_WSARecvMsg, sizeof(wsaid_WSARecvMsg),
&WSARecvMsg, sizeof(WSARecvMsg), &nb, NULL, 0);
if (result == 0) {
result = (*WSARecvMsg)(sockfd, msg, &nb, NULL, NULL) == 0
? (ssize_t)(nb - sizeof(protocol_info))
: 0;
}
if (result == 0) {
int *const pfd = (int *) CMSG_DATA(header);
if (protocol_info.iSocketType == 0 && protocol_info.iProtocol == 0) {
*pfd = *(int *) &protocol_info;
} else {
*pfd = FDAPI_WSASocket(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO,
FROM_PROTOCOL_INFO, &protocol_info, 0, 0);
}
header->cmsg_level = SOL_SOCKET;
header->cmsg_type = SCM_RIGHTS;
}
*msg = old_msg;
}
return result;
}
src/common/shims/windows/netdb.h
-4
View File
@@ -1,4 +0,0 @@
#ifndef NETDB_H
#define NETDB_H
#endif /* NETDB_H */
src/common/shims/windows/netinet/in.h
-4
View File
@@ -1,4 +0,0 @@
#ifndef IN_H
#define IN_H
#endif /* IN_H */
src/common/shims/windows/poll.h
-4
View File
@@ -1,4 +0,0 @@
#ifndef POLL_H
#define POLL_H
#endif /* POLL_H */
src/common/shims/windows/socketpair.c
-150
View File
@@ -1,150 +0,0 @@
/* socketpair.c
Copyright 2007, 2010 by Nathan C. Myers <ncm@cantrip.org>
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
The name of the author must not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/* Changes:
* 2014-02-12: merge David Woodhouse, Ger Hobbelt improvements
* git.infradead.org/users/dwmw2/openconnect.git/commitdiff/bdeefa54
* github.com/GerHobbelt/selectable-socketpair
* always init the socks[] to -1/INVALID_SOCKET on error, both on Win32/64
* and UNIX/other platforms
* 2013-07-18: Change to BSD 3-clause license
* 2010-03-31:
* set addr to 127.0.0.1 because win32 getsockname does not always set it.
* 2010-02-25:
* set SO_REUSEADDR option to avoid leaking some windows resource.
* Windows System Error 10049, "Event ID 4226 TCP/IP has reached
* the security limit imposed on the number of concurrent TCP connect
* attempts." Bleah.
* 2007-04-25:
* preserve value of WSAGetLastError() on all error returns.
* 2007-04-22: (Thanks to Matthew Gregan <kinetik@flim.org>)
* s/EINVAL/WSAEINVAL/ fix trivial compile failure
* s/socket/WSASocket/ enable creation of sockets suitable as stdin/stdout
* of a child process.
* add argument make_overlapped
*/
#include <string.h>
#ifdef WIN32
#include <ws2tcpip.h> /* socklen_t, et al (MSVC20xx) */
#include <windows.h>
#include <io.h>
#else
#ifdef _WIN32
#include <Win32_Interop/win32_types.h>
#include <Win32_Interop/Win32_FDAPI.h>
#endif
#include <sys/types.h>
#include <sys/socket.h>
#include <errno.h>
#endif
#ifdef WIN32
/* dumb_socketpair:
* If make_overlapped is nonzero, both sockets created will be usable for
* "overlapped" operations via WSASend etc. If make_overlapped is zero,
* socks[0] (only) will be usable with regular ReadFile etc., and thus
* suitable for use as stdin or stdout of a child process. Note that the
* sockets must be closed with closesocket() regardless.
*/
int dumb_socketpair(SOCKET socks[2]) {
union {
struct sockaddr_in inaddr;
struct sockaddr addr;
} a;
SOCKET listener;
int e;
socklen_t addrlen = sizeof(a.inaddr);
int reuse = 1;
if (socks == 0) {
return SOCKET_ERROR;
}
socks[0] = socks[1] = -1;
listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (listener == -1)
return SOCKET_ERROR;
memset(&a, 0, sizeof(a));
a.inaddr.sin_family = AF_INET;
a.inaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
a.inaddr.sin_port = 0;
for (;;) {
if (setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse,
(socklen_t) sizeof(reuse)) == -1)
break;
if (bind(listener, &a.addr, sizeof(a.inaddr)) == SOCKET_ERROR)
break;
memset(&a, 0, sizeof(a));
if (getsockname(listener, &a.addr, &addrlen) == SOCKET_ERROR)
break;
// win32 getsockname may only set the port number, p=0.0005.
// ( http://msdn.microsoft.com/library/ms738543.aspx ):
a.inaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
a.inaddr.sin_family = AF_INET;
if (listen(listener, 1) == SOCKET_ERROR)
break;
socks[0] = FDAPI_WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, 0);
if (socks[0] == -1)
break;
if (connect(socks[0], &a.addr, sizeof(a.inaddr)) == SOCKET_ERROR)
break;
socks[1] = accept(listener, NULL, NULL);
if (socks[1] == -1)
break;
FDAPI_close(listener);
return 0;
}
FDAPI_close(listener);
FDAPI_close(socks[0]);
FDAPI_close(socks[1]);
socks[0] = socks[1] = -1;
return SOCKET_ERROR;
}
#else
int dumb_socketpair(int socks[2], int dummy) {
if (socks == 0) {
errno = EINVAL;
return -1;
}
dummy = socketpair(AF_LOCAL, SOCK_STREAM, 0, socks);
if (dummy)
socks[0] = socks[1] = -1;
return dummy;
}
#endif
src/common/shims/windows/strings.h
-4
View File
@@ -1,4 +0,0 @@
#ifndef STRINGS_H
#define STRINGS_H
#endif /* STRINGS_H */
src/common/shims/windows/sys/ioctl.h
-4
View File
@@ -1,4 +0,0 @@
#ifndef IOCTL_H
#define IOCTL_H
#endif /* IOCTL_H */
src/common/shims/windows/sys/mman.h
-36
View File
@@ -1,36 +0,0 @@
#ifndef MMAN_H
#define MMAN_H
#include <unistd.h>
#define MAP_SHARED 0x0010 /* share changes */
#define MAP_FAILED ((void *) -1)
#define PROT_READ 0x04 /* pages can be read */
#define PROT_WRITE 0x02 /* pages can be written */
#define PROT_EXEC 0x01 /* pages can be executed */
static void *mmap(void *addr,
size_t len,
int prot,
int flags,
int fd,
off_t off) {
void *result = (void *) (-1);
if (!addr && prot == MAP_SHARED) {
/* HACK: we're assuming handle sizes can't exceed 32 bits, which is wrong...
* but works for now. */
void *ptr = MapViewOfFile((HANDLE)(intptr_t) fd, FILE_MAP_ALL_ACCESS,
(DWORD)(off >> (CHAR_BIT * sizeof(DWORD))),
(DWORD) off, (SIZE_T) len);
if (ptr) {
result = ptr;
}
}
return result;
}
static int munmap(void *addr, size_t length) {
(void) length;
return UnmapViewOfFile(addr) ? 0 : -1;
}
#endif /* MMAN_H */
src/common/shims/windows/sys/select.h
-4
View File
@@ -1,4 +0,0 @@
#ifndef SELECT_H
#define SELECT_H
#endif /* SELECT_H */
src/common/shims/windows/sys/socket.h
-36
View File
@@ -1,36 +0,0 @@
#ifndef SOCKET_H
#define SOCKET_H
typedef unsigned short sa_family_t;
#include "../../src/Win32_Interop/Win32_FDAPI.h"
#include "../../src/Win32_Interop/Win32_APIs.h"
#define cmsghdr _WSACMSGHDR
#undef CMSG_DATA
#define CMSG_DATA WSA_CMSG_DATA
#define CMSG_SPACE WSA_CMSG_SPACE
#define CMSG_FIRSTHDR WSA_CMSG_FIRSTHDR
#define CMSG_LEN WSA_CMSG_LEN
#define CMSG_NXTHDR WSA_CMSG_NXTHDR
#define SCM_RIGHTS 1
#define iovec _WSABUF
#define iov_base buf
#define iov_len len
#define msghdr _WSAMSG
#define msg_name name
#define msg_namelen namelen
#define msg_iov lpBuffers
#define msg_iovlen dwBufferCount
#define msg_control Control.buf
#define msg_controllen Control.len
#define msg_flags dwFlags
int dumb_socketpair(SOCKET socks[2]);
ssize_t sendmsg(int sockfd, struct msghdr *msg, int flags);
ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
int socketpair(int domain, int type, int protocol, int sv[2]);
#endif /* SOCKET_H */
src/common/shims/windows/sys/time.h
-12
View File
@@ -1,12 +0,0 @@
#ifndef TIME_H
#define TIME_H
#include <WinSock2.h> /* timeval */
int gettimeofday_highres(struct timeval *tv, struct timezone *tz);
static int gettimeofday(struct timeval *tv, struct timezone *tz) {
return gettimeofday_highres(tv, tz);
}
#endif /* TIME_H */
src/common/shims/windows/sys/un.h
-13
View File
@@ -1,13 +0,0 @@
#ifndef UN_H
#define UN_H
#include <sys/socket.h>
struct sockaddr_un {
/** AF_UNIX. */
sa_family_t sun_family;
/** The pathname. */
char sun_path[108];
};
#endif /* UN_H */
src/common/shims/windows/sys/wait.h
-4
View File
@@ -1,4 +0,0 @@
#ifndef WAIT_H
#define WAIT_H
#endif /* WAIT_H */
src/common/shims/windows/unistd.h
-11
View File
@@ -1,11 +0,0 @@
#ifndef UNISTD_H
#define UNISTD_H
extern char *optarg;
extern int optind, opterr, optopt;
int getopt(int nargc, char *const nargv[], const char *ostr);
#include "../../src/Win32_Interop/Win32_FDAPI.h"
#define close(...) FDAPI_close(__VA_ARGS__)
#endif /* UNISTD_H */
src/common/state/actor_notification_table.cc
-47
View File
@@ -1,47 +0,0 @@
#include "actor_notification_table.h"
#include "common_protocol.h"
#include "redis.h"
void publish_actor_creation_notification(DBHandle *db_handle,
const ActorID &actor_id,
const WorkerID &driver_id,
const DBClientID &local_scheduler_id) {
// Create a flatbuffer object to serialize and publish.
flatbuffers::FlatBufferBuilder fbb;
// Create the flatbuffers message.
auto message = CreateActorCreationNotification(
fbb, to_flatbuf(fbb, actor_id), to_flatbuf(fbb, driver_id),
to_flatbuf(fbb, local_scheduler_id));
fbb.Finish(message);
ActorCreationNotificationData *data =
(ActorCreationNotificationData *) malloc(
sizeof(ActorCreationNotificationData) + fbb.GetSize());
data->size = fbb.GetSize();
memcpy(&data->flatbuffer_data[0], fbb.GetBufferPointer(), fbb.GetSize());
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(data), NULL, NULL,
redis_publish_actor_creation_notification, NULL);
}
void actor_notification_table_subscribe(
DBHandle *db_handle,
actor_notification_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry) {
ActorNotificationTableSubscribeData *sub_data =
(ActorNotificationTableSubscribeData *) malloc(
sizeof(ActorNotificationTableSubscribeData));
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(sub_data), retry, NULL,
redis_actor_notification_table_subscribe, NULL);
}
void actor_table_mark_removed(DBHandle *db_handle, ActorID actor_id) {
redis_actor_table_mark_removed(db_handle, actor_id);
}
src/common/state/actor_notification_table.h
-74
View File
@@ -1,74 +0,0 @@
#ifndef ACTOR_NOTIFICATION_TABLE_H
#define ACTOR_NOTIFICATION_TABLE_H
#include "task.h"
#include "db.h"
#include "table.h"
/*
* ==== Subscribing to the actor notification table ====
*/
/* Callback for subscribing to the local scheduler table. */
typedef void (*actor_notification_table_subscribe_callback)(
const ActorID &actor_id,
const WorkerID &driver_id,
const DBClientID &local_scheduler_id,
void *user_context);
/// Publish an actor creation notification. This is published by a local
/// scheduler once it creates an actor.
///
/// \param db_handle Database handle.
/// \param actor_id The ID of the actor that was created.
/// \param driver_id The ID of the driver that created the actor.
/// \param local_scheduler_id The ID of the local scheduler that created the
/// actor.
/// \return Void.
void publish_actor_creation_notification(DBHandle *db_handle,
const ActorID &actor_id,
const WorkerID &driver_id,
const DBClientID &local_scheduler_id);
/// Data that is needed to publish an actor creation notification.
typedef struct {
/// The size of the flatbuffer object.
int64_t size;
/// The information to be sent.
uint8_t flatbuffer_data[0];
} ActorCreationNotificationData;
/**
* Register a callback to process actor notification events.
*
* @param db_handle Database handle.
* @param subscribe_callback Callback that will be called when the local
* scheduler event happens.
* @param subscribe_context Context that will be passed into the
* subscribe_callback.
* @param retry Information about retrying the request to the database.
* @return Void.
*/
void actor_notification_table_subscribe(
DBHandle *db_handle,
actor_notification_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry);
/* Data that is needed to register local scheduler table subscribe callbacks
* with the state database. */
typedef struct {
actor_notification_table_subscribe_callback subscribe_callback;
void *subscribe_context;
} ActorNotificationTableSubscribeData;
/**
* Marks an actor as removed. This prevents the actor from being resurrected.
*
* @param db The database handle.
* @param actor_id The actor id to mark as removed.
* @return Void.
*/
void actor_table_mark_removed(DBHandle *db_handle, ActorID actor_id);
#endif /* ACTOR_NOTIFICATION_TABLE_H */
src/common/state/db.h
-70
View File
@@ -1,70 +0,0 @@
#ifndef DB_H
#define DB_H
#include <vector>
#include "common.h"
#include "event_loop.h"
typedef struct DBHandle DBHandle;
/**
* Connect to the global system store.
*
* @param db_address The hostname to use to connect to the database.
* @param db_port The port to use to connect to the database.
* @param db_shards_addresses The list of database shard IP addresses.
* @param db_shards_ports The list of database shard ports, in the same order
* as db_shards_addresses.
* @param client_type The type of this client.
* @param node_ip_address The hostname of the client that is connecting.
* @param args A vector of extra arguments strings. They should alternate
* between the name of the argument and the value of the argument. For
* examples: "port", "1234", "socket_name", "/tmp/s1". This vector should
* have an even length.
* @return This returns a handle to the database, which must be freed with
* db_disconnect after use.
*/
DBHandle *db_connect(const std::string &db_primary_address,
int db_primary_port,
const char *client_type,
const char *node_ip_address,
const std::vector<std::string> &args);
/**
* Attach global system store connection to an event loop. Callbacks from
* queries to the global system store will trigger events in the event loop.
*
* @param db The handle to the database that is connected.
* @param loop The event loop the database gets connected to.
* @param reattach Can only be true in unit tests. If true, the database is
* reattached to the loop.
* @return Void.
*/
void db_attach(DBHandle *db, event_loop *loop, bool reattach);
/**
* Disconnect from the global system store.
*
* @param db The database connection to close and clean up.
* @return Void.
*/
void db_disconnect(DBHandle *db);
/**
* Free the database handle.
*
* @param db The database connection to clean up.
* @return Void.
*/
void DBHandle_free(DBHandle *db);
/**
* Returns the db client ID.
*
* @param db The handle to the database.
* @returns int The db client ID for this connection to the database.
*/
DBClientID get_db_client_id(DBHandle *db);
#endif
src/common/state/db_client_table.cc
-90
View File
@@ -1,90 +0,0 @@
#include "db_client_table.h"
#include "redis.h"
void db_client_table_remove(DBHandle *db_handle,
DBClientID db_client_id,
RetryInfo *retry,
db_client_table_done_callback done_callback,
void *user_context) {
init_table_callback(db_handle, db_client_id, __func__,
new CommonCallbackData(NULL), retry,
(table_done_callback) done_callback,
redis_db_client_table_remove, user_context);
}
void db_client_table_subscribe(
DBHandle *db_handle,
db_client_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry,
db_client_table_done_callback done_callback,
void *user_context) {
DBClientTableSubscribeData *sub_data =
(DBClientTableSubscribeData *) malloc(sizeof(DBClientTableSubscribeData));
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(sub_data), retry,
(table_done_callback) done_callback,
redis_db_client_table_subscribe, user_context);
}
const std::vector<std::string> db_client_table_get_ip_addresses(
DBHandle *db_handle,
const std::vector<DBClientID> &manager_ids) {
/* We time this function because in the past this loop has taken multiple
* seconds under stressful situations on hundreds of machines causing the
* plasma manager to die (because it went too long without sending
* heartbeats). */
int64_t start_time = current_time_ms();
/* Construct the manager vector from the flatbuffers object. */
std::vector<std::string> manager_vector;
for (auto const &manager_id : manager_ids) {
DBClient client = redis_cache_get_db_client(db_handle, manager_id);
RAY_CHECK(!client.manager_address.empty());
if (client.is_alive) {
manager_vector.push_back(client.manager_address);
}
}
int64_t end_time = current_time_ms();
if (end_time - start_time > RayConfig::instance().max_time_for_loop()) {
RAY_LOG(WARNING) << "calling redis_get_cached_db_client in a loop in with "
<< manager_ids.size() << " manager IDs took "
<< end_time - start_time << " milliseconds.";
}
return manager_vector;
}
void db_client_table_update_cache_callback(DBClient *db_client,
void *user_context) {
DBHandle *db_handle = (DBHandle *) user_context;
redis_cache_set_db_client(db_handle, *db_client);
}
void db_client_table_cache_init(DBHandle *db_handle) {
db_client_table_subscribe(db_handle, db_client_table_update_cache_callback,
db_handle, NULL, NULL, NULL);
}
DBClient db_client_table_cache_get(DBHandle *db_handle, DBClientID client_id) {
RAY_CHECK(!client_id.is_nil());
return redis_cache_get_db_client(db_handle, client_id);
}
void plasma_manager_send_heartbeat(DBHandle *db_handle) {
RetryInfo heartbeat_retry;
heartbeat_retry.num_retries = 0;
heartbeat_retry.timeout =
RayConfig::instance().heartbeat_timeout_milliseconds();
heartbeat_retry.fail_callback = NULL;
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(NULL),
(RetryInfo *) &heartbeat_retry, NULL,
redis_plasma_manager_send_heartbeat, NULL);
}
src/common/state/db_client_table.h
-120
View File
@@ -1,120 +0,0 @@
#ifndef DB_CLIENT_TABLE_H
#define DB_CLIENT_TABLE_H
#include <vector>
#include "db.h"
#include "table.h"
typedef void (*db_client_table_done_callback)(DBClientID db_client_id,
void *user_context);
/**
* Remove a client from the db clients table.
*
* @param db_handle Database handle.
* @param db_client_id The database client ID to remove.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Data that will be passed to done_callback and
* fail_callback.
* @return Void.
*
*/
void db_client_table_remove(DBHandle *db_handle,
DBClientID db_client_id,
RetryInfo *retry,
db_client_table_done_callback done_callback,
void *user_context);
/*
* ==== Subscribing to the db client table ====
*/
/* An entry in the db client table. */
typedef struct {
/** The database client ID. */
DBClientID id;
/** The database client type. */
std::string client_type;
/** An optional auxiliary address for the plasma manager associated with this
* database client. */
std::string manager_address;
/** Whether or not the database client exists. If this is false for an entry,
* then it will never again be true. */
bool is_alive;
} DBClient;
/* Callback for subscribing to the db client table. */
typedef void (*db_client_table_subscribe_callback)(DBClient *db_client,
void *user_context);
/**
* Register a callback for a db client table event.
*
* @param db_handle Database handle.
* @param subscribe_callback Callback that will be called when the db client
* table is updated.
* @param subscribe_context Context that will be passed into the
* subscribe_callback.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Data that will be passed to done_callback and
* fail_callback.
* @return Void.
*/
void db_client_table_subscribe(
DBHandle *db_handle,
db_client_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry,
db_client_table_done_callback done_callback,
void *user_context);
/* Data that is needed to register db client table subscribe callbacks with the
* state database. */
typedef struct {
db_client_table_subscribe_callback subscribe_callback;
void *subscribe_context;
} DBClientTableSubscribeData;
const std::vector<std::string> db_client_table_get_ip_addresses(
DBHandle *db,
const std::vector<DBClientID> &manager_ids);
/**
* Initialize the db client cache. The cache is updated with each notification
* from the db client table.
*
* @param db_handle Database handle.
* @return Void.
*/
void db_client_table_cache_init(DBHandle *db_handle);
/**
* Get a db client from the cache. If the requested client is not there,
* request the latest entry from the db client table.
*
* @param db_handle Database handle.
* @param client_id The ID of the client to look up in the cache.
* @return The database client in the cache.
*/
DBClient db_client_table_cache_get(DBHandle *db_handle, DBClientID client_id);
/*
* ==== Plasma manager heartbeats ====
*/
/**
* Start sending heartbeats to the plasma_managers channel. Each
* heartbeat contains this database client's ID. Heartbeats can be subscribed
* to through the plasma_managers channel. Once called, this "retries" the
* heartbeat operation forever, every heartbeat_timeout_milliseconds
* milliseconds.
*
* @param db_handle Database handle.
* @return Void.
*/
void plasma_manager_send_heartbeat(DBHandle *db_handle);
#endif /* DB_CLIENT_TABLE_H */
src/common/state/driver_table.cc
-23
View File
@@ -1,23 +0,0 @@
#include "driver_table.h"
#include "redis.h"
void driver_table_subscribe(DBHandle *db_handle,
driver_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry) {
DriverTableSubscribeData *sub_data =
(DriverTableSubscribeData *) malloc(sizeof(DriverTableSubscribeData));
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(sub_data), retry, NULL,
redis_driver_table_subscribe, NULL);
}
void driver_table_send_driver_death(DBHandle *db_handle,
WorkerID driver_id,
RetryInfo *retry) {
init_table_callback(db_handle, driver_id, __func__,
new CommonCallbackData(NULL), retry, NULL,
redis_driver_table_send_driver_death, NULL);
}
src/common/state/driver_table.h
-50
View File
@@ -1,50 +0,0 @@
#ifndef DRIVER_TABLE_H
#define DRIVER_TABLE_H
#include "db.h"
#include "table.h"
#include "task.h"
/*
* ==== Subscribing to the driver table ====
*/
/* Callback for subscribing to the driver table. */
typedef void (*driver_table_subscribe_callback)(WorkerID driver_id,
void *user_context);
/**
* Register a callback for a driver table event.
*
* @param db_handle Database handle.
* @param subscribe_callback Callback that will be called when the driver event
* happens.
* @param subscribe_context Context that will be passed into the
* subscribe_callback.
* @param retry Information about retrying the request to the database.
* @return Void.
*/
void driver_table_subscribe(DBHandle *db_handle,
driver_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry);
/* Data that is needed to register driver table subscribe callbacks with the
* state database. */
typedef struct {
driver_table_subscribe_callback subscribe_callback;
void *subscribe_context;
} DriverTableSubscribeData;
/**
* Send driver death update to all subscribers.
*
* @param db_handle Database handle.
* @param driver_id The ID of the driver that died.
* @param retry Information about retrying the request to the database.
*/
void driver_table_send_driver_death(DBHandle *db_handle,
WorkerID driver_id,
RetryInfo *retry);
#endif /* DRIVER_TABLE_H */
src/common/state/error_table.cc
-24
View File
@@ -1,24 +0,0 @@
#include "error_table.h"
#include "redis.h"
const char *error_types[] = {"object_hash_mismatch", "put_reconstruction",
"worker_died", "actor_not_created"};
void push_error(DBHandle *db_handle,
DBClientID driver_id,
ErrorIndex error_type,
const std::string &error_message) {
int64_t message_size = error_message.size();
/* Allocate a struct to hold the error information. */
ErrorInfo *info = (ErrorInfo *) malloc(sizeof(ErrorInfo) + message_size);
info->driver_id = driver_id;
info->error_type = error_type;
info->error_key = UniqueID::from_random();
info->size = message_size;
memcpy(info->error_message, error_message.data(), message_size);
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(info), NULL, NULL,
redis_push_error, NULL);
}
src/common/state/error_table.h
-50
View File
@@ -1,50 +0,0 @@
#ifndef ERROR_TABLE_H
#define ERROR_TABLE_H
#include "db.h"
#include "table.h"
/// An ErrorIndex may be used as an index into error_types.
enum class ErrorIndex : int32_t {
/// An object was added with a different hash from the existing one.
OBJECT_HASH_MISMATCH = 0,
/// An object that was created through a ray.put is lost.
PUT_RECONSTRUCTION,
/// A worker died or was killed while executing a task.
WORKER_DIED,
/// An actor hasn't been created for a while.
ACTOR_NOT_CREATED,
/// The total number of error types.
MAX
};
/// Data that is needed to push an error.
typedef struct {
/// The ID of the driver to push the error to.
DBClientID driver_id;
/// An index into the error_types array indicating the type of the error.
ErrorIndex error_type;
/// The key to use for the error message in Redis.
UniqueID error_key;
/// The length of the error message.
int64_t size;
/// The error message.
uint8_t error_message[0];
} ErrorInfo;
extern const char *error_types[];
/// Push an error to the given Python driver.
///
/// \param db_handle Database handle.
/// \param driver_id The ID of the Python driver to push the error to.
/// \param error_type An index specifying the type of the error. This should
/// be a value from the ErrorIndex enum.
/// \param error_message The error message to print.
/// \return Void.
void push_error(DBHandle *db_handle,
DBClientID driver_id,
ErrorIndex error_type,
const std::string &error_message);
#endif
src/common/state/local_scheduler_table.cc
-48
View File
@@ -1,48 +0,0 @@
#include "local_scheduler_table.h"
#include "common_protocol.h"
#include "redis.h"
void local_scheduler_table_subscribe(
DBHandle *db_handle,
local_scheduler_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry) {
LocalSchedulerTableSubscribeData *sub_data =
(LocalSchedulerTableSubscribeData *) malloc(
sizeof(LocalSchedulerTableSubscribeData));
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(sub_data), retry, NULL,
redis_local_scheduler_table_subscribe, NULL);
}
void local_scheduler_table_send_info(DBHandle *db_handle,
LocalSchedulerInfo *info,
RetryInfo *retry) {
/* Create a flatbuffer object to serialize and publish. */
flatbuffers::FlatBufferBuilder fbb;
/* Create the flatbuffers message. */
auto message = CreateLocalSchedulerInfoMessage(
fbb, to_flatbuf(fbb, db_handle->client), info->total_num_workers,
info->task_queue_length, info->available_workers,
map_to_flatbuf(fbb, info->static_resources),
map_to_flatbuf(fbb, info->dynamic_resources), false);
fbb.Finish(message);
LocalSchedulerTableSendInfoData *data =
(LocalSchedulerTableSendInfoData *) malloc(
sizeof(LocalSchedulerTableSendInfoData) + fbb.GetSize());
data->size = fbb.GetSize();
memcpy(&data->flatbuffer_data[0], fbb.GetBufferPointer(), fbb.GetSize());
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(data), retry, NULL,
redis_local_scheduler_table_send_info, NULL);
}
void local_scheduler_table_disconnect(DBHandle *db_handle) {
redis_local_scheduler_table_disconnect(db_handle);
}
src/common/state/local_scheduler_table.h
-98
View File
@@ -1,98 +0,0 @@
#ifndef LOCAL_SCHEDULER_TABLE_H
#define LOCAL_SCHEDULER_TABLE_H
#include <unordered_map>
#include "db.h"
#include "table.h"
#include "task.h"
/** This struct is sent with heartbeat messages from the local scheduler to the
* global scheduler, and it contains information about the load on the local
* scheduler. */
typedef struct {
/** The total number of workers that are connected to this local scheduler. */
int total_num_workers;
/** The number of tasks queued in this local scheduler. */
int task_queue_length;
/** The number of workers that are available and waiting for tasks. */
int available_workers;
/** The resource vector of resources generally available to this local
* scheduler. */
std::unordered_map<std::string, double> static_resources;
/** The resource vector of resources currently available to this local
* scheduler. */
std::unordered_map<std::string, double> dynamic_resources;
/** Whether the local scheduler is dead. If true, then all other fields
* should be ignored. */
bool is_dead;
} LocalSchedulerInfo;
/*
* ==== Subscribing to the local scheduler table ====
*/
/* Callback for subscribing to the local scheduler table. */
typedef void (*local_scheduler_table_subscribe_callback)(
DBClientID client_id,
LocalSchedulerInfo info,
void *user_context);
/**
* Register a callback for a local scheduler table event.
*
* @param db_handle Database handle.
* @param subscribe_callback Callback that will be called when the local
* scheduler event happens.
* @param subscribe_context Context that will be passed into the
* subscribe_callback.
* @param retry Information about retrying the request to the database.
* @return Void.
*/
void local_scheduler_table_subscribe(
DBHandle *db_handle,
local_scheduler_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry);
/* Data that is needed to register local scheduler table subscribe callbacks
* with the state database. */
typedef struct {
local_scheduler_table_subscribe_callback subscribe_callback;
void *subscribe_context;
} LocalSchedulerTableSubscribeData;
/**
* Send a heartbeat to all subscribers to the local scheduler table. This
* heartbeat contains some information about the load on the local scheduler.
*
* @param db_handle Database handle.
* @param info Information about the local scheduler, including the load on the
* local scheduler.
* @param retry Information about retrying the request to the database.
* @return Void.
*/
void local_scheduler_table_send_info(DBHandle *db_handle,
LocalSchedulerInfo *info,
RetryInfo *retry);
/* Data that is needed to publish local scheduler heartbeats to the local
* scheduler table. */
typedef struct {
/* The size of the flatbuffer object. */
int64_t size;
/* The information to be sent. */
uint8_t flatbuffer_data[0];
} LocalSchedulerTableSendInfoData;
/**
* Send a null heartbeat to all subscribers to the local scheduler table to
* notify them that we are about to exit. This operation is performed
* synchronously.
*
* @param db_handle Database handle.
* @return Void.
*/
void local_scheduler_table_disconnect(DBHandle *db_handle);
#endif /* LOCAL_SCHEDULER_TABLE_H */
src/common/state/object_table.cc
-119
View File
@@ -1,119 +0,0 @@
#include "object_table.h"
#include "redis.h"
void object_table_lookup(DBHandle *db_handle,
ObjectID object_id,
RetryInfo *retry,
object_table_lookup_done_callback done_callback,
void *user_context) {
RAY_CHECK(db_handle != NULL);
init_table_callback(db_handle, object_id, __func__,
new CommonCallbackData(NULL), retry,
(table_done_callback) done_callback,
redis_object_table_lookup, user_context);
}
void object_table_add(DBHandle *db_handle,
ObjectID object_id,
int64_t object_size,
unsigned char digest[],
RetryInfo *retry,
object_table_done_callback done_callback,
void *user_context) {
RAY_CHECK(db_handle != NULL);
ObjectTableAddData *info =
(ObjectTableAddData *) malloc(sizeof(ObjectTableAddData));
info->object_size = object_size;
memcpy(&info->digest[0], digest, DIGEST_SIZE);
init_table_callback(db_handle, object_id, __func__,
new CommonCallbackData(info), retry,
(table_done_callback) done_callback,
redis_object_table_add, user_context);
}
void object_table_remove(DBHandle *db_handle,
ObjectID object_id,
DBClientID *client_id,
RetryInfo *retry,
object_table_done_callback done_callback,
void *user_context) {
RAY_CHECK(db_handle != NULL);
/* Copy the client ID, if one was provided. */
DBClientID *client_id_copy = NULL;
if (client_id != NULL) {
client_id_copy = (DBClientID *) malloc(sizeof(DBClientID));
*client_id_copy = *client_id;
}
init_table_callback(db_handle, object_id, __func__,
new CommonCallbackData(client_id_copy), retry,
(table_done_callback) done_callback,
redis_object_table_remove, user_context);
}
void object_table_subscribe_to_notifications(
DBHandle *db_handle,
bool subscribe_all,
object_table_object_available_callback object_available_callback,
void *subscribe_context,
RetryInfo *retry,
object_table_lookup_done_callback done_callback,
void *user_context) {
RAY_CHECK(db_handle != NULL);
ObjectTableSubscribeData *sub_data =
(ObjectTableSubscribeData *) malloc(sizeof(ObjectTableSubscribeData));
sub_data->object_available_callback = object_available_callback;
sub_data->subscribe_context = subscribe_context;
sub_data->subscribe_all = subscribe_all;
init_table_callback(
db_handle, ObjectID::nil(), __func__, new CommonCallbackData(sub_data),
retry, (table_done_callback) done_callback,
redis_object_table_subscribe_to_notifications, user_context);
}
void object_table_request_notifications(DBHandle *db_handle,
int num_object_ids,
ObjectID object_ids[],
RetryInfo *retry) {
RAY_CHECK(db_handle != NULL);
RAY_CHECK(num_object_ids > 0);
ObjectTableRequestNotificationsData *data =
(ObjectTableRequestNotificationsData *) malloc(
sizeof(ObjectTableRequestNotificationsData) +
num_object_ids * sizeof(ObjectID));
data->num_object_ids = num_object_ids;
memcpy(data->object_ids, object_ids, num_object_ids * sizeof(ObjectID));
init_table_callback(db_handle, ObjectID::nil(), __func__,
new CommonCallbackData(data), retry, NULL,
redis_object_table_request_notifications, NULL);
}
void result_table_add(DBHandle *db_handle,
ObjectID object_id,
TaskID task_id,
bool is_put,
RetryInfo *retry,
result_table_done_callback done_callback,
void *user_context) {
ResultTableAddInfo *info =
(ResultTableAddInfo *) malloc(sizeof(ResultTableAddInfo));
info->task_id = task_id;
info->is_put = is_put;
init_table_callback(db_handle, object_id, __func__,
new CommonCallbackData(info), retry,
(table_done_callback) done_callback,
redis_result_table_add, user_context);
}
void result_table_lookup(DBHandle *db_handle,
ObjectID object_id,
RetryInfo *retry,
result_table_lookup_callback done_callback,
void *user_context) {
init_table_callback(db_handle, object_id, __func__,
new CommonCallbackData(NULL), retry,
(table_done_callback) done_callback,
redis_result_table_lookup, user_context);
}
src/common/state/object_table.h
-242
View File
@@ -1,242 +0,0 @@
#ifndef OBJECT_TABLE_H
#define OBJECT_TABLE_H
#include "common.h"
#include "table.h"
#include "db.h"
#include "task.h"
/*
* ==== Lookup call and callback ====
*/
/* Callback called when the lookup completes. The callback should free
* the manager_vector array, but NOT the strings they are pointing to. If there
* was no entry at all for the object (the object had never been created
* before), then never_created will be true.
*/
typedef void (*object_table_lookup_done_callback)(
ObjectID object_id,
bool never_created,
const std::vector<DBClientID> &manager_ids,
void *user_context);
/* Callback called when object ObjectID is available. */
typedef void (*object_table_object_available_callback)(
ObjectID object_id,
int64_t data_size,
const std::vector<DBClientID> &manager_ids,
void *user_context);
/**
* Return the list of nodes storing object_id in their plasma stores.
*
* @param db_handle Handle to object_table database.
* @param object_id ID of the object being looked up.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Context passed by the caller.
* @return Void.
*/
void object_table_lookup(DBHandle *db_handle,
ObjectID object_id,
RetryInfo *retry,
object_table_lookup_done_callback done_callback,
void *user_context);
/*
* ==== Add object call and callback ====
*/
/**
* Callback called when the object add/remove operation completes.
*
* @param object_id The ID of the object that was added or removed.
* @param success Whether the operation was successful or not. If this is false
* and the operation was an addition, the object was added, but there
* was a hash mismatch.
* @param user_context The user context that was passed into the add/remove
* call.
*/
typedef void (*object_table_done_callback)(ObjectID object_id,
bool success,
void *user_context);
/**
* Add the plasma manager that created the db_handle to the
* list of plasma managers that have the object_id.
*
* @param db_handle Handle to db.
* @param object_id Object unique identifier.
* @param data_size Object data size.
* @param retry Information about retrying the request to the database.
* @param done_callback Callback to be called when lookup completes.
* @param user_context User context to be passed in the callbacks.
* @return Void.
*/
void object_table_add(DBHandle *db_handle,
ObjectID object_id,
int64_t object_size,
unsigned char digest[],
RetryInfo *retry,
object_table_done_callback done_callback,
void *user_context);
/** Data that is needed to add new objects to the object table. */
typedef struct {
int64_t object_size;
unsigned char digest[DIGEST_SIZE];
} ObjectTableAddData;
/*
* ==== Remove object call and callback ====
*/
/**
* Object remove function.
*
* @param db_handle Handle to db.
* @param object_id Object unique identifier.
* @param client_id A pointer to the database client ID to remove. If this is
* set to NULL, then the client ID associated with db_handle will be
* removed.
* @param retry Information about retrying the request to the database.
* @param done_callback Callback to be called when lookup completes.
* @param user_context User context to be passed in the callbacks.
* @return Void.
*/
void object_table_remove(DBHandle *db_handle,
ObjectID object_id,
DBClientID *client_id,
RetryInfo *retry,
object_table_done_callback done_callback,
void *user_context);
/*
* ==== Subscribe to be announced when new object available ====
*/
/**
* Set up a client-specific channel for receiving notifications about available
* objects from the object table. The callback will be called once per
* notification received on this channel.
*
* @param db_handle Handle to db.
* @param object_available_callback Callback to be called when new object
* becomes available.
* @param subscribe_context Caller context which will be passed to the
* object_available_callback.
* @param retry Information about retrying the request to the database.
* @param done_callback Callback to be called when subscription is installed.
* This is only used for the tests.
* @param user_context User context to be passed into the done callback. This is
* only used for the tests.
* @return Void.
*/
void object_table_subscribe_to_notifications(
DBHandle *db_handle,
bool subscribe_all,
object_table_object_available_callback object_available_callback,
void *subscribe_context,
RetryInfo *retry,
object_table_lookup_done_callback done_callback,
void *user_context);
/**
* Request notifications about the availability of some objects from the object
* table. The notifications will be published to this client's object
* notification channel, which was set up by the method
* object_table_subscribe_to_notifications.
*
* @param db_handle Handle to db.
* @param object_ids The object IDs to receive notifications about.
* @param retry Information about retrying the request to the database.
* @return Void.
*/
void object_table_request_notifications(DBHandle *db,
int num_object_ids,
ObjectID object_ids[],
RetryInfo *retry);
/** Data that is needed to run object_request_notifications requests. */
typedef struct {
/** The number of object IDs. */
int num_object_ids;
/** This field is used to store a variable number of object IDs. */
ObjectID object_ids[0];
} ObjectTableRequestNotificationsData;
/** Data that is needed to register new object available callbacks with the
* state database. */
typedef struct {
bool subscribe_all;
object_table_object_available_callback object_available_callback;
void *subscribe_context;
} ObjectTableSubscribeData;
/*
* ==== Result table ====
*/
/**
* Callback called when the add/remove operation for a result table entry
* completes. */
typedef void (*result_table_done_callback)(ObjectID object_id,
void *user_context);
/** Information about a result table entry to add. */
typedef struct {
/** The task ID of the task that created the requested object. */
TaskID task_id;
/** True if the object was created through a put, and false if created by
* return value. */
bool is_put;
} ResultTableAddInfo;
/**
* Add information about a new object to the object table. This
* is immutable information like the ID of the task that
* created the object.
*
* @param db_handle Handle to object_table database.
* @param object_id ID of the object to add.
* @param task_id ID of the task that creates this object.
* @param is_put A boolean that is true if the object was created through a
* ray.put, and false if the object was created by return value.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Context passed by the caller.
* @return Void.
*/
void result_table_add(DBHandle *db_handle,
ObjectID object_id,
TaskID task_id,
bool is_put,
RetryInfo *retry,
result_table_done_callback done_callback,
void *user_context);
/** Callback called when the result table lookup completes. */
typedef void (*result_table_lookup_callback)(ObjectID object_id,
TaskID task_id,
bool is_put,
void *user_context);
/**
* Lookup the task that created an object in the result table. The return value
* is the task ID.
*
* @param db_handle Handle to object_table database.
* @param object_id ID of the object to lookup.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Context passed by the caller.
* @return Void.
*/
void result_table_lookup(DBHandle *db_handle,
ObjectID object_id,
RetryInfo *retry,
result_table_lookup_callback done_callback,
void *user_context);
#endif /* OBJECT_TABLE_H */
src/common/state/redis.cc
-1692
View File
File diff suppressed because it is too large Load Diff
src/common/state/redis.h
-356
View File
@@ -1,356 +0,0 @@
#ifndef REDIS_H
#define REDIS_H
#include <unordered_map>
#include "db.h"
#include "db_client_table.h"
#include "object_table.h"
#include "task_table.h"
#include "hiredis/hiredis.h"
#include "hiredis/async.h"
#define LOG_REDIS_ERROR(context, M, ...) \
RAY_LOG(ERROR) << "Redis error " << context->err << " " << context->errstr \
<< "; " << M
#define LOG_REDIS_DEBUG(context, M, ...) \
RAY_LOG(DEBUG) << "Redis error " << context->err << " " << context->errstr \
<< "; " << M;
struct DBHandle {
/** String that identifies this client type. */
char *client_type;
/** Unique ID for this client. */
DBClientID client;
/** Primary redis context for all non-subscribe connections. This is used for
* the database client table, heartbeats, and errors that should be pushed to
* the driver. */
redisAsyncContext *context;
/** Primary redis context for "subscribe" communication. A separate context
* is needed for this communication (see
* https://github.com/redis/hiredis/issues/55). This is used for the
* database client table, heartbeats, and errors that should be pushed to
* the driver. */
redisAsyncContext *subscribe_context;
/** Redis contexts for shards for all non-subscribe connections. All requests
* to the object table, task table, and event table should be directed here.
* The correct shard can be retrieved using get_redis_context below. */
std::vector<redisAsyncContext *> contexts;
/** Redis contexts for shards for "subscribe" communication. All requests
* to the object table, task table, and event table should be directed here.
* The correct shard can be retrieved using get_redis_context below. */
std::vector<redisAsyncContext *> subscribe_contexts;
/** The event loop this global state store connection is part of. */
event_loop *loop;
/** Index of the database connection in the event loop */
int64_t db_index;
/** Cache for the IP addresses of db clients. This is an unordered map mapping
* client IDs to addresses. */
std::unordered_map<DBClientID, DBClient> db_client_cache;
/** Redis context for synchronous connections. This should only be used very
* rarely, it is not asynchronous. */
redisContext *sync_context;
};
/**
* Get the Redis asynchronous context responsible for non-subscription
* communication for the given UniqueID.
*
* @param db The database handle.
* @param id The ID whose location we are querying for.
* @return The redisAsyncContext responsible for the given ID.
*/
redisAsyncContext *get_redis_context(DBHandle *db, UniqueID id);
/**
* Get the Redis asynchronous context responsible for subscription
* communication for the given UniqueID.
*
* @param db The database handle.
* @param id The ID whose location we are querying for.
* @return The redisAsyncContext responsible for the given ID.
*/
redisAsyncContext *get_redis_subscribe_context(DBHandle *db, UniqueID id);
/**
* Get a list of Redis shard IP addresses from the primary shard.
*
* @param context A Redis context connected to the primary shard.
* @param db_shards_addresses The IP addresses for the shards registered
* with the primary shard will be added to this vector.
* @param db_shards_ports The IP ports for the shards registered with the
* primary shard will be added to this vector, in the same order as
* db_shards_addresses.
*/
void get_redis_shards(redisContext *context,
std::vector<std::string> &db_shards_addresses,
std::vector<int> &db_shards_ports);
void redis_cache_set_db_client(DBHandle *db, DBClient client);
DBClient redis_cache_get_db_client(DBHandle *db, DBClientID db_client_id);
void redis_object_table_get_entry(redisAsyncContext *c,
void *r,
void *privdata);
void object_table_lookup_callback(redisAsyncContext *c,
void *r,
void *privdata);
/*
* ==== Redis object table functions ====
*/
/**
* Lookup object table entry in redis.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_object_table_lookup(TableCallbackData *callback_data);
/**
* Add a location entry to the object table in redis.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_object_table_add(TableCallbackData *callback_data);
/**
* Remove a location entry from the object table in redis.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_object_table_remove(TableCallbackData *callback_data);
/**
* Create a client-specific channel for receiving notifications from the object
* table.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_object_table_subscribe_to_notifications(
TableCallbackData *callback_data);
/**
* Request notifications about when certain objects become available.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_object_table_request_notifications(TableCallbackData *callback_data);
/**
* Add a new object to the object table in redis.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_result_table_add(TableCallbackData *callback_data);
/**
* Lookup the task that created the object in redis. The result is the task ID.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_result_table_lookup(TableCallbackData *callback_data);
/**
* Callback invoked when the reply from the object table lookup command is
* received.
*
* @param c Redis context.
* @param r Reply.
* @param privdata Data associated to the callback.
* @return Void.
*/
void redis_object_table_lookup_callback(redisAsyncContext *c,
void *r,
void *privdata);
/*
* ==== Redis task table function =====
*/
/**
* Get a task table entry, including the task spec and the task's scheduling
* information.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_task_table_get_task(TableCallbackData *callback_data);
/**
* Add a task table entry with a new task spec and the task's scheduling
* information.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_task_table_add_task(TableCallbackData *callback_data);
/**
* Update a task table entry with the task's scheduling information.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_task_table_update(TableCallbackData *callback_data);
/**
* Update a task table entry with the task's scheduling information, if the
* task's current scheduling information matches the test value.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_task_table_test_and_update(TableCallbackData *callback_data);
/**
* Callback invoked when the reply from the task push command is received.
*
* @param c Redis context.
* @param r Reply (not used).
* @param privdata Data associated to the callback.
* @return Void.
*/
void redis_task_table_publish_push_callback(redisAsyncContext *c,
void *r,
void *privdata);
/**
* Callback invoked when the reply from the task publish command is received.
*
* @param c Redis context.
* @param r Reply (not used).
* @param privdata Data associated to the callback.
* @return Void.
*/
void redis_task_table_publish_publish_callback(redisAsyncContext *c,
void *r,
void *privdata);
/**
* Subscribe to updates of the task table.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_task_table_subscribe(TableCallbackData *callback_data);
/**
* Remove a client from the db clients table.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_db_client_table_remove(TableCallbackData *callback_data);
/**
* Subscribe to updates from the db client table.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_db_client_table_subscribe(TableCallbackData *callback_data);
/**
* Subscribe to updates from the local scheduler table.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_local_scheduler_table_subscribe(TableCallbackData *callback_data);
/**
* Publish an update to the local scheduler table.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_local_scheduler_table_send_info(TableCallbackData *callback_data);
/**
* Synchronously publish a null update to the local scheduler table signifying
* that we are about to exit.
*
* @param db The database handle of the dying local scheduler.
* @return Void.
*/
void redis_local_scheduler_table_disconnect(DBHandle *db);
/**
* Subscribe to updates from the driver table.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_driver_table_subscribe(TableCallbackData *callback_data);
/**
* Publish an update to the driver table.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_driver_table_send_driver_death(TableCallbackData *callback_data);
void redis_plasma_manager_send_heartbeat(TableCallbackData *callback_data);
/**
* Marks an actor as removed. This prevents the actor from being resurrected.
*
* @param db The database handle.
* @param actor_id The actor id to mark as removed.
* @return Void.
*/
void redis_actor_table_mark_removed(DBHandle *db, ActorID actor_id);
/// Publish an actor creation notification.
///
/// \param callback_data Data structure containing redis connection and timeout
/// information.
/// \return Void.
void redis_publish_actor_creation_notification(
TableCallbackData *callback_data);
/**
* Subscribe to updates about newly created actors.
*
* @param callback_data Data structure containing redis connection and timeout
* information.
* @return Void.
*/
void redis_actor_notification_table_subscribe(TableCallbackData *callback_data);
void redis_object_info_subscribe(TableCallbackData *callback_data);
void redis_push_error(TableCallbackData *callback_data);
#endif /* REDIS_H */
src/common/state/table.cc
-200
View File
@@ -1,200 +0,0 @@
#include "table.h"
#include <unordered_map>
#include <inttypes.h>
#include "redis.h"
BaseCallbackData::BaseCallbackData(void *data) {
data_ = data;
}
BaseCallbackData::~BaseCallbackData(void) {}
void *BaseCallbackData::Get(void) {
return data_;
}
CommonCallbackData::CommonCallbackData(void *data) : BaseCallbackData(data) {}
CommonCallbackData::~CommonCallbackData(void) {
free(data_);
}
TaskCallbackData::TaskCallbackData(Task *task_data)
: BaseCallbackData(task_data) {}
TaskCallbackData::~TaskCallbackData(void) {
Task *task = (Task *) data_;
Task_free(task);
}
/* The default behavior is to retry every ten seconds forever. */
static const RetryInfo default_retry = {.num_retries = -1,
.timeout = 10000,
.fail_callback = NULL};
static int64_t callback_data_id = 0;
TableCallbackData *init_table_callback(DBHandle *db_handle,
UniqueID id,
const char *label,
OWNER BaseCallbackData *data,
RetryInfo *retry,
table_done_callback done_callback,
table_retry_callback retry_callback,
void *user_context) {
RAY_CHECK(db_handle);
RAY_CHECK(db_handle->loop);
RAY_CHECK(data);
/* If no retry info is provided, use the default retry info. */
if (retry == NULL) {
retry = (RetryInfo *) &default_retry;
}
RAY_CHECK(retry);
/* Allocate and initialize callback data structure for object table */
TableCallbackData *callback_data =
(TableCallbackData *) malloc(sizeof(TableCallbackData));
RAY_CHECK(callback_data != NULL) << "Memory allocation error!";
callback_data->id = id;
callback_data->label = label;
callback_data->retry = *retry;
callback_data->done_callback = done_callback;
callback_data->retry_callback = retry_callback;
callback_data->data = data;
callback_data->requests_info = NULL;
callback_data->user_context = user_context;
callback_data->db_handle = db_handle;
/* TODO(ekl) set a retry timer once we've figured out the retry conditions
* and have a solution to the O(n^2) ae timers issue. For now, use a dummy
* timer id to uniquely id this callback. */
callback_data->timer_id = callback_data_id++;
outstanding_callbacks_add(callback_data);
RAY_LOG(DEBUG) << "Initializing table command " << callback_data->label
<< " with timer ID " << callback_data->timer_id;
callback_data->retry_callback(callback_data);
return callback_data;
}
void destroy_timer_callback(event_loop *loop,
TableCallbackData *callback_data) {
/* This is commented out because we no longer add timers to the event loop for
* each Redis command. */
// event_loop_remove_timer(loop, callback_data->timer_id);
destroy_table_callback(callback_data);
}
void remove_timer_callback(event_loop *loop, TableCallbackData *callback_data) {
/* This is commented out because we no longer add timers to the event loop for
* each Redis command. */
// event_loop_remove_timer(loop, callback_data->timer_id);
}
void destroy_table_callback(TableCallbackData *callback_data) {
RAY_CHECK(callback_data != NULL);
if (callback_data->requests_info)
free(callback_data->requests_info);
RAY_CHECK(callback_data->data != NULL);
delete callback_data->data;
callback_data->data = NULL;
outstanding_callbacks_remove(callback_data);
/* Timer is removed via EVENT_LOOP_TIMER_DONE in the timeout callback. */
free(callback_data);
}
int64_t table_timeout_handler(event_loop *loop,
int64_t timer_id,
void *user_context) {
RAY_CHECK(loop != NULL);
RAY_CHECK(user_context != NULL);
TableCallbackData *callback_data = (TableCallbackData *) user_context;
RAY_CHECK(callback_data->retry.num_retries >= 0 ||
callback_data->retry.num_retries == -1);
RAY_LOG(WARNING) << "retrying operation " << callback_data->label
<< ", retry_count = " << callback_data->retry.num_retries;
if (callback_data->retry.num_retries == 0) {
/* We didn't get a response from the database after exhausting all retries;
* let user know, cleanup the state, and remove the timer. */
RAY_LOG(WARNING) << "Table command " << callback_data->label
<< " with timer ID " << timer_id << " failed";
if (callback_data->retry.fail_callback) {
callback_data->retry.fail_callback(callback_data->id,
callback_data->user_context,
callback_data->data->Get());
}
destroy_table_callback(callback_data);
return EVENT_LOOP_TIMER_DONE;
}
/* Decrement retry count and try again. We use -1 to indicate infinite
* retries. */
if (callback_data->retry.num_retries != -1) {
callback_data->retry.num_retries--;
}
callback_data->retry_callback(callback_data);
return callback_data->retry.timeout;
}
/**
* Unordered map maintaining the outstanding callbacks.
*
* This unordered map is used to handle the following case:
* - a table command is issued with an associated callback and a callback data
* structure;
* - the last timeout associated to this command expires, as a result the
* callback data structure is freed;
* - a reply arrives, but now the callback data structure is gone, so we have
* to ignore this reply;
*
* This unordered map enables us to ignore such replies. The operations on the
* unordered map are as follows.
*
* When we issue a table command and a timeout event to wait for the reply, we
* add a new entry to the unordered map that is keyed by the ID of the timer.
* Note that table commands must have unique timer IDs, which are assigned by
* the Redis ae event loop.
*
* When we receive the reply, we check whether the callback still exists in
* this unordered map, and if not we just ignore the reply. If the callback does
* exist, the reply receiver is responsible for removing the timer and the
* entry associated to the callback, or else the timeout handler will continue
* firing.
*
* When the last timeout associated to the command expires we remove the entry
* associated to the callback.
*/
static std::unordered_map<timer_id, TableCallbackData *> outstanding_callbacks;
void outstanding_callbacks_add(TableCallbackData *callback_data) {
outstanding_callbacks[callback_data->timer_id] = callback_data;
}
TableCallbackData *outstanding_callbacks_find(int64_t key) {
auto it = outstanding_callbacks.find(key);
if (it != outstanding_callbacks.end()) {
return it->second;
}
return NULL;
}
void outstanding_callbacks_remove(TableCallbackData *callback_data) {
outstanding_callbacks.erase(callback_data->timer_id);
}
void destroy_outstanding_callbacks(event_loop *loop) {
/* We have to be careful because destroy_timer_callback modifies
* outstanding_callbacks in place */
auto it = outstanding_callbacks.begin();
while (it != outstanding_callbacks.end()) {
auto next_it = std::next(it, 1);
destroy_timer_callback(loop, it->second);
it = next_it;
}
}
src/common/state/table.h
-216
View File
@@ -1,216 +0,0 @@
#ifndef TABLE_H
#define TABLE_H
#include "common.h"
#include "task.h"
#include "db.h"
typedef struct TableCallbackData TableCallbackData;
/* An abstract class for any data passed by the user into a table operation.
* This class wraps arbitrary pointers and allows the caller to define a custom
* destructor, for data that is not allocated with malloc. */
class BaseCallbackData {
public:
BaseCallbackData(void *data);
virtual ~BaseCallbackData(void) = 0;
/* Return the pointer to the data. */
void *Get(void);
protected:
/* The pointer to the data. */
void *data_;
};
/* A common class for malloc'ed data passed by the user into a table operation.
* This should ONLY be used when only a free is necessary. */
class CommonCallbackData : public BaseCallbackData {
public:
CommonCallbackData(void *data);
~CommonCallbackData(void);
};
/* A class for Task data passed by the user into a table operation. This calls
* task cleanup in the destructor. */
class TaskCallbackData : public BaseCallbackData {
public:
TaskCallbackData(Task *task_data);
~TaskCallbackData(void);
};
typedef void *table_done_callback;
/* The callback called when the database operation hasn't completed after
* the number of retries specified for the operation.
*
* @param id The unique ID that identifies this callback. Examples include an
* object ID or task ID.
* @param user_context The state context for the callback. This is equivalent
* to the user_context field in TableCallbackData.
* @param user_data A data argument for the callback. This is equivalent to the
* data field in TableCallbackData. The user is responsible for
* freeing user_data.
*/
typedef void (*table_fail_callback)(UniqueID id,
void *user_context,
void *user_data);
typedef void (*table_retry_callback)(TableCallbackData *callback_data);
/**
* Data structure consolidating the retry related variables. If a NULL
* RetryInfo struct is used, the default behavior will be to retry infinitely
* many times.
*/
typedef struct {
/** Number of retries. This field will be decremented every time a retry
* occurs (unless the value is -1). If this value is -1, then there will be
* infinitely many retries. */
int num_retries;
/** Timeout, in milliseconds. */
uint64_t timeout;
/** The callback that will be called if there are no more retries left. */
table_fail_callback fail_callback;
} RetryInfo;
struct TableCallbackData {
/** ID of the entry in the table that we are going to look up, remove or add.
*/
UniqueID id;
/** A label to identify the original request for logging purposes. */
const char *label;
/** The callback that will be called when results is returned. */
table_done_callback done_callback;
/** The callback that will be called to initiate the next try. */
table_retry_callback retry_callback;
/** Retry information containing the remaining number of retries, the timeout
* before the next retry, and a pointer to the failure callback.
*/
RetryInfo retry;
/** Pointer to the data that is entered into the table. This can be used to
* pass the result of the call to the callback. The callback takes ownership
* over this data and will free it. */
BaseCallbackData *data;
/** Pointer to the data used internally to handle multiple database requests.
*/
void *requests_info;
/** User context. */
void *user_context;
/** Handle to db. */
DBHandle *db_handle;
/** Handle to timer. */
int64_t timer_id;
};
/**
* Function to handle the timeout event.
*
* @param loop Event loop.
* @param timer_id Timer identifier.
* @param context Pointer to the callback data for the object table
* @return Timeout to reset the timer if we need to try again, or
* EVENT_LOOP_TIMER_DONE if retry_count == 0.
*/
int64_t table_timeout_handler(event_loop *loop,
int64_t timer_id,
void *context);
/**
* Initialize the table callback and call the retry_callback for the first time.
*
* @param db_handle Database handle.
* @param id ID of the object that is looked up, added or removed.
* @param label A string label to identify the type of table request for
* logging purposes.
* @param data Data entered into the table. Shall be freed by the user. Caller
* must specify a destructor by wrapping a void *pointer in a
* BaseCallbackData class.
* @param retry Retry relevant information: retry timeout, number of remaining
* retries, and retry callback.
* @param done_callback Function to be called when database returns result.
* @param fail_callback Function to be called when number of retries is
* exhausted.
* @param user_context Context that can be provided by the user and will be
* passed on to the various callbacks.
* @return New table callback data struct.
*/
TableCallbackData *init_table_callback(DBHandle *db_handle,
UniqueID id,
const char *label,
OWNER BaseCallbackData *data,
RetryInfo *retry,
table_done_callback done_callback,
table_retry_callback retry_callback,
void *user_context);
/**
* Destroy any state associated with the callback data. This removes all
* associated state from the outstanding callbacks unordered map and frees any
* associated memory. This does not remove any associated timer events.
*
* @param callback_data The pointer to the data structure of the callback we
* want to remove.
* @return Void.
*/
void destroy_table_callback(TableCallbackData *callback_data);
/**
* Destroy all state events associated with the callback data, including memory
* and timer events.
*
* @param loop The event loop.
* @param callback_data The pointer to the data structure of the callback we
* want to remove.
* @return Void.
*/
void destroy_timer_callback(event_loop *loop, TableCallbackData *callback_data);
/**
* Remove the callback timer without destroying the callback data.
*
* @param loop The event loop.
* @param callback_data The pointer to the data structure of the callback.
* @return Void.
*/
void remove_timer_callback(event_loop *loop, TableCallbackData *callback_data);
/**
* Add an outstanding callback entry.
*
* @param callback_data The pointer to the data structure of the callback we
* want to insert.
* @return None.
*/
void outstanding_callbacks_add(TableCallbackData *callback_data);
/**
* Find an outstanding callback entry.
*
* @param key The key for the outstanding callbacks unordered map. We use the
* timer ID assigned by the Redis ae event loop.
* @return Returns the callback data if found, NULL otherwise.
*/
TableCallbackData *outstanding_callbacks_find(int64_t key);
/**
* Remove an outstanding callback entry. This only removes the callback entry
* from the unordered map. It does not free the entry or remove any associated
* timer events.
*
* @param callback_data The pointer to the data structure of the callback we
* want to remove.
* @return Void.
*/
void outstanding_callbacks_remove(TableCallbackData *callback_data);
/**
* Destroy all outstanding callbacks and remove their associated timer events
* from the event loop.
*
* @param loop The event loop from which we want to remove the timer events.
* @return Void.
*/
void destroy_outstanding_callbacks(event_loop *loop);
#endif /* TABLE_H */
src/common/state/task_table.cc
-80
View File
@@ -1,80 +0,0 @@
#include "task_table.h"
#include "redis.h"
#define NUM_DB_REQUESTS 2
void task_table_get_task(DBHandle *db_handle,
TaskID task_id,
RetryInfo *retry,
task_table_get_callback get_callback,
void *user_context) {
init_table_callback(
db_handle, task_id, __func__, new CommonCallbackData(NULL), retry,
(void *) get_callback, redis_task_table_get_task, user_context);
}
void task_table_add_task(DBHandle *db_handle,
OWNER Task *task,
RetryInfo *retry,
task_table_done_callback done_callback,
void *user_context) {
init_table_callback(db_handle, Task_task_id(task), __func__,
new TaskCallbackData(task), retry,
(table_done_callback) done_callback,
redis_task_table_add_task, user_context);
}
void task_table_update(DBHandle *db_handle,
OWNER Task *task,
RetryInfo *retry,
task_table_done_callback done_callback,
void *user_context) {
init_table_callback(db_handle, Task_task_id(task), __func__,
new TaskCallbackData(task), retry,
(table_done_callback) done_callback,
redis_task_table_update, user_context);
}
void task_table_test_and_update(
DBHandle *db_handle,
TaskID task_id,
DBClientID test_local_scheduler_id,
TaskStatus test_state_bitmask,
TaskStatus update_state,
RetryInfo *retry,
task_table_test_and_update_callback done_callback,
void *user_context) {
TaskTableTestAndUpdateData *update_data =
(TaskTableTestAndUpdateData *) malloc(sizeof(TaskTableTestAndUpdateData));
update_data->test_local_scheduler_id = test_local_scheduler_id;
update_data->test_state_bitmask = test_state_bitmask;
update_data->update_state = update_state;
/* Update the task entry's local scheduler with this client's ID. */
update_data->local_scheduler_id = db_handle->client;
init_table_callback(db_handle, task_id, __func__,
new CommonCallbackData(update_data), retry,
(table_done_callback) done_callback,
redis_task_table_test_and_update, user_context);
}
/* TODO(swang): A corresponding task_table_unsubscribe. */
void task_table_subscribe(DBHandle *db_handle,
DBClientID local_scheduler_id,
TaskStatus state_filter,
task_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry,
task_table_done_callback done_callback,
void *user_context) {
TaskTableSubscribeData *sub_data =
(TaskTableSubscribeData *) malloc(sizeof(TaskTableSubscribeData));
sub_data->local_scheduler_id = local_scheduler_id;
sub_data->state_filter = state_filter;
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, local_scheduler_id, __func__,
new CommonCallbackData(sub_data), retry,
(table_done_callback) done_callback,
redis_task_table_subscribe, user_context);
}
src/common/state/task_table.h
-190
View File
@@ -1,190 +0,0 @@
#ifndef task_table_H
#define task_table_H
#include "db.h"
#include "table.h"
#include "task.h"
/**
* The task table is a message bus that is used for communication between local
* and global schedulers (and also persisted to the state database). Here are
* examples of events that are recorded by the task table:
*
* 1) Local schedulers write to it when submitting a task to the global
* scheduler.
* 2) The global scheduler subscribes to updates to the task table to get tasks
* submitted by local schedulers.
* 3) The global scheduler writes to it when assigning a task to a local
* scheduler.
* 4) Local schedulers subscribe to updates to the task table to get tasks
* assigned to them by the global scheduler.
* 5) Local schedulers write to it when a task finishes execution.
*/
/* Callback called when a task table write operation completes. */
typedef void (*task_table_done_callback)(TaskID task_id, void *user_context);
/* Callback called when a task table read operation completes. If the task ID
* was not in the task table, then the task pointer will be NULL. */
typedef void (*task_table_get_callback)(Task *task, void *user_context);
/* Callback called when a task table test-and-update operation completes. If
* the task ID was not in the task table, then the task pointer will be NULL.
* If the update succeeded, the updated field will be set to true. */
typedef void (*task_table_test_and_update_callback)(Task *task,
void *user_context,
bool updated);
/**
* Get a task's entry from the task table.
*
* @param db_handle Database handle.
* @param task_id The ID of the task we want to look up.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Data that will be passed to done_callback and
* fail_callback.
* @return Void.
*/
void task_table_get_task(DBHandle *db,
TaskID task_id,
RetryInfo *retry,
task_table_get_callback get_callback,
void *user_context);
/**
* Add a task entry, including task spec and scheduling information, to the task
* table. This will overwrite any task already in the task table with the same
* task ID.
*
* @param db_handle Database handle.
* @param task The task entry to add to the table.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Data that will be passed to done_callback and
* fail_callback.
* @return Void.
*/
void task_table_add_task(DBHandle *db_handle,
OWNER Task *task,
RetryInfo *retry,
task_table_done_callback done_callback,
void *user_context);
/*
* ==== Publish the task table ====
*/
/**
* Update a task's scheduling information in the task table. This assumes that
* the task spec already exists in the task table entry.
*
* @param db_handle Database handle.
* @param task The task entry to add to the table. The task spec in the entry is
* ignored.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Data that will be passed to done_callback and
* fail_callback.
* @return Void.
*/
void task_table_update(DBHandle *db_handle,
OWNER Task *task,
RetryInfo *retry,
task_table_done_callback done_callback,
void *user_context);
/**
* Update a task's scheduling information in the task table, if the current
* value matches the given test value. If the update succeeds, it also updates
* the task entry's local scheduler ID with the ID of the client who called
* this function. This assumes that the task spec already exists in the task
* table entry.
*
* @param db_handle Database handle.
* @param task_id The task ID of the task entry to update.
* @param test_local_scheduler_id The local scheduler ID to test the current
* local scheduler ID against. If not NIL_ID, and if the current local
* scheduler ID does not match it, then the update will not happen.
* @param test_state_bitmask The bitmask to apply to the task entry's current
* scheduling state. The update happens if and only if the current
* scheduling state AND-ed with the bitmask is greater than 0 and the
* local scheduler ID test passes.
* @param update_state The value to update the task entry's scheduling state
* with, if the current state matches test_state_bitmask.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Data that will be passed to done_callback and
* fail_callback.
* @return Void.
*/
void task_table_test_and_update(
DBHandle *db_handle,
TaskID task_id,
DBClientID test_local_scheduler_id,
TaskStatus test_state_bitmask,
TaskStatus update_state,
RetryInfo *retry,
task_table_test_and_update_callback done_callback,
void *user_context);
/* Data that is needed to test and set the task's scheduling state. */
typedef struct {
/** The value to test the current local scheduler ID against. This field is
* ignored if equal to NIL_ID. */
DBClientID test_local_scheduler_id;
TaskStatus test_state_bitmask;
TaskStatus update_state;
DBClientID local_scheduler_id;
} TaskTableTestAndUpdateData;
/*
* ==== Subscribing to the task table ====
*/
/* Callback for subscribing to the task table. */
typedef void (*task_table_subscribe_callback)(Task *task, void *user_context);
/**
* Register a callback for a task event. An event is any update of a task in
* the task table, produced by task_table_add_task or task_table_add_task.
* Events include changes to the task's scheduling state or changes to the
* task's local scheduler ID.
*
* @param db_handle Database handle.
* @param subscribe_callback Callback that will be called when the task table is
* updated.
* @param subscribe_context Context that will be passed into the
* subscribe_callback.
* @param local_scheduler_id The db_client_id of the local scheduler whose
* events we want to listen to. If you want to subscribe to updates from
* all local schedulers, pass in NIL_ID.
* @param state_filter Events we want to listen to. Can have values from the
* enum "scheduling_state" in task.h.
* TODO(pcm): Make it possible to combine these using flags like
* TASK_STATUS_WAITING | TASK_STATUS_SCHEDULED.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Data that will be passed to done_callback and
* fail_callback.
* @return Void.
*/
void task_table_subscribe(DBHandle *db_handle,
DBClientID local_scheduler_id,
TaskStatus state_filter,
task_table_subscribe_callback subscribe_callback,
void *subscribe_context,
RetryInfo *retry,
task_table_done_callback done_callback,
void *user_context);
/* Data that is needed to register task table subscribe callbacks with the state
* database. */
typedef struct {
DBClientID local_scheduler_id;
TaskStatus state_filter;
task_table_subscribe_callback subscribe_callback;
void *subscribe_context;
} TaskTableSubscribeData;
#endif /* task_table_H */
src/common/task.cc
-606
View File
@@ -1,606 +0,0 @@
#include <limits.h>
#include "common_protocol.h"
#include "task.h"
extern "C" {
#include "sha256.h"
}
ObjectID task_compute_return_id(TaskID task_id, int64_t return_index) {
/* Here, return_indices need to be >= 0, so we can use negative
* indices for put. */
RAY_DCHECK(return_index >= 0);
/* TODO(rkn): This line requires object and task IDs to be the same size. */
ObjectID return_id = task_id;
int64_t *first_bytes = (int64_t *) &return_id;
/* XOR the first bytes of the object ID with the return index. We add one so
* the first return ID is not the same as the task ID. */
*first_bytes = *first_bytes ^ (return_index + 1);
return return_id;
}
ObjectID task_compute_put_id(TaskID task_id, int64_t put_index) {
RAY_DCHECK(put_index >= 0);
/* TODO(pcm): This line requires object and task IDs to be the same size. */
ObjectID put_id = task_id;
int64_t *first_bytes = (int64_t *) &put_id;
/* XOR the first bytes of the object ID with the return index. We add one so
* the first return ID is not the same as the task ID. */
*first_bytes = *first_bytes ^ (-put_index - 1);
return put_id;
}
class TaskBuilder {
public:
void Start(UniqueID driver_id,
TaskID parent_task_id,
int64_t parent_counter,
ActorID actor_creation_id,
ObjectID actor_creation_dummy_object_id,
ActorID actor_id,
ActorHandleID actor_handle_id,
int64_t actor_counter,
bool is_actor_checkpoint_method,
FunctionID function_id,
int64_t num_returns) {
driver_id_ = driver_id;
parent_task_id_ = parent_task_id;
parent_counter_ = parent_counter;
actor_creation_id_ = actor_creation_id;
actor_creation_dummy_object_id_ = actor_creation_dummy_object_id;
actor_id_ = actor_id;
actor_handle_id_ = actor_handle_id;
actor_counter_ = actor_counter;
is_actor_checkpoint_method_ = is_actor_checkpoint_method;
function_id_ = function_id;
num_returns_ = num_returns;
/* Compute hashes. */
sha256_init(&ctx);
sha256_update(&ctx, (BYTE *) &driver_id, sizeof(driver_id));
sha256_update(&ctx, (BYTE *) &parent_task_id, sizeof(parent_task_id));
sha256_update(&ctx, (BYTE *) &parent_counter, sizeof(parent_counter));
sha256_update(&ctx, (BYTE *) &actor_creation_id, sizeof(actor_creation_id));
sha256_update(&ctx, (BYTE *) &actor_creation_dummy_object_id,
sizeof(actor_creation_dummy_object_id));
sha256_update(&ctx, (BYTE *) &actor_id, sizeof(actor_id));
sha256_update(&ctx, (BYTE *) &actor_counter, sizeof(actor_counter));
sha256_update(&ctx, (BYTE *) &is_actor_checkpoint_method,
sizeof(is_actor_checkpoint_method));
sha256_update(&ctx, (BYTE *) &function_id, sizeof(function_id));
}
void NextReferenceArgument(ObjectID object_ids[], int num_object_ids) {
args.push_back(
CreateArg(fbb, to_flatbuf(fbb, &object_ids[0], num_object_ids)));
sha256_update(&ctx, (BYTE *) &object_ids[0],
sizeof(object_ids[0]) * num_object_ids);
}
void NextValueArgument(uint8_t *value, int64_t length) {
auto arg = fbb.CreateString((const char *) value, length);
auto empty_ids = fbb.CreateVectorOfStrings({});
args.push_back(CreateArg(fbb, empty_ids, arg));
sha256_update(&ctx, (BYTE *) value, length);
}
void SetRequiredResource(const std::string &resource_name, double value) {
RAY_CHECK(resource_map_.count(resource_name) == 0);
resource_map_[resource_name] = value;
}
uint8_t *Finish(int64_t *size) {
/* Add arguments. */
auto arguments = fbb.CreateVector(args);
/* Update hash. */
BYTE buff[DIGEST_SIZE];
sha256_final(&ctx, buff);
TaskID task_id;
RAY_CHECK(sizeof(task_id) <= DIGEST_SIZE);
memcpy(&task_id, buff, sizeof(task_id));
/* Add return object IDs. */
std::vector<flatbuffers::Offset<flatbuffers::String>> returns;
for (int64_t i = 0; i < num_returns_; i++) {
ObjectID return_id = task_compute_return_id(task_id, i);
returns.push_back(to_flatbuf(fbb, return_id));
}
/* Create TaskInfo. */
auto message = CreateTaskInfo(
fbb, to_flatbuf(fbb, driver_id_), to_flatbuf(fbb, task_id),
to_flatbuf(fbb, parent_task_id_), parent_counter_,
to_flatbuf(fbb, actor_creation_id_),
to_flatbuf(fbb, actor_creation_dummy_object_id_),
to_flatbuf(fbb, actor_id_), to_flatbuf(fbb, actor_handle_id_),
actor_counter_, is_actor_checkpoint_method_,
to_flatbuf(fbb, function_id_), arguments, fbb.CreateVector(returns),
map_to_flatbuf(fbb, resource_map_));
/* Finish the TaskInfo. */
fbb.Finish(message);
*size = fbb.GetSize();
uint8_t *result = (uint8_t *) malloc(*size);
memcpy(result, fbb.GetBufferPointer(), *size);
fbb.Clear();
args.clear();
resource_map_.clear();
return result;
}
private:
flatbuffers::FlatBufferBuilder fbb;
std::vector<flatbuffers::Offset<Arg>> args;
SHA256_CTX ctx;
/* Data for the builder. */
UniqueID driver_id_;
TaskID parent_task_id_;
int64_t parent_counter_;
ActorID actor_creation_id_;
ObjectID actor_creation_dummy_object_id_;
ActorID actor_id_;
ActorID actor_handle_id_;
int64_t actor_counter_;
bool is_actor_checkpoint_method_;
FunctionID function_id_;
int64_t num_returns_;
std::unordered_map<std::string, double> resource_map_;
};
TaskBuilder *make_task_builder(void) {
return new TaskBuilder();
}
void free_task_builder(TaskBuilder *builder) {
delete builder;
}
bool TaskID_equal(TaskID first_id, TaskID second_id) {
return first_id == second_id;
}
bool TaskID_is_nil(TaskID id) {
return id.is_nil();
}
bool ActorID_equal(ActorID first_id, ActorID second_id) {
return first_id == second_id;
}
bool FunctionID_equal(FunctionID first_id, FunctionID second_id) {
return first_id == second_id;
}
bool FunctionID_is_nil(FunctionID id) {
return id.is_nil();
}
/* Functions for building tasks. */
void TaskSpec_start_construct(TaskBuilder *builder,
UniqueID driver_id,
TaskID parent_task_id,
int64_t parent_counter,
ActorID actor_creation_id,
ObjectID actor_creation_dummy_object_id,
ActorID actor_id,
ActorID actor_handle_id,
int64_t actor_counter,
bool is_actor_checkpoint_method,
FunctionID function_id,
int64_t num_returns) {
builder->Start(driver_id, parent_task_id, parent_counter, actor_creation_id,
actor_creation_dummy_object_id, actor_id, actor_handle_id,
actor_counter, is_actor_checkpoint_method, function_id,
num_returns);
}
TaskSpec *TaskSpec_finish_construct(TaskBuilder *builder, int64_t *size) {
return reinterpret_cast<TaskSpec *>(builder->Finish(size));
}
void TaskSpec_args_add_ref(TaskBuilder *builder,
ObjectID object_ids[],
int num_object_ids) {
builder->NextReferenceArgument(&object_ids[0], num_object_ids);
}
void TaskSpec_args_add_val(TaskBuilder *builder,
uint8_t *value,
int64_t length) {
builder->NextValueArgument(value, length);
}
void TaskSpec_set_required_resource(TaskBuilder *builder,
const std::string &resource_name,
double value) {
builder->SetRequiredResource(resource_name, value);
}
/* Functions for reading tasks. */
TaskID TaskSpec_task_id(const TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->task_id());
}
FunctionID TaskSpec_function(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->function_id());
}
ActorID TaskSpec_actor_id(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->actor_id());
}
ActorID TaskSpec_actor_handle_id(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->actor_handle_id());
}
bool TaskSpec_is_actor_task(TaskSpec *spec) {
return !TaskSpec_actor_id(spec).is_nil();
}
ActorID TaskSpec_actor_creation_id(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->actor_creation_id());
}
ObjectID TaskSpec_actor_creation_dummy_object_id(TaskSpec *spec) {
RAY_CHECK(spec);
// The task must be an actor method.
RAY_CHECK(TaskSpec_is_actor_task(spec));
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->actor_creation_dummy_object_id());
}
bool TaskSpec_is_actor_creation_task(TaskSpec *spec) {
return !TaskSpec_actor_creation_id(spec).is_nil();
}
int64_t TaskSpec_actor_counter(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return std::abs(message->actor_counter());
}
bool TaskSpec_is_actor_checkpoint_method(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return message->is_actor_checkpoint_method();
}
ObjectID TaskSpec_actor_dummy_object(TaskSpec *spec) {
RAY_CHECK(TaskSpec_is_actor_task(spec));
/* The last return value for actor tasks is the dummy object that
* represents that this task has completed execution. */
int64_t num_returns = TaskSpec_num_returns(spec);
return TaskSpec_return(spec, num_returns - 1);
}
UniqueID TaskSpec_driver_id(const TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->driver_id());
}
TaskID TaskSpec_parent_task_id(const TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->parent_task_id());
}
int64_t TaskSpec_parent_counter(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return message->parent_counter();
}
int64_t TaskSpec_num_args(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return message->args()->size();
}
int64_t TaskSpec_num_args_by_ref(TaskSpec *spec) {
int64_t num_args = TaskSpec_num_args(spec);
int64_t num_args_by_ref = 0;
for (int64_t i = 0; i < num_args; i++) {
if (TaskSpec_arg_by_ref(spec, i)) {
num_args_by_ref++;
}
}
return num_args_by_ref;
}
int TaskSpec_arg_id_count(TaskSpec *spec, int64_t arg_index) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
auto ids = message->args()->Get(arg_index)->object_ids();
if (ids == nullptr) {
return 0;
} else {
return ids->size();
}
}
ObjectID TaskSpec_arg_id(TaskSpec *spec, int64_t arg_index, int64_t id_index) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(
*message->args()->Get(arg_index)->object_ids()->Get(id_index));
}
const uint8_t *TaskSpec_arg_val(TaskSpec *spec, int64_t arg_index) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return (uint8_t *) message->args()->Get(arg_index)->data()->c_str();
}
int64_t TaskSpec_arg_length(TaskSpec *spec, int64_t arg_index) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return message->args()->Get(arg_index)->data()->size();
}
int64_t TaskSpec_num_returns(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return message->returns()->size();
}
bool TaskSpec_arg_by_ref(TaskSpec *spec, int64_t arg_index) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return message->args()->Get(arg_index)->object_ids()->size() != 0;
}
ObjectID TaskSpec_return(TaskSpec *spec, int64_t return_index) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->returns()->Get(return_index));
}
double TaskSpec_get_required_resource(const TaskSpec *spec,
const std::string &resource_name) {
// This is a bit ugly. However it shouldn't be much of a performance issue
// because there shouldn't be many distinct resources in a single task spec.
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
for (size_t i = 0; i < message->required_resources()->size(); i++) {
const ResourcePair *resource_pair = message->required_resources()->Get(i);
if (string_from_flatbuf(*resource_pair->key()) == resource_name) {
return resource_pair->value();
}
}
return 0;
}
const std::unordered_map<std::string, double> TaskSpec_get_required_resources(
const TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return map_from_flatbuf(*message->required_resources());
}
TaskSpec *TaskSpec_copy(TaskSpec *spec, int64_t task_spec_size) {
TaskSpec *copy = (TaskSpec *) malloc(task_spec_size);
memcpy(copy, spec, task_spec_size);
return copy;
}
void TaskSpec_free(TaskSpec *spec) {
free(spec);
}
TaskExecutionSpec::TaskExecutionSpec(
const std::vector<ObjectID> &execution_dependencies,
const TaskSpec *spec,
int64_t task_spec_size,
int spillback_count)
: execution_dependencies_(execution_dependencies),
task_spec_size_(task_spec_size),
last_timestamp_(0),
spillback_count_(spillback_count) {
TaskSpec *spec_copy = new TaskSpec[task_spec_size_];
memcpy(spec_copy, spec, task_spec_size);
spec_ = std::unique_ptr<TaskSpec[]>(spec_copy);
}
TaskExecutionSpec::TaskExecutionSpec(
const std::vector<ObjectID> &execution_dependencies,
const TaskSpec *spec,
int64_t task_spec_size)
: TaskExecutionSpec(execution_dependencies, spec, task_spec_size, 0) {}
TaskExecutionSpec::TaskExecutionSpec(TaskExecutionSpec *other)
: execution_dependencies_(other->execution_dependencies_),
task_spec_size_(other->task_spec_size_),
last_timestamp_(other->last_timestamp_),
spillback_count_(other->spillback_count_) {
TaskSpec *spec_copy = new TaskSpec[task_spec_size_];
memcpy(spec_copy, other->spec_.get(), task_spec_size_);
spec_ = std::unique_ptr<TaskSpec[]>(spec_copy);
}
const std::vector<ObjectID> &TaskExecutionSpec::ExecutionDependencies() const {
return execution_dependencies_;
}
void TaskExecutionSpec::SetExecutionDependencies(
const std::vector<ObjectID> &dependencies) {
execution_dependencies_ = dependencies;
}
int64_t TaskExecutionSpec::SpecSize() const {
return task_spec_size_;
}
int TaskExecutionSpec::SpillbackCount() const {
return spillback_count_;
}
void TaskExecutionSpec::IncrementSpillbackCount() {
++spillback_count_;
}
int64_t TaskExecutionSpec::LastTimeStamp() const {
return last_timestamp_;
}
void TaskExecutionSpec::SetLastTimeStamp(int64_t new_timestamp) {
last_timestamp_ = new_timestamp;
}
TaskSpec *TaskExecutionSpec::Spec() const {
return spec_.get();
}
int64_t TaskExecutionSpec::NumDependencies() const {
TaskSpec *spec = Spec();
int64_t num_dependencies = TaskSpec_num_args(spec);
num_dependencies += execution_dependencies_.size();
return num_dependencies;
}
int TaskExecutionSpec::DependencyIdCount(int64_t dependency_index) const {
TaskSpec *spec = Spec();
/* The first dependencies are the arguments of the task itself, followed by
* the execution dependencies. Find the total number of task arguments so
* that we can index into the correct list. */
int64_t num_args = TaskSpec_num_args(spec);
if (dependency_index < num_args) {
/* Index into the task arguments. */
return TaskSpec_arg_id_count(spec, dependency_index);
} else {
/* Index into the execution dependencies. */
dependency_index -= num_args;
RAY_CHECK((size_t) dependency_index < execution_dependencies_.size());
/* All elements in the execution dependency list have exactly one ID. */
return 1;
}
}
ObjectID TaskExecutionSpec::DependencyId(int64_t dependency_index,
int64_t id_index) const {
TaskSpec *spec = Spec();
/* The first dependencies are the arguments of the task itself, followed by
* the execution dependencies. Find the total number of task arguments so
* that we can index into the correct list. */
int64_t num_args = TaskSpec_num_args(spec);
if (dependency_index < num_args) {
/* Index into the task arguments. */
return TaskSpec_arg_id(spec, dependency_index, id_index);
} else {
/* Index into the execution dependencies. */
dependency_index -= num_args;
RAY_CHECK((size_t) dependency_index < execution_dependencies_.size());
return execution_dependencies_[dependency_index];
}
}
bool TaskExecutionSpec::DependsOn(ObjectID object_id) const {
// Iterate through the task arguments to see if it contains object_id.
TaskSpec *spec = Spec();
int64_t num_args = TaskSpec_num_args(spec);
for (int i = 0; i < num_args; ++i) {
int count = TaskSpec_arg_id_count(spec, i);
for (int j = 0; j < count; j++) {
ObjectID arg_id = TaskSpec_arg_id(spec, i, j);
if (arg_id == object_id) {
return true;
}
}
}
// Iterate through the execution dependencies to see if it contains object_id.
for (auto dependency_id : execution_dependencies_) {
if (dependency_id == object_id) {
return true;
}
}
// The requested object ID was not a task argument or an execution dependency.
// This task is not dependent on it.
return false;
}
bool TaskExecutionSpec::IsStaticDependency(int64_t dependency_index) const {
TaskSpec *spec = Spec();
/* The first dependencies are the arguments of the task itself, followed by
* the execution dependencies. If the requested dependency index is a task
* argument, then it is a task dependency. */
int64_t num_args = TaskSpec_num_args(spec);
return (dependency_index < num_args);
}
/* TASK INSTANCES */
Task *Task_alloc(const TaskSpec *spec,
int64_t task_spec_size,
TaskStatus state,
DBClientID local_scheduler_id,
const std::vector<ObjectID> &execution_dependencies) {
Task *result = new Task();
auto execution_spec =
new TaskExecutionSpec(execution_dependencies, spec, task_spec_size);
result->execution_spec = std::unique_ptr<TaskExecutionSpec>(execution_spec);
result->state = state;
result->local_scheduler_id = local_scheduler_id;
return result;
}
Task *Task_alloc(TaskExecutionSpec &execution_spec,
TaskStatus state,
DBClientID local_scheduler_id) {
Task *result = new Task();
result->execution_spec = std::unique_ptr<TaskExecutionSpec>(
new TaskExecutionSpec(&execution_spec));
result->state = state;
result->local_scheduler_id = local_scheduler_id;
return result;
}
Task *Task_copy(Task *other) {
return Task_alloc(*Task_task_execution_spec(other), other->state,
other->local_scheduler_id);
}
int64_t Task_size(Task *task_arg) {
return sizeof(Task) - sizeof(TaskSpec) + task_arg->execution_spec->SpecSize();
}
TaskStatus Task_state(Task *task) {
return task->state;
}
void Task_set_state(Task *task, TaskStatus state) {
task->state = state;
}
DBClientID Task_local_scheduler(Task *task) {
return task->local_scheduler_id;
}
void Task_set_local_scheduler(Task *task, DBClientID local_scheduler_id) {
task->local_scheduler_id = local_scheduler_id;
}
TaskExecutionSpec *Task_task_execution_spec(Task *task) {
return task->execution_spec.get();
}
TaskID Task_task_id(Task *task) {
TaskExecutionSpec *execution_spec = Task_task_execution_spec(task);
TaskSpec *spec = execution_spec->Spec();
return TaskSpec_task_id(spec);
}
void Task_free(Task *task) {
delete task;
}
src/common/task.h
-609
View File
@@ -1,609 +0,0 @@
#ifndef TASK_H
#define TASK_H
#include <unordered_map>
#include <stddef.h>
#include <stdint.h>
#include "common.h"
#include <string>
#include "format/common_generated.h"
using namespace ray;
typedef char TaskSpec;
class TaskExecutionSpec {
public:
TaskExecutionSpec(const std::vector<ObjectID> &execution_dependencies,
const TaskSpec *spec,
int64_t task_spec_size);
TaskExecutionSpec(const std::vector<ObjectID> &execution_dependencies,
const TaskSpec *spec,
int64_t task_spec_size,
int spillback_count);
TaskExecutionSpec(TaskExecutionSpec *execution_spec);
/// Get the task's execution dependencies.
///
/// @return A vector of object IDs representing this task's execution
/// dependencies.
const std::vector<ObjectID> &ExecutionDependencies() const;
/// Set the task's execution dependencies.
///
/// @param dependencies The value to set the execution dependencies to.
/// @return Void.
void SetExecutionDependencies(const std::vector<ObjectID> &dependencies);
/// Get the task spec size.
///
/// @return The size of the immutable task spec.
int64_t SpecSize() const;
/// Get the task's spillback count, which tracks the number of times
/// this task was spilled back from local to the global scheduler.
///
/// @return The spillback count for this task.
int SpillbackCount() const;
/// Increment the spillback count for this task.
///
/// @return Void.
void IncrementSpillbackCount();
/// Get the task's last timestamp.
///
/// @return The timestamp when this task was last received for scheduling.
int64_t LastTimeStamp() const;
/// Set the task's last timestamp to the specified value.
///
/// @param new_timestamp The new timestamp in millisecond to set the task's
/// time stamp to. Tracks the last time this task entered a local
/// scheduler.
/// @return Void.
void SetLastTimeStamp(int64_t new_timestamp);
/// Get the task spec.
///
/// @return A pointer to the immutable task spec.
TaskSpec *Spec() const;
/// Get the number of dependencies. This comprises the immutable task
/// arguments and the mutable execution dependencies.
///
/// @return The number of dependencies.
int64_t NumDependencies() const;
/// Get the number of object IDs at the given dependency index.
///
/// @param dependency_index The dependency index whose object IDs to count.
/// @return The number of object IDs at the given dependency_index.
int DependencyIdCount(int64_t dependency_index) const;
/// Get the object ID of a given dependency index.
///
/// @param dependency_index The index at which we should look up the object
/// ID.
/// @param id_index The index of the object ID.
ObjectID DependencyId(int64_t dependency_index, int64_t id_index) const;
/// Compute whether the task is dependent on an object ID.
///
/// @param object_id The object ID that the task may be dependent on.
/// @return bool This returns true if the task is dependent on the given
/// object ID and false otherwise.
bool DependsOn(ObjectID object_id) const;
/// Returns whether the given dependency index is a static dependency (an
/// argument of the immutable task).
///
/// @param dependency_index The requested dependency index.
/// @return bool This returns true if the requested dependency index is
/// immutable (an argument of the task).
bool IsStaticDependency(int64_t dependency_index) const;
private:
/** A list of object IDs representing this task's dependencies at execution
* time. */
std::vector<ObjectID> execution_dependencies_;
/** The size of the task specification for this task. */
int64_t task_spec_size_;
/** Last time this task was received for scheduling. */
int64_t last_timestamp_;
/** Number of times this task was spilled back by local schedulers. */
int spillback_count_;
/** The task specification for this task. */
std::unique_ptr<TaskSpec[]> spec_;
};
class TaskBuilder;
typedef UniqueID FunctionID;
/** The task ID is a deterministic hash of the function ID that the task
* executes and the argument IDs or argument values. */
typedef UniqueID TaskID;
/** The actor ID is the ID of the actor that a task must run on. If the task is
* not run on an actor, then NIL_ACTOR_ID should be used. */
typedef UniqueID ActorID;
/**
* Compare two task IDs.
*
* @param first_id The first task ID to compare.
* @param second_id The first task ID to compare.
* @return True if the task IDs are the same and false otherwise.
*/
bool TaskID_equal(TaskID first_id, TaskID second_id);
/**
* Compare a task ID to the nil ID.
*
* @param id The task ID to compare to nil.
* @return True if the task ID is equal to nil.
*/
bool TaskID_is_nil(TaskID id);
/**
* Compare two actor IDs.
*
* @param first_id The first actor ID to compare.
* @param second_id The first actor ID to compare.
* @return True if the actor IDs are the same and false otherwise.
*/
bool ActorID_equal(ActorID first_id, ActorID second_id);
/**
* Compare two function IDs.
*
* @param first_id The first function ID to compare.
* @param second_id The first function ID to compare.
* @return True if the function IDs are the same and false otherwise.
*/
bool FunctionID_equal(FunctionID first_id, FunctionID second_id);
/**
* Compare a function ID to the nil ID.
*
* @param id The function ID to compare to nil.
* @return True if the function ID is equal to nil.
*/
bool FunctionID_is_nil(FunctionID id);
/* Construct and modify task specifications. */
TaskBuilder *make_task_builder(void);
void free_task_builder(TaskBuilder *builder);
/**
* Begin constructing a task_spec. After this is called, the arguments must be
* added to the task_spec and then finish_construct_task_spec must be called.
*
* @param driver_id The ID of the driver whose job is responsible for the
* creation of this task.
* @param parent_task_id The task ID of the task that submitted this task.
* @param parent_counter A counter indicating how many tasks were submitted by
* the parent task prior to this one.
* @param actor_creation_id The actor creation ID of this task.
* @param actor_creation_dummy_object_id The dummy object for the corresponding
* actor creation task, assuming this is an actor method.
* @param actor_id The ID of the actor that this task is for. If it is not an
* actor task, then this if NIL_ACTOR_ID.
* @param actor_handle_id The ID of the actor handle that this task was
* submitted through. If it is not an actor task, or if this is the
* original handle, then this is NIL_ACTOR_ID.
* @param actor_counter A counter indicating how many tasks have been submitted
* to the same actor before this one.
* @param is_actor_checkpoint_method True if this is an actor checkpoint method
* and false otherwise.
* @param function_id The function ID of the function to execute in this task.
* @param num_args The number of arguments that this task has.
* @param num_returns The number of return values that this task has.
* @param args_value_size The total size in bytes of the arguments to this task
ignoring object ID arguments.
* @return The partially constructed task_spec.
*/
void TaskSpec_start_construct(TaskBuilder *B,
UniqueID driver_id,
TaskID parent_task_id,
int64_t parent_counter,
ActorID actor_creation_id,
ObjectID actor_creation_dummy_object_id,
ActorID actor_id,
ActorHandleID actor_handle_id,
int64_t actor_counter,
bool is_actor_checkpoint_method,
FunctionID function_id,
int64_t num_returns);
/**
* Finish constructing a task_spec. This computes the task ID and the object IDs
* of the task return values. This must be called after all of the arguments
* have been added to the task.
*
* @param spec The task spec whose ID and return object IDs should be computed.
* @return Void.
*/
TaskSpec *TaskSpec_finish_construct(TaskBuilder *builder, int64_t *size);
/**
* Return the function ID of the task.
*
* @param spec The task_spec in question.
* @return The function ID of the function to execute in this task.
*/
FunctionID TaskSpec_function(TaskSpec *spec);
/**
* Return the actor ID of the task.
*
* @param spec The task_spec in question.
* @return The actor ID of the actor the task is part of.
*/
ActorID TaskSpec_actor_id(TaskSpec *spec);
/**
* Return the actor handle ID of the task.
*
* @param spec The task_spec in question.
* @return The ID of the actor handle that the task was submitted through.
*/
ActorID TaskSpec_actor_handle_id(TaskSpec *spec);
/**
* Return whether this task is for an actor.
*
* @param spec The task_spec in question.
* @return Whether the task is for an actor.
*/
bool TaskSpec_is_actor_task(TaskSpec *spec);
/// Return whether this task is an actor creation task or not.
///
/// \param spec The task_spec in question.
/// \return True if this task is an actor creation task and false otherwise.
bool TaskSpec_is_actor_creation_task(TaskSpec *spec);
/// Return the actor creation ID of the task. The task must be an actor creation
/// task.
///
/// \param spec The task_spec in question.
/// \return The actor creation ID if this is an actor creation task.
ActorID TaskSpec_actor_creation_id(TaskSpec *spec);
/// Return the actor creation dummy object ID of the task. The task must be an
/// actor task.
///
/// \param spec The task_spec in question.
/// \return The actor creation dummy object ID corresponding to this actor task.
ObjectID TaskSpec_actor_creation_dummy_object_id(TaskSpec *spec);
/**
* Return the actor counter of the task. This starts at 0 and increments by 1
* every time a new task is submitted to run on the actor.
*
* @param spec The task_spec in question.
* @return The actor counter of the task.
*/
int64_t TaskSpec_actor_counter(TaskSpec *spec);
/**
* Return whether the task is a checkpoint method execution.
*
* @param spec The task_spec in question.
* @return Whether the task is a checkpoint method.
*/
bool TaskSpec_is_actor_checkpoint_method(TaskSpec *spec);
/**
* Return an actor task's dummy return value. Dummy objects are used to
* encode an actor's state dependencies in the task graph. The dummy object
* is local if and only if the task that returned it has completed
* execution.
*
* @param spec The task_spec in question.
* @return The dummy object ID that the actor task will return.
*/
ObjectID TaskSpec_actor_dummy_object(TaskSpec *spec);
/**
* Return the driver ID of the task.
*
* @param spec The task_spec in question.
* @return The driver ID of the task.
*/
UniqueID TaskSpec_driver_id(const TaskSpec *spec);
/**
* Return the task ID of the parent task.
*
* @param spec The task_spec in question.
* @return The task ID of the parent task.
*/
TaskID TaskSpec_parent_task_id(const TaskSpec *spec);
/**
* Return the task counter of the parent task. For example, this equals 5 if
* this task was the 6th task submitted by the parent task.
*
* @param spec The task_spec in question.
* @return The task counter of the parent task.
*/
int64_t TaskSpec_parent_counter(TaskSpec *spec);
/**
* Return the task ID of the task.
*
* @param spec The task_spec in question.
* @return The task ID of the task.
*/
TaskID TaskSpec_task_id(const TaskSpec *spec);
/**
* Get the number of arguments to this task.
*
* @param spec The task_spec in question.
* @return The number of arguments to this task.
*/
int64_t TaskSpec_num_args(TaskSpec *spec);
/**
* Get the number of return values expected from this task.
*
* @param spec The task_spec in question.
* @return The number of return values expected from this task.
*/
int64_t TaskSpec_num_returns(TaskSpec *spec);
/**
* Return true if this argument is passed by reference.
*
* @param spec The task_spec in question.
* @param arg_index The index of the argument in question.
* @return True if this argument is passed by reference.
*/
bool TaskSpec_arg_by_ref(TaskSpec *spec, int64_t arg_index);
/**
* Get number of object IDs in a given argument
*
* @param spec The task_spec in question.
* @param arg_index The index of the argument in question.
* @return number of object IDs in this argument
*/
int TaskSpec_arg_id_count(TaskSpec *spec, int64_t arg_index);
/**
* Get a particular argument to this task. This assumes the argument is an
* object ID.
*
* @param spec The task_spec in question.
* @param arg_index The index of the argument in question.
* @param id_index The index of the object ID in this arg.
* @return The argument at that index.
*/
ObjectID TaskSpec_arg_id(TaskSpec *spec, int64_t arg_index, int64_t id_index);
/**
* Get a particular argument to this task. This assumes the argument is a value.
*
* @param spec The task_spec in question.
* @param arg_index The index of the argument in question.
* @return The argument at that index.
*/
const uint8_t *TaskSpec_arg_val(TaskSpec *spec, int64_t arg_index);
/**
* Get the number of bytes in a particular argument to this task. This assumes
* the argument is a value.
*
* @param spec The task_spec in question.
* @param arg_index The index of the argument in question.
* @return The number of bytes in the argument.
*/
int64_t TaskSpec_arg_length(TaskSpec *spec, int64_t arg_index);
/**
* Set the next task argument. Note that this API only allows you to set the
* arguments in their order of appearance.
*
* @param spec The task_spec in question.
* @param object_ids The object IDs to set the argument to.
* @param num_object_ids number of IDs in this param, usually 1.
* @return The number of task arguments that have been set before this one. This
* is only used for testing.
*/
void TaskSpec_args_add_ref(TaskBuilder *spec,
ObjectID object_ids[],
int num_object_ids);
/**
* Set the next task argument. Note that this API only allows you to set the
* arguments in their order of appearance.
*
* @param spec The task_spec in question.
* @param The value to set the argument to.
* @param The length of the value to set the argument to.
* @return The number of task arguments that have been set before this one. This
* is only used for testing.
*/
void TaskSpec_args_add_val(TaskBuilder *builder,
uint8_t *value,
int64_t length);
/**
* Set the value associated to a resource index.
*
* @param spec Task specification.
* @param resource_name Name of the resource in the resource vector.
* @param value Value for the resource. This can be a quantity of this resource
* this task needs or a value for an attribute this task requires.
* @return Void.
*/
void TaskSpec_set_required_resource(TaskBuilder *builder,
const std::string &resource_name,
double value);
/**
* Get a particular return object ID of a task.
*
* @param spec The task_spec in question.
* @param return_index The index of the return object ID in question.
* @return The relevant return object ID.
*/
ObjectID TaskSpec_return(TaskSpec *data, int64_t return_index);
/**
* Get the value associated to a resource name.
*
* @param spec Task specification.
* @param resource_name Name of the resource.
* @return How many of this resource the task needs to execute.
*/
double TaskSpec_get_required_resource(const TaskSpec *spec,
const std::string &resource_name);
/**
*
*/
const std::unordered_map<std::string, double> TaskSpec_get_required_resources(
const TaskSpec *spec);
/**
* Compute the object id associated to a put call.
*
* @param task_id The task id of the parent task that called the put.
* @param put_index The number of put calls in this task so far.
* @return The object ID for the object that was put.
*/
ObjectID task_compute_put_id(TaskID task_id, int64_t put_index);
/**
* Print the task as a humanly readable string.
*
* @param spec The task_spec in question.
* @return The humanly readable string.
*/
std::string TaskSpec_print(TaskSpec *spec);
/**
* Create a copy of the task spec. Must be freed with TaskSpec_free after use.
*
* @param spec The task specification that will be copied.
* @param task_spec_size The size of the task specification in bytes.
* @returns Pointer to the copy of the task specification.
*/
TaskSpec *TaskSpec_copy(TaskSpec *spec, int64_t task_spec_size);
/**
* Free a task_spec.
*
* @param The task_spec in question.
* @return Void.
*/
void TaskSpec_free(TaskSpec *spec);
/**
* ==== Task ====
* Contains information about a scheduled task: The task specification, the
* task scheduling state (WAITING, SCHEDULED, QUEUED, RUNNING, DONE), and which
* local scheduler the task is scheduled on.
*/
/** The scheduling_state can be used as a flag when we are listening
* for an event, for example TASK_WAITING | TASK_SCHEDULED. */
enum class TaskStatus : uint {
/** The task is waiting to be scheduled. */
WAITING = 1,
/** The task has been scheduled to a node, but has not been queued yet. */
SCHEDULED = 2,
/** The task has been queued on a node, where it will wait for its
* dependencies to become ready and a worker to become available. */
QUEUED = 4,
/** The task is running on a worker. */
RUNNING = 8,
/** The task is done executing. */
DONE = 16,
/** The task was not able to finish. */
LOST = 32,
/** The task will be submitted for reexecution. */
RECONSTRUCTING = 64,
/** An actor task is cached at a local scheduler and is waiting for the
* corresponding actor to be created. */
ACTOR_CACHED = 128
};
inline TaskStatus operator|(const TaskStatus &a, const TaskStatus &b) {
uint c = static_cast<uint>(a) | static_cast<uint>(b);
return static_cast<TaskStatus>(c);
}
/** A task is an execution of a task specification. It has a state of execution
* (see scheduling_state) and the ID of the local scheduler it is scheduled on
* or running on. */
struct Task {
/** The scheduling state of the task. */
TaskStatus state;
/** The ID of the local scheduler involved. */
DBClientID local_scheduler_id;
/** The execution specification for this task. */
std::unique_ptr<TaskExecutionSpec> execution_spec;
};
/**
* Allocate a new task. Must be freed with free_task after use.
*
* @param spec The task spec for the new task.
* @param state The scheduling state for the new task.
* @param local_scheduler_id The ID of the local scheduler that the task is
* scheduled on, if any.
*/
Task *Task_alloc(const TaskSpec *spec,
int64_t task_spec_size,
TaskStatus state,
DBClientID local_scheduler_id,
const std::vector<ObjectID> &execution_dependencies);
Task *Task_alloc(TaskExecutionSpec &execution_spec,
TaskStatus state,
DBClientID local_scheduler_id);
/**
* Create a copy of the task. Must be freed with Task_free after use.
*
* @param other The task that will be copied.
* @returns Pointer to the copy of the task.
*/
Task *Task_copy(Task *other);
/** Size of task structure in bytes. */
int64_t Task_size(Task *task);
/** The scheduling state of the task. */
TaskStatus Task_state(Task *task);
/** Update the schedule state of the task. */
void Task_set_state(Task *task, TaskStatus state);
/** Local scheduler this task has been assigned to or is running on. */
DBClientID Task_local_scheduler(Task *task);
/** Set the local scheduler ID for this task. */
void Task_set_local_scheduler(Task *task, DBClientID local_scheduler_id);
TaskExecutionSpec *Task_task_execution_spec(Task *task);
/** Task ID of this task. */
TaskID Task_task_id(Task *task);
/** Free this task datastructure. */
void Task_free(Task *task);
#endif /* TASK_H */
src/common/test/db_tests.cc
-246
View File
@@ -1,246 +0,0 @@
#include "greatest.h"
#include <assert.h>
#include <unistd.h>
#include <sys/wait.h>
#include "event_loop.h"
#include "test_common.h"
#include "example_task.h"
#include "net.h"
#include "state/db.h"
#include "state/db_client_table.h"
#include "state/object_table.h"
#include "state/task_table.h"
#include "state/redis.h"
#include "task.h"
SUITE(db_tests);
TaskBuilder *g_task_builder = NULL;
/* Retry 10 times with an 100ms timeout. */
const int NUM_RETRIES = 10;
const uint64_t TIMEOUT = 50;
const char *manager_addr = "127.0.0.1";
int manager_port1 = 12345;
int manager_port2 = 12346;
char received_addr1[16] = {0};
int received_port1;
char received_addr2[16] = {0};
int received_port2;
typedef struct { int test_number; } user_context;
const int TEST_NUMBER = 10;
/* Test if entries have been written to the database. */
void lookup_done_callback(ObjectID object_id,
bool never_created,
const std::vector<DBClientID> &manager_ids,
void *user_context) {
DBHandle *db = (DBHandle *) user_context;
RAY_CHECK(manager_ids.size() == 2);
const std::vector<std::string> managers =
db_client_table_get_ip_addresses(db, manager_ids);
RAY_CHECK(parse_ip_addr_port(managers.at(0).c_str(), received_addr1,
&received_port1) == 0);
RAY_CHECK(parse_ip_addr_port(managers.at(1).c_str(), received_addr2,
&received_port2) == 0);
}
/* Entry added to database successfully. */
void add_done_callback(ObjectID object_id, bool success, void *user_context) {}
/* Test if we got a timeout callback if we couldn't connect database. */
void timeout_callback(ObjectID object_id, void *context, void *user_data) {
user_context *uc = (user_context *) context;
RAY_CHECK(uc->test_number == TEST_NUMBER);
}
int64_t timeout_handler(event_loop *loop, int64_t id, void *context) {
event_loop_stop(loop);
return EVENT_LOOP_TIMER_DONE;
}
TEST object_table_lookup_test(void) {
event_loop *loop = event_loop_create();
/* This uses manager_port1. */
std::vector<std::string> db_connect_args1;
db_connect_args1.push_back("manager_address");
db_connect_args1.push_back("127.0.0.1:12345");
DBHandle *db1 = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
manager_addr, db_connect_args1);
/* This uses manager_port2. */
std::vector<std::string> db_connect_args2;
db_connect_args2.push_back("manager_address");
db_connect_args2.push_back("127.0.0.1:12346");
DBHandle *db2 = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
manager_addr, db_connect_args2);
db_attach(db1, loop, false);
db_attach(db2, loop, false);
UniqueID id = UniqueID::from_random();
RetryInfo retry = {
.num_retries = NUM_RETRIES,
.timeout = TIMEOUT,
.fail_callback = timeout_callback,
};
object_table_add(db1, id, 0, (unsigned char *) NIL_DIGEST, &retry,
add_done_callback, NULL);
object_table_add(db2, id, 0, (unsigned char *) NIL_DIGEST, &retry,
add_done_callback, NULL);
event_loop_add_timer(loop, 200, (event_loop_timer_handler) timeout_handler,
NULL);
event_loop_run(loop);
object_table_lookup(db1, id, &retry, lookup_done_callback, db1);
event_loop_add_timer(loop, 200, (event_loop_timer_handler) timeout_handler,
NULL);
event_loop_run(loop);
ASSERT_STR_EQ(&received_addr1[0], manager_addr);
ASSERT((received_port1 == manager_port1 && received_port2 == manager_port2) ||
(received_port2 == manager_port1 && received_port1 == manager_port2));
db_disconnect(db1);
db_disconnect(db2);
destroy_outstanding_callbacks(loop);
event_loop_destroy(loop);
PASS();
}
int task_table_test_callback_called = 0;
Task *task_table_test_task;
void task_table_test_fail_callback(UniqueID id,
void *context,
void *user_data) {
event_loop *loop = (event_loop *) user_data;
event_loop_stop(loop);
}
int64_t task_table_delayed_add_task(event_loop *loop,
int64_t id,
void *context) {
DBHandle *db = (DBHandle *) context;
RetryInfo retry = {
.num_retries = NUM_RETRIES,
.timeout = TIMEOUT,
.fail_callback = task_table_test_fail_callback,
};
task_table_add_task(db, Task_copy(task_table_test_task), &retry, NULL,
(void *) loop);
return EVENT_LOOP_TIMER_DONE;
}
void task_table_test_callback(Task *callback_task, void *user_data) {
task_table_test_callback_called = 1;
RAY_CHECK(Task_state(callback_task) == TaskStatus::SCHEDULED);
RAY_CHECK(Task_size(callback_task) == Task_size(task_table_test_task));
RAY_CHECK(Task_equals(callback_task, task_table_test_task));
event_loop *loop = (event_loop *) user_data;
event_loop_stop(loop);
}
TEST task_table_test(void) {
task_table_test_callback_called = 0;
event_loop *loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "local_scheduler",
"127.0.0.1", std::vector<std::string>());
db_attach(db, loop, false);
DBClientID local_scheduler_id = DBClientID::from_random();
TaskExecutionSpec spec = example_task_execution_spec(1, 1);
task_table_test_task =
Task_alloc(spec, TaskStatus::SCHEDULED, local_scheduler_id);
RetryInfo retry = {
.num_retries = NUM_RETRIES,
.timeout = TIMEOUT,
.fail_callback = task_table_test_fail_callback,
};
task_table_subscribe(db, local_scheduler_id, TaskStatus::SCHEDULED,
task_table_test_callback, (void *) loop, &retry, NULL,
(void *) loop);
event_loop_add_timer(
loop, 200, (event_loop_timer_handler) task_table_delayed_add_task, db);
event_loop_run(loop);
Task_free(task_table_test_task);
db_disconnect(db);
destroy_outstanding_callbacks(loop);
event_loop_destroy(loop);
ASSERT(task_table_test_callback_called);
PASS();
}
int num_test_callback_called = 0;
void task_table_all_test_callback(Task *task, void *user_data) {
num_test_callback_called += 1;
}
TEST task_table_all_test(void) {
event_loop *loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "local_scheduler",
"127.0.0.1", std::vector<std::string>());
db_attach(db, loop, false);
TaskExecutionSpec spec = example_task_execution_spec(1, 1);
/* Schedule two tasks on different local local schedulers. */
Task *task1 =
Task_alloc(spec, TaskStatus::SCHEDULED, DBClientID::from_random());
Task *task2 =
Task_alloc(spec, TaskStatus::SCHEDULED, DBClientID::from_random());
RetryInfo retry = {
.num_retries = NUM_RETRIES, .timeout = TIMEOUT, .fail_callback = NULL,
};
task_table_subscribe(db, UniqueID::nil(), TaskStatus::SCHEDULED,
task_table_all_test_callback, NULL, &retry, NULL, NULL);
event_loop_add_timer(loop, 50, (event_loop_timer_handler) timeout_handler,
NULL);
event_loop_run(loop);
/* TODO(pcm): Get rid of this sleep once the robust pubsub is implemented. */
task_table_add_task(db, task1, &retry, NULL, NULL);
task_table_add_task(db, task2, &retry, NULL, NULL);
event_loop_add_timer(loop, 200, (event_loop_timer_handler) timeout_handler,
NULL);
event_loop_run(loop);
db_disconnect(db);
destroy_outstanding_callbacks(loop);
event_loop_destroy(loop);
ASSERT(num_test_callback_called == 2);
PASS();
}
TEST unique_client_id_test(void) {
const int num_conns = 100;
DBClientID ids[num_conns];
DBHandle *db;
for (int i = 0; i < num_conns; ++i) {
db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
ids[i] = get_db_client_id(db);
db_disconnect(db);
}
for (int i = 0; i < num_conns; ++i) {
for (int j = 0; j < i; ++j) {
ASSERT(!(ids[i] == ids[j]));
}
}
PASS();
}
SUITE(db_tests) {
RUN_REDIS_TEST(object_table_lookup_test);
RUN_REDIS_TEST(task_table_test);
RUN_REDIS_TEST(task_table_all_test);
RUN_REDIS_TEST(unique_client_id_test);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
g_task_builder = make_task_builder();
GREATEST_MAIN_BEGIN();
RUN_SUITE(db_tests);
GREATEST_MAIN_END();
}
src/common/test/example_task.h
-77
View File
@@ -1,77 +0,0 @@
#ifndef EXAMPLE_TASK_H
#define EXAMPLE_TASK_H
#include "task.h"
extern TaskBuilder *g_task_builder;
const int64_t arg_value_size = 1000;
static inline TaskExecutionSpec example_task_execution_spec_with_args(
int64_t num_args,
int64_t num_returns,
ObjectID arg_ids[]) {
TaskID parent_task_id = TaskID::from_random();
FunctionID func_id = FunctionID::from_random();
TaskSpec_start_construct(g_task_builder, UniqueID::nil(), parent_task_id, 0,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, num_returns);
for (int64_t i = 0; i < num_args; ++i) {
ObjectID arg_id;
if (arg_ids == NULL) {
arg_id = ObjectID::from_random();
} else {
arg_id = arg_ids[i];
}
TaskSpec_args_add_ref(g_task_builder, &arg_id, 1);
}
int64_t task_spec_size;
TaskSpec *spec = TaskSpec_finish_construct(g_task_builder, &task_spec_size);
std::vector<ObjectID> execution_dependencies;
auto execution_spec =
TaskExecutionSpec(execution_dependencies, spec, task_spec_size);
TaskSpec_free(spec);
return execution_spec;
}
static inline TaskExecutionSpec example_task_execution_spec(
int64_t num_args,
int64_t num_returns) {
return example_task_execution_spec_with_args(num_args, num_returns, NULL);
}
static inline Task *example_task_with_args(int64_t num_args,
int64_t num_returns,
TaskStatus task_state,
ObjectID arg_ids[]) {
TaskExecutionSpec spec =
example_task_execution_spec_with_args(num_args, num_returns, arg_ids);
Task *instance = Task_alloc(spec, task_state, UniqueID::nil());
return instance;
}
static inline Task *example_task(int64_t num_args,
int64_t num_returns,
TaskStatus task_state) {
TaskExecutionSpec spec = example_task_execution_spec(num_args, num_returns);
Task *instance = Task_alloc(spec, task_state, UniqueID::nil());
return instance;
}
static inline bool Task_equals(Task *task1, Task *task2) {
if (task1->state != task2->state) {
return false;
}
if (!(task1->local_scheduler_id == task2->local_scheduler_id)) {
return false;
}
auto execution_spec1 = Task_task_execution_spec(task1);
auto execution_spec2 = Task_task_execution_spec(task2);
if (execution_spec1->SpecSize() != execution_spec2->SpecSize()) {
return false;
}
return memcmp(execution_spec1->Spec(), execution_spec2->Spec(),
execution_spec1->SpecSize()) == 0;
}
#endif /* EXAMPLE_TASK_H */
src/common/test/io_tests.cc
-114
View File
@@ -1,114 +0,0 @@
#include "greatest.h"
#include <assert.h>
#include <unistd.h>
#include <inttypes.h>
#include <sstream>
#include <string>
#include "io.h"
SUITE(io_tests);
TEST ipc_socket_test(void) {
#ifndef _WIN32
const char *socket_pathname = "/tmp/test-socket";
int socket_fd = bind_ipc_sock(socket_pathname, true);
ASSERT(socket_fd >= 0);
const char *test_string = "hello world";
const char *test_bytes = "another string";
pid_t pid = fork();
if (pid == 0) {
close(socket_fd);
socket_fd = connect_ipc_sock(socket_pathname);
ASSERT(socket_fd >= 0);
write_log_message(socket_fd, test_string);
write_message(socket_fd,
static_cast<int64_t>(CommonMessageType::LOG_MESSAGE),
strlen(test_bytes), (uint8_t *) test_bytes);
close(socket_fd);
exit(0);
} else {
int client_fd = accept_client(socket_fd);
ASSERT(client_fd >= 0);
char *message = read_log_message(client_fd);
ASSERT(message != NULL);
ASSERT_STR_EQ(test_string, message);
free(message);
int64_t type;
int64_t len;
uint8_t *bytes;
read_message(client_fd, &type, &len, &bytes);
ASSERT(static_cast<CommonMessageType>(type) ==
CommonMessageType::LOG_MESSAGE);
ASSERT(memcmp(test_bytes, bytes, len) == 0);
free(bytes);
close(client_fd);
close(socket_fd);
unlink(socket_pathname);
}
#endif
PASS();
}
TEST long_ipc_socket_test(void) {
#ifndef _WIN32
const char *socket_pathname = "/tmp/long-test-socket";
int socket_fd = bind_ipc_sock(socket_pathname, true);
ASSERT(socket_fd >= 0);
std::stringstream test_string_ss;
for (int i = 0; i < 10000; i++) {
test_string_ss << "hello world ";
}
std::string test_string = test_string_ss.str();
const char *test_bytes = "another string";
pid_t pid = fork();
if (pid == 0) {
close(socket_fd);
socket_fd = connect_ipc_sock(socket_pathname);
ASSERT(socket_fd >= 0);
write_log_message(socket_fd, test_string.c_str());
write_message(socket_fd,
static_cast<int64_t>(CommonMessageType::LOG_MESSAGE),
strlen(test_bytes), (uint8_t *) test_bytes);
close(socket_fd);
exit(0);
} else {
int client_fd = accept_client(socket_fd);
ASSERT(client_fd >= 0);
char *message = read_log_message(client_fd);
ASSERT(message != NULL);
ASSERT_STR_EQ(test_string.c_str(), message);
free(message);
int64_t type;
int64_t len;
uint8_t *bytes;
read_message(client_fd, &type, &len, &bytes);
ASSERT(static_cast<CommonMessageType>(type) ==
CommonMessageType::LOG_MESSAGE);
ASSERT(memcmp(test_bytes, bytes, len) == 0);
free(bytes);
close(client_fd);
close(socket_fd);
unlink(socket_pathname);
}
#endif
PASS();
}
SUITE(io_tests) {
RUN_TEST(ipc_socket_test);
RUN_TEST(long_ipc_socket_test);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(io_tests);
GREATEST_MAIN_END();
}
src/common/test/object_table_tests.cc
-919
View File
@@ -1,919 +0,0 @@
#include "greatest.h"
#include "event_loop.h"
#include "example_task.h"
#include "test_common.h"
#include "common.h"
#include "state/db_client_table.h"
#include "state/object_table.h"
#include "state/redis.h"
#include <unistd.h>
SUITE(object_table_tests);
static event_loop *g_loop;
TaskBuilder *g_task_builder = NULL;
/* ==== Test adding and looking up metadata ==== */
int new_object_failed = 0;
int new_object_succeeded = 0;
ObjectID new_object_id;
Task *new_object_task;
TaskSpec *new_object_task_spec;
TaskID new_object_task_id;
void new_object_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
new_object_failed = 1;
event_loop_stop(g_loop);
}
/* === Test adding an object with an associated task === */
void new_object_done_callback(ObjectID object_id,
TaskID task_id,
bool is_put,
void *user_context) {
new_object_succeeded = 1;
RAY_CHECK(object_id == new_object_id);
RAY_CHECK(task_id == new_object_task_id);
event_loop_stop(g_loop);
}
void new_object_lookup_callback(ObjectID object_id, void *user_context) {
RAY_CHECK(object_id == new_object_id);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = new_object_fail_callback,
};
DBHandle *db = (DBHandle *) user_context;
result_table_lookup(db, new_object_id, &retry, new_object_done_callback,
NULL);
}
void new_object_task_callback(TaskID task_id, void *user_context) {
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = new_object_fail_callback,
};
DBHandle *db = (DBHandle *) user_context;
result_table_add(db, new_object_id, new_object_task_id, false, &retry,
new_object_lookup_callback, (void *) db);
}
void task_table_subscribe_done(TaskID task_id, void *user_context) {
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = NULL,
};
DBHandle *db = (DBHandle *) user_context;
task_table_add_task(db, Task_copy(new_object_task), &retry,
new_object_task_callback, db);
}
TEST new_object_test(void) {
new_object_failed = 0;
new_object_succeeded = 0;
new_object_id = ObjectID::from_random();
new_object_task = example_task(1, 1, TaskStatus::WAITING);
new_object_task_spec = Task_task_execution_spec(new_object_task)->Spec();
new_object_task_id = TaskSpec_task_id(new_object_task_spec);
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = new_object_fail_callback,
};
task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL,
&retry, task_table_subscribe_done, db);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(new_object_succeeded);
ASSERT(!new_object_failed);
PASS();
}
/* === Test adding an object without an associated task === */
void new_object_no_task_callback(ObjectID object_id,
TaskID task_id,
bool is_put,
void *user_context) {
new_object_succeeded = 1;
RAY_CHECK(task_id.is_nil());
event_loop_stop(g_loop);
}
TEST new_object_no_task_test(void) {
new_object_failed = 0;
new_object_succeeded = 0;
new_object_id = ObjectID::from_random();
new_object_task_id = TaskID::from_random();
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = new_object_fail_callback,
};
result_table_lookup(db, new_object_id, &retry, new_object_no_task_callback,
NULL);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(new_object_succeeded);
ASSERT(!new_object_failed);
PASS();
}
/* ==== Test if operations time out correctly ==== */
/* === Test lookup timeout === */
const char *lookup_timeout_context = "lookup_timeout";
int lookup_failed = 0;
void lookup_done_callback(ObjectID object_id,
bool never_created,
const std::vector<DBClientID> &manager_vector,
void *context) {
/* The done callback should not be called. */
RAY_CHECK(0);
}
void lookup_fail_callback(UniqueID id, void *user_context, void *user_data) {
lookup_failed = 1;
RAY_CHECK(user_context == (void *) lookup_timeout_context);
event_loop_stop(g_loop);
}
TEST lookup_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = lookup_fail_callback,
};
object_table_lookup(db, UniqueID::nil(), &retry, lookup_done_callback,
(void *) lookup_timeout_context);
/* Disconnect the database to see if the lookup times out. */
close(db->context->c.fd);
for (auto context : db->contexts) {
close(context->c.fd);
}
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(lookup_failed);
PASS();
}
/* === Test add timeout === */
const char *add_timeout_context = "add_timeout";
int add_failed = 0;
void add_done_callback(ObjectID object_id, bool success, void *user_context) {
/* The done callback should not be called. */
RAY_CHECK(0);
}
void add_fail_callback(UniqueID id, void *user_context, void *user_data) {
add_failed = 1;
RAY_CHECK(user_context == (void *) add_timeout_context);
event_loop_stop(g_loop);
}
TEST add_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = add_fail_callback,
};
object_table_add(db, UniqueID::nil(), 0, (unsigned char *) NIL_DIGEST, &retry,
add_done_callback, (void *) add_timeout_context);
/* Disconnect the database to see if the lookup times out. */
close(db->context->c.fd);
for (auto context : db->contexts) {
close(context->c.fd);
}
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(add_failed);
PASS();
}
/* === Test subscribe timeout === */
int subscribe_failed = 0;
void subscribe_done_callback(ObjectID object_id,
int64_t data_size,
const std::vector<DBClientID> &manager_vector,
void *user_context) {
/* The done callback should not be called. */
RAY_CHECK(0);
}
void subscribe_fail_callback(UniqueID id, void *user_context, void *user_data) {
subscribe_failed = 1;
event_loop_stop(g_loop);
}
TEST subscribe_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = subscribe_fail_callback,
};
object_table_subscribe_to_notifications(db, false, subscribe_done_callback,
NULL, &retry, NULL, NULL);
/* Disconnect the database to see if the lookup times out. */
close(db->subscribe_context->c.fd);
for (auto subscribe_context : db->subscribe_contexts) {
close(subscribe_context->c.fd);
}
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(subscribe_failed);
PASS();
}
/* ==== Test if the retry is working correctly ==== */
int64_t reconnect_context_callback(event_loop *loop,
int64_t timer_id,
void *context) {
DBHandle *db = (DBHandle *) context;
/* Reconnect to redis. This is not reconnecting the pub/sub channel. */
redisAsyncFree(db->context);
redisFree(db->sync_context);
db->context = redisAsyncConnect("127.0.0.1", 6379);
db->context->data = (void *) db;
db->sync_context = redisConnect("127.0.0.1", 6379);
/* Re-attach the database to the event loop (the file descriptor changed). */
db_attach(db, loop, true);
RAY_LOG(DEBUG) << "Reconnected to Redis";
return EVENT_LOOP_TIMER_DONE;
}
int64_t terminate_event_loop_callback(event_loop *loop,
int64_t timer_id,
void *context) {
event_loop_stop(loop);
return EVENT_LOOP_TIMER_DONE;
}
/* === Test lookup retry === */
const char *lookup_retry_context = "lookup_retry";
int lookup_retry_succeeded = 0;
void lookup_retry_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
/* The fail callback should not be called. */
RAY_CHECK(0);
}
/* === Test add retry === */
const char *add_retry_context = "add_retry";
int add_retry_succeeded = 0;
/* === Test add then lookup retry === */
void add_lookup_done_callback(ObjectID object_id,
bool never_created,
const std::vector<DBClientID> &manager_ids,
void *context) {
DBHandle *db = (DBHandle *) context;
RAY_CHECK(manager_ids.size() == 1);
const std::vector<std::string> managers =
db_client_table_get_ip_addresses(db, manager_ids);
RAY_CHECK(managers.at(0) == "127.0.0.1:11235");
lookup_retry_succeeded = 1;
}
void add_lookup_callback(ObjectID object_id, bool success, void *user_context) {
RAY_CHECK(success);
DBHandle *db = (DBHandle *) user_context;
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = lookup_retry_fail_callback,
};
object_table_lookup(db, UniqueID::nil(), &retry, add_lookup_done_callback,
(void *) db);
}
TEST add_lookup_test(void) {
g_loop = event_loop_create();
lookup_retry_succeeded = 0;
/* Construct the arguments to db_connect. */
std::vector<std::string> db_connect_args;
db_connect_args.push_back("manager_address");
db_connect_args.push_back("127.0.0.1:11235");
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", db_connect_args);
db_attach(db, g_loop, true);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = lookup_retry_fail_callback,
};
object_table_add(db, UniqueID::nil(), 0, (unsigned char *) NIL_DIGEST, &retry,
add_lookup_callback, (void *) db);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(lookup_retry_succeeded);
PASS();
}
/* === Test add, remove, then lookup === */
void add_remove_lookup_done_callback(
ObjectID object_id,
bool never_created,
const std::vector<DBClientID> &manager_vector,
void *context) {
RAY_CHECK(context == (void *) lookup_retry_context);
RAY_CHECK(manager_vector.size() == 0);
lookup_retry_succeeded = 1;
}
void add_remove_lookup_callback(ObjectID object_id,
bool success,
void *user_context) {
RAY_CHECK(success);
DBHandle *db = (DBHandle *) user_context;
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = lookup_retry_fail_callback,
};
object_table_lookup(db, UniqueID::nil(), &retry,
add_remove_lookup_done_callback,
(void *) lookup_retry_context);
}
void add_remove_callback(ObjectID object_id, bool success, void *user_context) {
RAY_CHECK(success);
DBHandle *db = (DBHandle *) user_context;
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = lookup_retry_fail_callback,
};
object_table_remove(db, UniqueID::nil(), NULL, &retry,
add_remove_lookup_callback, (void *) db);
}
TEST add_remove_lookup_test(void) {
g_loop = event_loop_create();
lookup_retry_succeeded = 0;
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, true);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = lookup_retry_fail_callback,
};
object_table_add(db, UniqueID::nil(), 0, (unsigned char *) NIL_DIGEST, &retry,
add_remove_callback, (void *) db);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(lookup_retry_succeeded);
PASS();
}
/* ==== Test if late succeed is working correctly ==== */
/* === Test lookup late succeed === */
const char *lookup_late_context = "lookup_late";
int lookup_late_failed = 0;
void lookup_late_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
RAY_CHECK(user_context == (void *) lookup_late_context);
lookup_late_failed = 1;
}
void lookup_late_done_callback(ObjectID object_id,
bool never_created,
const std::vector<DBClientID> &manager_vector,
void *context) {
/* This function should never be called. */
RAY_CHECK(0);
}
TEST lookup_late_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0,
.timeout = 0,
.fail_callback = lookup_late_fail_callback,
};
object_table_lookup(db, UniqueID::nil(), &retry, lookup_late_done_callback,
(void *) lookup_late_context);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* First process timer events to make sure the timeout is processed before
* anything else. */
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(lookup_late_failed);
PASS();
}
/* === Test add late succeed === */
const char *add_late_context = "add_late";
int add_late_failed = 0;
void add_late_fail_callback(UniqueID id, void *user_context, void *user_data) {
RAY_CHECK(user_context == (void *) add_late_context);
add_late_failed = 1;
}
void add_late_done_callback(ObjectID object_id,
bool success,
void *user_context) {
/* This function should never be called. */
RAY_CHECK(0);
}
TEST add_late_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0, .timeout = 0, .fail_callback = add_late_fail_callback,
};
object_table_add(db, UniqueID::nil(), 0, (unsigned char *) NIL_DIGEST, &retry,
add_late_done_callback, (void *) add_late_context);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* First process timer events to make sure the timeout is processed before
* anything else. */
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(add_late_failed);
PASS();
}
/* === Test subscribe late succeed === */
const char *subscribe_late_context = "subscribe_late";
int subscribe_late_failed = 0;
void subscribe_late_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
RAY_CHECK(user_context == (void *) subscribe_late_context);
subscribe_late_failed = 1;
}
void subscribe_late_done_callback(ObjectID object_id,
bool never_created,
const std::vector<DBClientID> &manager_vector,
void *user_context) {
/* This function should never be called. */
RAY_CHECK(0);
}
TEST subscribe_late_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0,
.timeout = 0,
.fail_callback = subscribe_late_fail_callback,
};
object_table_subscribe_to_notifications(db, false, NULL, NULL, &retry,
subscribe_late_done_callback,
(void *) subscribe_late_context);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* First process timer events to make sure the timeout is processed before
* anything else. */
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(subscribe_late_failed);
PASS();
}
/* === Test subscribe object available succeed === */
const char *subscribe_success_context = "subscribe_success";
int subscribe_success_done = 0;
int subscribe_success_succeeded = 0;
ObjectID subscribe_id;
void subscribe_success_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
/* This function should never be called. */
RAY_CHECK(0);
}
void subscribe_success_done_callback(
ObjectID object_id,
bool never_created,
const std::vector<DBClientID> &manager_vector,
void *user_context) {
RetryInfo retry = {
.num_retries = 0, .timeout = 750, .fail_callback = NULL,
};
object_table_add((DBHandle *) user_context, subscribe_id, 0,
(unsigned char *) NIL_DIGEST, &retry, NULL, NULL);
subscribe_success_done = 1;
}
void subscribe_success_object_available_callback(
ObjectID object_id,
int64_t data_size,
const std::vector<DBClientID> &manager_vector,
void *user_context) {
RAY_CHECK(user_context == (void *) subscribe_success_context);
RAY_CHECK(object_id == subscribe_id);
RAY_CHECK(manager_vector.size() == 1);
subscribe_success_succeeded = 1;
}
TEST subscribe_success_test(void) {
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
std::vector<std::string> db_connect_args;
db_connect_args.push_back("manager_address");
db_connect_args.push_back("127.0.0.1:11236");
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", db_connect_args);
db_attach(db, g_loop, false);
subscribe_id = ObjectID::from_random();
RetryInfo retry = {
.num_retries = 0,
.timeout = 100,
.fail_callback = subscribe_success_fail_callback,
};
object_table_subscribe_to_notifications(
db, false, subscribe_success_object_available_callback,
(void *) subscribe_success_context, &retry,
subscribe_success_done_callback, (void *) db);
ObjectID object_ids[1] = {subscribe_id};
object_table_request_notifications(db, 1, object_ids, &retry);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(subscribe_success_done);
ASSERT(subscribe_success_succeeded);
PASS();
}
/* Test if subscribe succeeds if the object is already present. */
typedef struct {
const char *teststr;
int64_t data_size;
} subscribe_object_present_context_t;
const char *subscribe_object_present_str = "subscribe_object_present";
int subscribe_object_present_succeeded = 0;
void subscribe_object_present_object_available_callback(
ObjectID object_id,
int64_t data_size,
const std::vector<DBClientID> &manager_vector,
void *user_context) {
subscribe_object_present_context_t *ctx =
(subscribe_object_present_context_t *) user_context;
RAY_CHECK(ctx->data_size == data_size);
RAY_CHECK(strcmp(subscribe_object_present_str, ctx->teststr) == 0);
subscribe_object_present_succeeded = 1;
RAY_CHECK(manager_vector.size() == 1);
}
void fatal_fail_callback(UniqueID id, void *user_context, void *user_data) {
/* This function should never be called. */
RAY_CHECK(0);
}
TEST subscribe_object_present_test(void) {
int64_t data_size = 0xF1F0;
subscribe_object_present_context_t myctx = {subscribe_object_present_str,
data_size};
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
std::vector<std::string> db_connect_args;
db_connect_args.push_back("manager_address");
db_connect_args.push_back("127.0.0.1:11236");
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", db_connect_args);
db_attach(db, g_loop, false);
UniqueID id = UniqueID::from_random();
RetryInfo retry = {
.num_retries = 0, .timeout = 100, .fail_callback = fatal_fail_callback,
};
object_table_add(db, id, data_size, (unsigned char *) NIL_DIGEST, &retry,
NULL, NULL);
object_table_subscribe_to_notifications(
db, false, subscribe_object_present_object_available_callback,
(void *) &myctx, &retry, NULL, (void *) db);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* Run the event loop to create do the add and subscribe. */
event_loop_run(g_loop);
ObjectID object_ids[1] = {id};
object_table_request_notifications(db, 1, object_ids, &retry);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* Run the event loop to do the request notifications. */
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(subscribe_object_present_succeeded == 1);
PASS();
}
/* Test if subscribe is not called if object is not present. */
const char *subscribe_object_not_present_context =
"subscribe_object_not_present";
void subscribe_object_not_present_object_available_callback(
ObjectID object_id,
int64_t data_size,
const std::vector<DBClientID> &manager_vector,
void *user_context) {
/* This should not be called. */
RAY_CHECK(0);
}
TEST subscribe_object_not_present_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
UniqueID id = UniqueID::from_random();
RetryInfo retry = {
.num_retries = 0, .timeout = 100, .fail_callback = NULL,
};
object_table_subscribe_to_notifications(
db, false, subscribe_object_not_present_object_available_callback,
(void *) subscribe_object_not_present_context, &retry, NULL, (void *) db);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* Run the event loop to do the subscribe. */
event_loop_run(g_loop);
ObjectID object_ids[1] = {id};
object_table_request_notifications(db, 1, object_ids, &retry);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* Run the event loop to do the request notifications. */
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
PASS();
}
/* Test if subscribe is called if object becomes available later. */
const char *subscribe_object_available_later_context =
"subscribe_object_available_later";
int subscribe_object_available_later_succeeded = 0;
void subscribe_object_available_later_object_available_callback(
ObjectID object_id,
int64_t data_size,
const std::vector<DBClientID> &manager_vector,
void *user_context) {
subscribe_object_present_context_t *myctx =
(subscribe_object_present_context_t *) user_context;
RAY_CHECK(myctx->data_size == data_size);
RAY_CHECK(strcmp(myctx->teststr, subscribe_object_available_later_context) ==
0);
/* Make sure the callback is only called once. */
subscribe_object_available_later_succeeded += 1;
RAY_CHECK(manager_vector.size() == 1);
}
TEST subscribe_object_available_later_test(void) {
int64_t data_size = 0xF1F0;
subscribe_object_present_context_t *myctx =
(subscribe_object_present_context_t *) malloc(
sizeof(subscribe_object_present_context_t));
myctx->teststr = subscribe_object_available_later_context;
myctx->data_size = data_size;
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
std::vector<std::string> db_connect_args;
db_connect_args.push_back("manager_address");
db_connect_args.push_back("127.0.0.1:11236");
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", db_connect_args);
db_attach(db, g_loop, false);
UniqueID id = UniqueID::from_random();
RetryInfo retry = {
.num_retries = 0, .timeout = 100, .fail_callback = NULL,
};
object_table_subscribe_to_notifications(
db, false, subscribe_object_available_later_object_available_callback,
(void *) myctx, &retry, NULL, (void *) db);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* Run the event loop to do the subscribe. */
event_loop_run(g_loop);
ObjectID object_ids[1] = {id};
object_table_request_notifications(db, 1, object_ids, &retry);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* Run the event loop to do the request notifications. */
event_loop_run(g_loop);
ASSERT_EQ(subscribe_object_available_later_succeeded, 0);
object_table_add(db, id, data_size, (unsigned char *) NIL_DIGEST, &retry,
NULL, NULL);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* Run the event loop to do the object table add. */
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT_EQ(subscribe_object_available_later_succeeded, 1);
/* Reset the global variable before exiting this unit test. */
subscribe_object_available_later_succeeded = 0;
free(myctx);
PASS();
}
TEST subscribe_object_available_subscribe_all(void) {
int64_t data_size = 0xF1F0;
subscribe_object_present_context_t myctx = {
subscribe_object_available_later_context, data_size};
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
std::vector<std::string> db_connect_args;
db_connect_args.push_back("manager_address");
db_connect_args.push_back("127.0.0.1:11236");
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", db_connect_args);
db_attach(db, g_loop, false);
UniqueID id = UniqueID::from_random();
RetryInfo retry = {
.num_retries = 0, .timeout = 100, .fail_callback = NULL,
};
object_table_subscribe_to_notifications(
db, true, subscribe_object_available_later_object_available_callback,
(void *) &myctx, &retry, NULL, (void *) db);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* Run the event loop to do the subscribe. */
event_loop_run(g_loop);
/* At this point we don't expect any object notifications received. */
ASSERT_EQ(subscribe_object_available_later_succeeded, 0);
object_table_add(db, id, data_size, (unsigned char *) NIL_DIGEST, &retry,
NULL, NULL);
/* Install handler to terminate event loop after 750ms. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* Run the event loop to do the object table add. */
event_loop_run(g_loop);
/* At this point we assume that object table add completed. */
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
/* Assert that the object table add completed and notification callback fired.
*/
printf("subscribe_all object info test: callback fired: %d times\n",
subscribe_object_available_later_succeeded);
fflush(stdout);
ASSERT_EQ(subscribe_object_available_later_succeeded, 1);
/* Reset the global variable before exiting this unit test. */
subscribe_object_available_later_succeeded = 0;
PASS();
}
SUITE(object_table_tests) {
RUN_REDIS_TEST(new_object_test);
RUN_REDIS_TEST(new_object_no_task_test);
// RUN_REDIS_TEST(lookup_timeout_test);
// RUN_REDIS_TEST(add_timeout_test);
// RUN_REDIS_TEST(subscribe_timeout_test);
RUN_REDIS_TEST(add_lookup_test);
RUN_REDIS_TEST(add_remove_lookup_test);
// RUN_REDIS_TEST(lookup_late_test);
// RUN_REDIS_TEST(add_late_test);
// RUN_REDIS_TEST(subscribe_late_test);
RUN_REDIS_TEST(subscribe_success_test);
RUN_REDIS_TEST(subscribe_object_not_present_test);
RUN_REDIS_TEST(subscribe_object_available_later_test);
RUN_REDIS_TEST(subscribe_object_available_subscribe_all);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
g_task_builder = make_task_builder();
GREATEST_MAIN_BEGIN();
RUN_SUITE(object_table_tests);
GREATEST_MAIN_END();
}
src/common/test/redis_tests.cc
-238
View File
@@ -1,238 +0,0 @@
#include "greatest.h"
#include <assert.h>
#include <unistd.h>
#include <vector>
#include "event_loop.h"
#include "state/db.h"
#include "state/redis.h"
#include "io.h"
#include "logging.h"
#include "test_common.h"
SUITE(redis_tests);
const char *test_set_format = "SET %s %s";
const char *test_get_format = "GET %s";
const char *test_key = "foo";
const char *test_value = "bar";
std::vector<int> connections;
void write_formatted_log_message(int socket_fd, const char *format, ...) {
va_list ap;
/* Get cmd size */
va_start(ap, format);
size_t cmd_size = vsnprintf(nullptr, 0, format, ap) + 1;
va_end(ap);
/* Print va to cmd */
char cmd[cmd_size];
va_start(ap, format);
vsnprintf(cmd, cmd_size, format, ap);
va_end(ap);
write_log_message(socket_fd, cmd);
}
int async_redis_socket_test_callback_called = 0;
void async_redis_socket_test_callback(redisAsyncContext *ac,
void *r,
void *privdata) {
async_redis_socket_test_callback_called = 1;
redisContext *context = redisConnect("127.0.0.1", 6379);
redisReply *reply =
(redisReply *) redisCommand(context, test_get_format, test_key);
redisFree(context);
RAY_CHECK(reply != NULL);
if (strcmp(reply->str, test_value)) {
freeReplyObject(reply);
RAY_CHECK(0);
}
freeReplyObject(reply);
}
TEST redis_socket_test(void) {
const char *socket_pathname = "/tmp/redis-test-socket";
redisContext *context = redisConnect("127.0.0.1", 6379);
ASSERT(context != NULL);
int socket_fd = bind_ipc_sock(socket_pathname, true);
ASSERT(socket_fd >= 0);
int client_fd = connect_ipc_sock(socket_pathname);
ASSERT(client_fd >= 0);
write_formatted_log_message(client_fd, test_set_format, test_key, test_value);
int server_fd = accept_client(socket_fd);
char *cmd = read_log_message(server_fd);
close(client_fd);
close(server_fd);
close(socket_fd);
unlink(socket_pathname);
redisReply *reply = (redisReply *) redisCommand(context, cmd, 0, 0);
freeReplyObject(reply);
reply = (redisReply *) redisCommand(context, "GET %s", test_key);
ASSERT(reply != NULL);
ASSERT_STR_EQ(reply->str, test_value);
freeReplyObject(reply);
free(cmd);
redisFree(context);
PASS();
}
void redis_read_callback(event_loop *loop, int fd, void *context, int events) {
DBHandle *db = (DBHandle *) context;
char *cmd = read_log_message(fd);
redisAsyncCommand(db->context, async_redis_socket_test_callback, NULL, cmd);
free(cmd);
}
void redis_accept_callback(event_loop *loop,
int socket_fd,
void *context,
int events) {
int accept_fd = accept_client(socket_fd);
RAY_CHECK(accept_fd >= 0);
connections.push_back(accept_fd);
event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, redis_read_callback,
context);
}
int timeout_handler(event_loop *loop, timer_id timer_id, void *context) {
event_loop_stop(loop);
return EVENT_LOOP_TIMER_DONE;
}
TEST async_redis_socket_test(void) {
event_loop *loop = event_loop_create();
/* Start IPC channel. */
const char *socket_pathname = "/tmp/async-redis-test-socket";
int socket_fd = bind_ipc_sock(socket_pathname, true);
ASSERT(socket_fd >= 0);
connections.push_back(socket_fd);
/* Start connection to Redis. */
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "test_process",
"127.0.0.1", std::vector<std::string>());
db_attach(db, loop, false);
/* Send a command to the Redis process. */
int client_fd = connect_ipc_sock(socket_pathname);
ASSERT(client_fd >= 0);
connections.push_back(client_fd);
write_formatted_log_message(client_fd, test_set_format, test_key, test_value);
event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, redis_read_callback,
db);
event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, redis_accept_callback,
db);
event_loop_add_timer(loop, 100, timeout_handler, NULL);
event_loop_run(loop);
ASSERT(async_redis_socket_test_callback_called);
db_disconnect(db);
event_loop_destroy(loop);
for (int const &p : connections) {
close(p);
}
unlink(socket_pathname);
connections.clear();
PASS();
}
int logging_test_callback_called = 0;
void logging_test_callback(redisAsyncContext *ac, void *r, void *privdata) {
logging_test_callback_called = 1;
redisContext *context = redisConnect("127.0.0.1", 6379);
redisReply *reply = (redisReply *) redisCommand(context, "KEYS %s", "log:*");
redisFree(context);
RAY_CHECK(reply != NULL);
RAY_CHECK(reply->elements > 0);
freeReplyObject(reply);
}
void logging_read_callback(event_loop *loop,
int fd,
void *context,
int events) {
DBHandle *conn = (DBHandle *) context;
char *cmd = read_log_message(fd);
redisAsyncCommand(conn->context, logging_test_callback, NULL, cmd,
(char *) conn->client.data(), sizeof(conn->client));
free(cmd);
}
void logging_accept_callback(event_loop *loop,
int socket_fd,
void *context,
int events) {
int accept_fd = accept_client(socket_fd);
RAY_CHECK(accept_fd >= 0);
connections.push_back(accept_fd);
event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, logging_read_callback,
context);
}
TEST logging_test(void) {
event_loop *loop = event_loop_create();
/* Start IPC channel. */
const char *socket_pathname = "/tmp/logging-test-socket";
int socket_fd = bind_ipc_sock(socket_pathname, true);
ASSERT(socket_fd >= 0);
connections.push_back(socket_fd);
/* Start connection to Redis. */
DBHandle *conn = db_connect(std::string("127.0.0.1"), 6379, "test_process",
"127.0.0.1", std::vector<std::string>());
db_attach(conn, loop, false);
/* Send a command to the Redis process. */
int client_fd = connect_ipc_sock(socket_pathname);
ASSERT(client_fd >= 0);
connections.push_back(client_fd);
RayLogger *logger = RayLogger_init("worker", RAY_LOG_INFO, 0, &client_fd);
RayLogger_log(logger, RAY_LOG_INFO, "TEST", "Message");
event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, logging_accept_callback,
conn);
event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, logging_read_callback,
conn);
event_loop_add_timer(loop, 100, timeout_handler, NULL);
event_loop_run(loop);
ASSERT(logging_test_callback_called);
RayLogger_free(logger);
db_disconnect(conn);
event_loop_destroy(loop);
for (int const &p : connections) {
close(p);
}
unlink(socket_pathname);
connections.clear();
PASS();
}
SUITE(redis_tests) {
RUN_REDIS_TEST(redis_socket_test);
RUN_REDIS_TEST(async_redis_socket_test);
RUN_REDIS_TEST(logging_test);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(redis_tests);
GREATEST_MAIN_END();
}
src/common/test/run_tests.sh
-43
View File
@@ -1,43 +0,0 @@
#!/usr/bin/env bash
# This needs to be run in the build tree, which is normally ray/build
# Cause the script to exit if a single command fails.
set -ex
LaunchRedis() {
port=$1
if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
./src/credis/redis/src/redis-server \
--loglevel warning \
--loadmodule ./src/credis/build/src/libmember.so \
--loadmodule ./src/common/redis_module/libray_redis_module.so \
--port $port &
else
./src/common/thirdparty/redis/src/redis-server \
--loglevel warning \
--loadmodule ./src/common/redis_module/libray_redis_module.so \
--port $port &
fi
sleep 1s
}
# Start the Redis shards.
LaunchRedis 6379
LaunchRedis 6380
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
if [ -z "$RAY_USE_NEW_GCS" ]; then
./src/common/db_tests
./src/common/io_tests
./src/common/task_tests
./src/common/redis_tests
./src/common/task_table_tests
./src/common/object_table_tests
fi
./src/common/thirdparty/redis/src/redis-cli -p 6379 shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
src/common/test/run_valgrind.sh
-27
View File
@@ -1,27 +0,0 @@
#!/usr/bin/env bash
# This needs to be run in the build tree, which is normally ray/build
set -x
# Cause the script to exit if a single command fails.
set -e
if [ -z "$RAY_USE_NEW_GCS" ]; then
# Start the Redis shards.
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
sleep 1s
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/db_tests
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/io_tests
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/task_tests
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/redis_tests
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/task_table_tests
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/object_table_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
fi
src/common/test/task_table_tests.cc
-460
View File
@@ -1,460 +0,0 @@
#include "greatest.h"
#include "event_loop.h"
#include "example_task.h"
#include "test_common.h"
#include "common.h"
#include "state/object_table.h"
#include "state/redis.h"
#include <unistd.h>
#include <ae.h>
SUITE(task_table_tests);
event_loop *g_loop;
TaskBuilder *g_task_builder = NULL;
/* ==== Test operations in non-failure scenario ==== */
/* === A lookup of a task not in the table === */
TaskID lookup_nil_id;
int lookup_nil_success = 0;
const char *lookup_nil_context = "lookup_nil";
void lookup_nil_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
/* The fail callback should not be called. */
RAY_CHECK(0);
}
void lookup_nil_success_callback(Task *task, void *context) {
lookup_nil_success = 1;
RAY_CHECK(task == NULL);
RAY_CHECK(context == (void *) lookup_nil_context);
event_loop_stop(g_loop);
}
TEST lookup_nil_test(void) {
lookup_nil_id = TaskID::from_random();
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 1000,
.fail_callback = lookup_nil_fail_callback,
};
task_table_get_task(db, lookup_nil_id, &retry, lookup_nil_success_callback,
(void *) lookup_nil_context);
/* Disconnect the database to see if the lookup times out. */
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(lookup_nil_success);
PASS();
}
/* === A lookup of a task after it's added returns the same spec === */
int add_success = 0;
int lookup_success = 0;
Task *add_lookup_task;
const char *add_lookup_context = "add_lookup";
void add_lookup_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
/* The fail callback should not be called. */
RAY_CHECK(0);
}
void lookup_success_callback(Task *task, void *context) {
lookup_success = 1;
RAY_CHECK(Task_equals(task, add_lookup_task));
event_loop_stop(g_loop);
}
void add_success_callback(TaskID task_id, void *context) {
add_success = 1;
RAY_CHECK(TaskID_equal(task_id, Task_task_id(add_lookup_task)));
DBHandle *db = (DBHandle *) context;
RetryInfo retry = {
.num_retries = 5,
.timeout = 1000,
.fail_callback = add_lookup_fail_callback,
};
task_table_get_task(db, task_id, &retry, lookup_success_callback,
(void *) add_lookup_context);
}
void subscribe_success_callback(TaskID task_id, void *context) {
DBHandle *db = (DBHandle *) context;
RetryInfo retry = {
.num_retries = 5,
.timeout = 1000,
.fail_callback = add_lookup_fail_callback,
};
task_table_add_task(db, Task_copy(add_lookup_task), &retry,
add_success_callback, (void *) db);
}
TEST add_lookup_test(void) {
add_lookup_task = example_task(1, 1, TaskStatus::WAITING);
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 1000,
.fail_callback = add_lookup_fail_callback,
};
/* Wait for subscription to succeed before adding the task. */
task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL,
&retry, subscribe_success_callback, (void *) db);
/* Disconnect the database to see if the lookup times out. */
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(add_success);
ASSERT(lookup_success);
PASS();
}
/* ==== Test if operations time out correctly ==== */
/* === Test subscribe timeout === */
const char *subscribe_timeout_context = "subscribe_timeout";
int subscribe_failed = 0;
void subscribe_done_callback(TaskID task_id, void *user_context) {
/* The done callback should not be called. */
RAY_CHECK(0);
}
void subscribe_fail_callback(UniqueID id, void *user_context, void *user_data) {
subscribe_failed = 1;
RAY_CHECK(user_context == (void *) subscribe_timeout_context);
event_loop_stop(g_loop);
}
TEST subscribe_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = subscribe_fail_callback,
};
task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL,
&retry, subscribe_done_callback,
(void *) subscribe_timeout_context);
/* Disconnect the database to see if the subscribe times out. */
close(db->subscribe_context->c.fd);
for (size_t i = 0; i < db->subscribe_contexts.size(); ++i) {
close(db->subscribe_contexts[i]->c.fd);
}
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(subscribe_failed);
PASS();
}
/* === Test publish timeout === */
const char *publish_timeout_context = "publish_timeout";
int publish_failed = 0;
void publish_done_callback(TaskID task_id, void *user_context) {
/* The done callback should not be called. */
RAY_CHECK(0);
}
void publish_fail_callback(UniqueID id, void *user_context, void *user_data) {
publish_failed = 1;
RAY_CHECK(user_context == (void *) publish_timeout_context);
event_loop_stop(g_loop);
}
TEST publish_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
Task *task = example_task(1, 1, TaskStatus::WAITING);
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = publish_fail_callback,
};
task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL,
&retry, NULL, NULL);
task_table_add_task(db, task, &retry, publish_done_callback,
(void *) publish_timeout_context);
/* Disconnect the database to see if the publish times out. */
close(db->context->c.fd);
for (size_t i = 0; i < db->contexts.size(); ++i) {
close(db->contexts[i]->c.fd);
}
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(publish_failed);
PASS();
}
/* ==== Test if the retry is working correctly ==== */
int64_t reconnect_db_callback(event_loop *loop,
int64_t timer_id,
void *context) {
DBHandle *db = (DBHandle *) context;
/* Reconnect to redis. */
redisAsyncFree(db->subscribe_context);
db->subscribe_context = redisAsyncConnect("127.0.0.1", 6379);
db->subscribe_context->data = (void *) db;
for (size_t i = 0; i < db->subscribe_contexts.size(); ++i) {
redisAsyncFree(db->subscribe_contexts[i]);
db->subscribe_contexts[i] = redisAsyncConnect("127.0.0.1", 6380 + i);
db->subscribe_contexts[i]->data = (void *) db;
}
/* Re-attach the database to the event loop (the file descriptor changed). */
db_attach(db, loop, true);
return EVENT_LOOP_TIMER_DONE;
}
int64_t terminate_event_loop_callback(event_loop *loop,
int64_t timer_id,
void *context) {
event_loop_stop(loop);
return EVENT_LOOP_TIMER_DONE;
}
/* === Test subscribe retry === */
const char *subscribe_retry_context = "subscribe_retry";
int subscribe_retry_succeeded = 0;
void subscribe_retry_done_callback(ObjectID object_id, void *user_context) {
RAY_CHECK(user_context == (void *) subscribe_retry_context);
subscribe_retry_succeeded = 1;
}
void subscribe_retry_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
/* The fail callback should not be called. */
RAY_CHECK(0);
}
TEST subscribe_retry_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = subscribe_retry_fail_callback,
};
task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL,
&retry, subscribe_retry_done_callback,
(void *) subscribe_retry_context);
/* Disconnect the database to see if the subscribe times out. */
close(db->subscribe_context->c.fd);
for (size_t i = 0; i < db->subscribe_contexts.size(); ++i) {
close(db->subscribe_contexts[i]->c.fd);
}
/* Install handler for reconnecting the database. */
event_loop_add_timer(g_loop, 150,
(event_loop_timer_handler) reconnect_db_callback, db);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(subscribe_retry_succeeded);
PASS();
}
/* === Test publish retry === */
const char *publish_retry_context = "publish_retry";
int publish_retry_succeeded = 0;
void publish_retry_done_callback(ObjectID object_id, void *user_context) {
RAY_CHECK(user_context == (void *) publish_retry_context);
publish_retry_succeeded = 1;
}
void publish_retry_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
/* The fail callback should not be called. */
RAY_CHECK(0);
}
TEST publish_retry_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
Task *task = example_task(1, 1, TaskStatus::WAITING);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = publish_retry_fail_callback,
};
task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL,
&retry, NULL, NULL);
task_table_add_task(db, task, &retry, publish_retry_done_callback,
(void *) publish_retry_context);
/* Disconnect the database to see if the publish times out. */
close(db->subscribe_context->c.fd);
for (size_t i = 0; i < db->subscribe_contexts.size(); ++i) {
close(db->subscribe_contexts[i]->c.fd);
}
/* Install handler for reconnecting the database. */
event_loop_add_timer(g_loop, 150,
(event_loop_timer_handler) reconnect_db_callback, db);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(publish_retry_succeeded);
PASS();
}
/* ==== Test if late succeed is working correctly ==== */
/* === Test subscribe late succeed === */
const char *subscribe_late_context = "subscribe_late";
int subscribe_late_failed = 0;
void subscribe_late_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
RAY_CHECK(user_context == (void *) subscribe_late_context);
subscribe_late_failed = 1;
}
void subscribe_late_done_callback(TaskID task_id, void *user_context) {
/* This function should never be called. */
RAY_CHECK(0);
}
TEST subscribe_late_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0,
.timeout = 0,
.fail_callback = subscribe_late_fail_callback,
};
task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL,
&retry, subscribe_late_done_callback,
(void *) subscribe_late_context);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* First process timer events to make sure the timeout is processed before
* anything else. */
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(subscribe_late_failed);
PASS();
}
/* === Test publish late succeed === */
const char *publish_late_context = "publish_late";
int publish_late_failed = 0;
void publish_late_fail_callback(UniqueID id,
void *user_context,
void *user_data) {
RAY_CHECK(user_context == (void *) publish_late_context);
publish_late_failed = 1;
}
void publish_late_done_callback(TaskID task_id, void *user_context) {
/* This function should never be called. */
RAY_CHECK(0);
}
TEST publish_late_test(void) {
g_loop = event_loop_create();
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", std::vector<std::string>());
db_attach(db, g_loop, false);
Task *task = example_task(1, 1, TaskStatus::WAITING);
RetryInfo retry = {
.num_retries = 0,
.timeout = 0,
.fail_callback = publish_late_fail_callback,
};
task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL,
NULL, NULL, NULL);
task_table_add_task(db, task, &retry, publish_late_done_callback,
(void *) publish_late_context);
/* Install handler for terminating the event loop. */
event_loop_add_timer(g_loop, 750,
(event_loop_timer_handler) terminate_event_loop_callback,
NULL);
/* First process timer events to make sure the timeout is processed before
* anything else. */
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
event_loop_destroy(g_loop);
ASSERT(publish_late_failed);
PASS();
}
SUITE(task_table_tests) {
RUN_REDIS_TEST(lookup_nil_test);
RUN_REDIS_TEST(add_lookup_test);
// RUN_REDIS_TEST(subscribe_timeout_test);
// RUN_REDIS_TEST(publish_timeout_test);
// RUN_REDIS_TEST(subscribe_retry_test);
// RUN_REDIS_TEST(publish_retry_test);
// RUN_REDIS_TEST(subscribe_late_test);
// RUN_REDIS_TEST(publish_late_test);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
g_task_builder = make_task_builder();
GREATEST_MAIN_BEGIN();
RUN_SUITE(task_table_tests);
GREATEST_MAIN_END();
}
src/common/test/task_tests.cc
-212
View File
@@ -1,212 +0,0 @@
#include "greatest.h"
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include "common.h"
#include "test_common.h"
#include "task.h"
#include "io.h"
SUITE(task_tests);
TEST task_test(void) {
TaskID parent_task_id = TaskID::from_random();
FunctionID func_id = FunctionID::from_random();
TaskBuilder *builder = make_task_builder();
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 2);
UniqueID arg1 = UniqueID::from_random();
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, (uint8_t *) "hello", 5);
UniqueID arg2 = UniqueID::from_random();
TaskSpec_args_add_ref(builder, &arg2, 1);
TaskSpec_args_add_val(builder, (uint8_t *) "world", 5);
/* Finish constructing the spec. This constructs the task ID and the
* return IDs. */
int64_t size;
TaskSpec *spec = TaskSpec_finish_construct(builder, &size);
/* Check that the spec was constructed as expected. */
ASSERT(TaskSpec_num_args(spec) == 4);
ASSERT(TaskSpec_num_returns(spec) == 2);
ASSERT(FunctionID_equal(TaskSpec_function(spec), func_id));
ASSERT(TaskSpec_arg_id(spec, 0, 0) == arg1);
ASSERT(memcmp(TaskSpec_arg_val(spec, 1), (uint8_t *) "hello",
TaskSpec_arg_length(spec, 1)) == 0);
ASSERT(TaskSpec_arg_id(spec, 2, 0) == arg2);
ASSERT(memcmp(TaskSpec_arg_val(spec, 3), (uint8_t *) "world",
TaskSpec_arg_length(spec, 3)) == 0);
TaskSpec_free(spec);
free_task_builder(builder);
PASS();
}
TEST deterministic_ids_test(void) {
TaskBuilder *builder = make_task_builder();
/* Define the inputs to the task construction. */
TaskID parent_task_id = TaskID::from_random();
FunctionID func_id = FunctionID::from_random();
UniqueID arg1 = UniqueID::from_random();
uint8_t *arg2 = (uint8_t *) "hello world";
/* Construct a first task. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size1;
TaskSpec *spec1 = TaskSpec_finish_construct(builder, &size1);
/* Construct a second identical task. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size2;
TaskSpec *spec2 = TaskSpec_finish_construct(builder, &size2);
/* Check that these tasks have the same task IDs and the same return IDs. */
ASSERT(TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec2)));
ASSERT(TaskSpec_return(spec1, 0) == TaskSpec_return(spec2, 0));
ASSERT(TaskSpec_return(spec1, 1) == TaskSpec_return(spec2, 1));
ASSERT(TaskSpec_return(spec1, 2) == TaskSpec_return(spec2, 2));
/* Check that the return IDs are all distinct. */
ASSERT(!(TaskSpec_return(spec1, 0) == TaskSpec_return(spec2, 1)));
ASSERT(!(TaskSpec_return(spec1, 0) == TaskSpec_return(spec2, 2)));
ASSERT(!(TaskSpec_return(spec1, 1) == TaskSpec_return(spec2, 2)));
/* Create more tasks that are only mildly different. */
/* Construct a task with a different parent task ID. */
TaskSpec_start_construct(builder, DriverID::nil(), TaskID::from_random(), 0,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size3;
TaskSpec *spec3 = TaskSpec_finish_construct(builder, &size3);
/* Construct a task with a different parent counter. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 1,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size4;
TaskSpec *spec4 = TaskSpec_finish_construct(builder, &size4);
/* Construct a task with a different function ID. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, FunctionID::from_random(),
3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size5;
TaskSpec *spec5 = TaskSpec_finish_construct(builder, &size5);
/* Construct a task with a different object ID argument. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
ObjectID object_id = ObjectID::from_random();
TaskSpec_args_add_ref(builder, &object_id, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size6;
TaskSpec *spec6 = TaskSpec_finish_construct(builder, &size6);
/* Construct a task with a different value argument. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, (uint8_t *) "hello_world", 11);
int64_t size7;
TaskSpec *spec7 = TaskSpec_finish_construct(builder, &size7);
/* Check that the task IDs are all distinct from the original. */
ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec3)));
ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec4)));
ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec5)));
ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec6)));
ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec7)));
/* Check that the return object IDs are distinct from the originals. */
TaskSpec *specs[6] = {spec1, spec3, spec4, spec5, spec6, spec7};
for (int task_index1 = 0; task_index1 < 6; ++task_index1) {
for (int return_index1 = 0; return_index1 < 3; ++return_index1) {
for (int task_index2 = 0; task_index2 < 6; ++task_index2) {
for (int return_index2 = 0; return_index2 < 3; ++return_index2) {
if (task_index1 != task_index2 && return_index1 != return_index2) {
ASSERT(!(TaskSpec_return(specs[task_index1], return_index1) ==
TaskSpec_return(specs[task_index2], return_index2)));
}
}
}
}
}
TaskSpec_free(spec1);
TaskSpec_free(spec2);
TaskSpec_free(spec3);
TaskSpec_free(spec4);
TaskSpec_free(spec5);
TaskSpec_free(spec6);
TaskSpec_free(spec7);
free_task_builder(builder);
PASS();
}
TEST send_task(void) {
TaskBuilder *builder = make_task_builder();
TaskID parent_task_id = TaskID::from_random();
FunctionID func_id = FunctionID::from_random();
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 2);
ObjectID object_id = ObjectID::from_random();
TaskSpec_args_add_ref(builder, &object_id, 1);
TaskSpec_args_add_val(builder, (uint8_t *) "Hello", 5);
TaskSpec_args_add_val(builder, (uint8_t *) "World", 5);
object_id = ObjectID::from_random();
TaskSpec_args_add_ref(builder, &object_id, 1);
int64_t size;
TaskSpec *spec = TaskSpec_finish_construct(builder, &size);
int fd[2];
socketpair(AF_UNIX, SOCK_STREAM, 0, fd);
write_message(fd[0], static_cast<int64_t>(CommonMessageType::SUBMIT_TASK),
size, (uint8_t *) spec);
int64_t type;
int64_t length;
uint8_t *message;
read_message(fd[1], &type, &length, &message);
TaskSpec *result = (TaskSpec *) message;
ASSERT(static_cast<CommonMessageType>(type) ==
CommonMessageType::SUBMIT_TASK);
ASSERT(memcmp(spec, result, size) == 0);
TaskSpec_free(spec);
free(result);
free_task_builder(builder);
PASS();
}
SUITE(task_tests) {
RUN_TEST(task_test);
RUN_TEST(deterministic_ids_test);
RUN_TEST(send_task);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(task_tests);
GREATEST_MAIN_END();
}
src/common/test/test_common.h
-91
View File
@@ -1,91 +0,0 @@
#ifndef TEST_COMMON_H
#define TEST_COMMON_H
#include <unistd.h>
#include <cstdio>
#include <string>
#include <vector>
#include "common.h"
#include "io.h"
#include "hiredis/hiredis.h"
#include "state/redis.h"
#ifndef _WIN32
/* This function is actually not declared in standard POSIX, so declare it. */
extern int usleep(useconds_t usec);
#endif
/* I/O helper methods to retry binding to sockets. */
static inline std::string bind_ipc_sock_retry(const char *socket_name_format,
int *fd) {
std::string socket_name;
for (int num_retries = 0; num_retries < 5; ++num_retries) {
RAY_LOG(INFO) << "trying to find plasma socket (attempt " << num_retries
<< ")";
size_t size = std::snprintf(nullptr, 0, socket_name_format, rand()) + 1;
char socket_name_c_str[size];
std::snprintf(socket_name_c_str, size, socket_name_format, rand());
socket_name = std::string(socket_name_c_str);
*fd = bind_ipc_sock(socket_name.c_str(), true);
if (*fd < 0) {
/* Sleep for 100ms. */
usleep(100000);
continue;
}
break;
}
return socket_name;
}
static inline int bind_inet_sock_retry(int *fd) {
int port = -1;
for (int num_retries = 0; num_retries < 5; ++num_retries) {
port = 10000 + rand() % 40000;
*fd = bind_inet_sock(port, true);
if (*fd < 0) {
/* Sleep for 100ms. */
usleep(100000);
continue;
}
break;
}
return port;
}
/* Flush redis. */
static inline void flushall_redis(void) {
/* Flush the primary shard. */
redisContext *context = redisConnect("127.0.0.1", 6379);
std::vector<std::string> db_shards_addresses;
std::vector<int> db_shards_ports;
get_redis_shards(context, db_shards_addresses, db_shards_ports);
freeReplyObject(redisCommand(context, "FLUSHALL"));
/* Readd the shard locations. */
freeReplyObject(redisCommand(context, "SET NumRedisShards %d",
db_shards_addresses.size()));
for (size_t i = 0; i < db_shards_addresses.size(); ++i) {
freeReplyObject(redisCommand(context, "RPUSH RedisShards %s:%d",
db_shards_addresses[i].c_str(),
db_shards_ports[i]));
}
redisFree(context);
/* Flush the remaining shards. */
for (size_t i = 0; i < db_shards_addresses.size(); ++i) {
context = redisConnect(db_shards_addresses[i].c_str(), db_shards_ports[i]);
freeReplyObject(redisCommand(context, "FLUSHALL"));
redisFree(context);
}
}
/* Cleanup method for running tests with the greatest library.
* Runs the test, then clears the Redis database. */
#define RUN_REDIS_TEST(test) \
flushall_redis(); \
RUN_TEST(test); \
flushall_redis();
#endif /* TEST_COMMON */
src/common/thirdparty/download_thirdparty.bat
Vendored
-15
View File
@@ -1,15 +0,0 @@
@SetLocal
@Echo Off
@PushD "%~dp0"
git submodule update --init --jobs="%NUMBER_OF_PROCESSORS%"
@If Not Exist "python\.git" git clone "https://github.com/austinsc/python.git"
Call :GitApply "python" "%CD%/patches/windows/python-pyconfig.patch"
Call :GitApply "redis-windows" "%CD%/patches/windows/redis.patch"
@PopD
@EndLocal
@GoTo :EOF
:GitApply <ChangeToFolder> <Patch>
@REM Check if patch already applied by attempting to apply it in reverse; if not, then force-reapply it
git -C "%~1" apply "%~2" -R --check 2> NUL || git -C "%~1" apply "%~2" --3way 2> NUL || git -C "%~1" reset --hard && git -C "%~1" apply "%~2" --3way
@GoTo :EOF
src/common/thirdparty/greatest.h
Vendored
-1023
View File
File diff suppressed because it is too large Load Diff
src/common/thirdparty/patches/.gitattributes
-1
View File
@@ -1 +0,0 @@
*.patch text eol=lf
src/common/thirdparty/patches/windows/python-pyconfig.patch
Vendored
-25
View File
@@ -1,25 +0,0 @@
diff --git a/inc/Windows/pyconfig.h b/inc/Windows/pyconfig.h
index 1cfc59b..d4861cb
--- a/inc/Windows/pyconfig.h
+++ b/inc/Windows/pyconfig.h
@@ -1,6 +1,11 @@
#ifndef Py_CONFIG_H
#define Py_CONFIG_H
+#ifdef _MSC_VER
+#pragma push_macro("_DEBUG")
+#undef _DEBUG
+#endif
+
/* pyconfig.h. NOT Generated automatically by configure.
This is a manually maintained version used for the Watcom,
@@ -756,4 +761,8 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */
least significant byte first */
#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1
+#ifdef _MSC_VER
+#pragma pop_macro("_DEBUG")
+#endif
+
#endif /* !Py_CONFIG_H */
src/common/thirdparty/patches/windows/redis.patch
Vendored
-772
View File
@@ -1,772 +0,0 @@
diff --git a/msvs/RedisServer.vcxproj b/msvs/RedisServer.vcxproj
index 115ce90..68afb44
--- a/msvs/RedisServer.vcxproj
+++ b/msvs/RedisServer.vcxproj
@@ -24,26 +24,26 @@
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
+ <ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
<CLRSupport>false</CLRSupport>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
+ <ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
<CLRSupport>false</CLRSupport>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
+ <ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
+ <ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@@ -61,41 +61,23 @@
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <PropertyGroup>
<LinkIncremental>false</LinkIncremental>
<TargetName>redis-server</TargetName>
<GenerateManifest>false</GenerateManifest>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <LinkIncremental>false</LinkIncremental>
- <TargetName>redis-server</TargetName>
- <GenerateManifest>false</GenerateManifest>
- <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
- <IntDir>$(Platform)\$(Configuration)\</IntDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
- <LinkIncremental>false</LinkIncremental>
- <TargetName>redis-server</TargetName>
- <GenerateManifest>false</GenerateManifest>
- <CustomBuildAfterTargets>Build</CustomBuildAfterTargets>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <LinkIncremental>false</LinkIncremental>
- <TargetName>redis-server</TargetName>
- <GenerateManifest>false</GenerateManifest>
- <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
- <IntDir>$(Platform)\$(Configuration)\</IntDir>
+ <OutDir>$(SolutionDir)build\$(Platform)\$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)build\$(Platform)\$(Configuration)\$(MSBuildProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
- <PreprocessorDefinitions>USE_JEMALLOC;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;_DEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <AdditionalIncludeDirectories>$(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;_DEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <AdditionalIncludeDirectories>$(ProjectDir)..\deps\lua\src;$(ProjectDir)..\deps\hiredis;$(ProjectDir)..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
<WarningLevel>Level3</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<Optimization>Disabled</Optimization>
<DisableSpecificWarnings>4996;4146</DisableSpecificWarnings>
- <WholeProgramOptimization>true</WholeProgramOptimization>
+ <MinimalRebuild>false</MinimalRebuild>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -109,14 +91,14 @@
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
- <PreprocessorDefinitions>USE_JEMALLOC;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;_DEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501</PreprocessorDefinitions>
- <AdditionalIncludeDirectories>$(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;_DEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501</PreprocessorDefinitions>
+ <AdditionalIncludeDirectories>$(ProjectDir)..\deps\lua\src;$(ProjectDir)..\deps\hiredis;$(ProjectDir)..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
<WarningLevel>Level3</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<Optimization>Disabled</Optimization>
<DisableSpecificWarnings>4996;4146</DisableSpecificWarnings>
- <WholeProgramOptimization>true</WholeProgramOptimization>
+ <MinimalRebuild>false</MinimalRebuild>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -130,14 +112,13 @@
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
- <PreprocessorDefinitions>USE_JEMALLOC;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;NDEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <AdditionalIncludeDirectories>$(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;NDEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <AdditionalIncludeDirectories>$(ProjectDir)..\deps\lua\src;$(ProjectDir)..\deps\hiredis;$(ProjectDir)..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
<WarningLevel>Level3</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<DisableSpecificWarnings>4996;4146</DisableSpecificWarnings>
- <WholeProgramOptimization>true</WholeProgramOptimization>
<Optimization>Full</Optimization>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -162,13 +143,12 @@
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
- <PreprocessorDefinitions>USE_JEMALLOC;_OFF_T_DEFINED;_WIN32;LACKS_STDLIB_H;NDEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501</PreprocessorDefinitions>
- <AdditionalIncludeDirectories>$(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;_OFF_T_DEFINED;_WIN32;LACKS_STDLIB_H;NDEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501</PreprocessorDefinitions>
+ <AdditionalIncludeDirectories>$(ProjectDir)..\deps\lua\src;$(ProjectDir)..\deps\hiredis;$(ProjectDir)..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
<WarningLevel>Level3</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<DisableSpecificWarnings>4996;4146</DisableSpecificWarnings>
- <WholeProgramOptimization>true</WholeProgramOptimization>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -271,9 +251,6 @@
<ClInclude Include="..\src\zmalloc.h" />
</ItemGroup>
<ItemGroup>
- <ProjectReference Include="..\deps\jemalloc-win\projects\jemalloc\proj.win32\vc2013\jemalloc.vcxproj">
- <Project>{8b897e33-6428-4254-8335-4911d179bad1}</Project>
- </ProjectReference>
<ProjectReference Include="..\src\Win32_Interop\Win32_Interop.vcxproj">
<Project>{8c07f811-c81c-432c-b334-1ae6faecf951}</Project>
</ProjectReference>
diff --git a/msvs/hiredis/hiredis.vcxproj b/msvs/hiredis/hiredis.vcxproj
index 0622958..efaedae
--- a/msvs/hiredis/hiredis.vcxproj
+++ b/msvs/hiredis/hiredis.vcxproj
@@ -28,27 +28,25 @@
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@@ -66,30 +64,20 @@
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <PropertyGroup>
<TargetName>hiredis</TargetName>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <TargetName>hiredis</TargetName>
- <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
- <IntDir>$(Platform)\$(Configuration)\</IntDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
- <TargetName>hiredis</TargetName>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <TargetName>hiredis</TargetName>
- <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
- <IntDir>$(Platform)\$(Configuration)\</IntDir>
+ <OutDir>$(ProjectDir)..\$(Platform)\$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)build\$(Platform)\$(Configuration)\$(MSBuildProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_LIB;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;_LIB;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
+ <MinimalRebuild>false</MinimalRebuild>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
@@ -101,9 +89,10 @@
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_LIB;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;_LIB;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
+ <MinimalRebuild>false</MinimalRebuild>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
@@ -117,10 +106,9 @@
<Optimization>Full</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
- <WholeProgramOptimization>true</WholeProgramOptimization>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
@@ -136,10 +124,9 @@
<Optimization>Full</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
- <WholeProgramOptimization>true</WholeProgramOptimization>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
diff --git a/msvs/lua/lua/lua.vcxproj b/msvs/lua/lua/lua.vcxproj
index b187130..adef07b
--- a/msvs/lua/lua/lua.vcxproj
+++ b/msvs/lua/lua/lua.vcxproj
@@ -30,28 +30,28 @@
<UseDebugLibraries>true</UseDebugLibraries>
<CLRSupport>false</CLRSupport>
<CharacterSet>MultiByte</CharacterSet>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CLRSupport>false</CLRSupport>
<CharacterSet>MultiByte</CharacterSet>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<CLRSupport>false</CLRSupport>
<CharacterSet>MultiByte</CharacterSet>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<CLRSupport>false</CLRSupport>
<CharacterSet>MultiByte</CharacterSet>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@@ -69,25 +69,16 @@
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <PropertyGroup Condition="'$(Configuration)'=='Debug'">
<LinkIncremental>true</LinkIncremental>
- <TargetExt>.lib</TargetExt>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <LinkIncremental>true</LinkIncremental>
- <TargetExt>.lib</TargetExt>
- <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
- <IntDir>$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <PropertyGroup Condition="'$(Configuration)'=='Release'">
<LinkIncremental>false</LinkIncremental>
- <TargetExt>.lib</TargetExt>
</PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <LinkIncremental>false</LinkIncremental>
+ <PropertyGroup>
<TargetExt>.lib</TargetExt>
- <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
- <IntDir>$(Platform)\$(Configuration)\</IntDir>
+ <OutDir>$(SolutionDir)build\$(Platform)\$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)build\$(Platform)\$(Configuration)\$(MSBuildProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
@@ -95,8 +86,9 @@
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_DEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);LUA_ANSI;ENABLE_CJSON_GLOBAL</PreprocessorDefinitions>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
<DisableSpecificWarnings>4244;4018</DisableSpecificWarnings>
+ <MinimalRebuild>false</MinimalRebuild>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -110,8 +102,9 @@
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_DEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501;LUA_ANSI;ENABLE_CJSON_GLOBAL</PreprocessorDefinitions>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
<DisableSpecificWarnings>4244;4018</DisableSpecificWarnings>
+ <MinimalRebuild>false</MinimalRebuild>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -124,10 +117,10 @@
<WarningLevel>Level3</WarningLevel>
<PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;NDEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);LUA_ANSI;ENABLE_CJSON_GLOBAL</PreprocessorDefinitions>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DisableSpecificWarnings>4244;4018</DisableSpecificWarnings>
<Optimization>Full</Optimization>
<WholeProgramOptimization>true</WholeProgramOptimization>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -140,8 +133,8 @@
<WarningLevel>Level3</WarningLevel>
<PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;NDEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501;LUA_ANSI;ENABLE_CJSON_GLOBAL</PreprocessorDefinitions>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DisableSpecificWarnings>4244;4018</DisableSpecificWarnings>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
diff --git a/src/Win32_Interop/Win32_ANSI.c b/src/Win32_Interop/Win32_ANSI.c
index 404b84f..e7c55d2
--- a/src/Win32_Interop/Win32_ANSI.c
+++ b/src/Win32_Interop/Win32_ANSI.c
@@ -737,7 +737,7 @@ void ANSI_printf(char *format, ...) {
memset(buffer, 0, cBufLen);
va_start(args, format);
- retVal = vsprintf_s(buffer, cBufLen, format, args);
+ retVal = vsnprintf(buffer, cBufLen - 1, format, args);
va_end(args);
if (retVal > 0) {
diff --git a/src/Win32_Interop/Win32_EventLog.cpp b/src/Win32_Interop/Win32_EventLog.cpp
index 1856540..3db4ddd
--- a/src/Win32_Interop/Win32_EventLog.cpp
+++ b/src/Win32_Interop/Win32_EventLog.cpp
@@ -30,7 +30,6 @@ using namespace std;
#include "Win32_EventLog.h"
#include "Win32_SmartHandle.h"
-#include "EventLog.h"
static bool eventLogEnabled = true;
static string eventLogIdentity = "redis";
@@ -129,17 +128,17 @@ void RedisEventLog::LogMessage(LPCSTR msg, const WORD type) {
DWORD eventID;
switch (type) {
case EVENTLOG_ERROR_TYPE:
- eventID = MSG_ERROR_1;
+ eventID = 0x2;
break;
case EVENTLOG_WARNING_TYPE:
- eventID = MSG_WARNING_1;
+ eventID = 0x1;
break;
case EVENTLOG_INFORMATION_TYPE:
- eventID = MSG_INFO_1;
+ eventID = 0x0;
break;
default:
std::cerr << "Unrecognized type: " << type << "\n";
- eventID = MSG_INFO_1;
+ eventID = 0x0;
break;
}
diff --git a/src/Win32_Interop/Win32_FDAPI.cpp b/src/Win32_Interop/Win32_FDAPI.cpp
index 3df9af1..f60e3d4
--- a/src/Win32_Interop/Win32_FDAPI.cpp
+++ b/src/Win32_Interop/Win32_FDAPI.cpp
@@ -46,11 +46,13 @@ fdapi_access access = NULL;
fdapi_bind bind = NULL;
fdapi_connect connect = NULL;
fdapi_fcntl fcntl = NULL;
+fdapi_ioctl ioctl = NULL;
fdapi_fstat fdapi_fstat64 = NULL;
fdapi_fsync fsync = NULL;
fdapi_ftruncate ftruncate = NULL;
fdapi_freeaddrinfo freeaddrinfo = NULL;
fdapi_getaddrinfo getaddrinfo = NULL;
+fdapi_gethostbyname gethostbyname = NULL;
fdapi_getpeername getpeername = NULL;
fdapi_getsockname getsockname = NULL;
fdapi_getsockopt getsockopt = NULL;
@@ -67,7 +69,9 @@ fdapi_open open = NULL;
fdapi_pipe pipe = NULL;
fdapi_poll poll = NULL;
fdapi_read read = NULL;
+fdapi_recv recv = NULL;
fdapi_select select = NULL;
+fdapi_send send = NULL;
fdapi_setsockopt setsockopt = NULL;
fdapi_socket socket = NULL;
fdapi_write write = NULL;
@@ -622,6 +626,23 @@ int FDAPI_fcntl(int rfd, int cmd, int flags = 0 ) {
return -1;
}
+int FDAPI_ioctl(int rfd, int cmd, char *buf) {
+ try {
+ SocketInfo* socket_info = RFDMap::getInstance().lookupSocketInfo(rfd);
+ if (socket_info != NULL && socket_info->socket != INVALID_SOCKET) {
+ if (f_ioctlsocket(socket_info->socket, cmd, (u_long *)buf) != SOCKET_ERROR) {
+ return 0;
+ } else {
+ errno = f_WSAGetLastError();
+ return -1;
+ }
+ }
+ } CATCH_AND_REPORT();
+
+ errno = EBADF;
+ return -1;
+}
+
int FDAPI_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
try {
struct pollfd* pollCopy = new struct pollfd[nfds];
@@ -777,6 +798,42 @@ ssize_t FDAPI_read(int rfd, void *buf, size_t count) {
return -1;
}
+ssize_t FDAPI_recv(int rfd, void *buf, size_t count, int flags) {
+ try {
+ SOCKET socket = RFDMap::getInstance().lookupSocket(rfd);
+ if (socket != INVALID_SOCKET) {
+ int retval = f_recv(socket, (char*) buf, (unsigned int) count, flags);
+ if (retval == -1) {
+ errno = GetLastError();
+ if (errno == WSAEWOULDBLOCK) {
+ errno = EAGAIN;
+ }
+ }
+ return retval;
+ }
+ } CATCH_AND_REPORT();
+ errno = EBADF;
+ return -1;
+}
+
+ssize_t FDAPI_send(int rfd, const void *buf, size_t count, int flags) {
+ try {
+ SOCKET socket = RFDMap::getInstance().lookupSocket(rfd);
+ if (socket != INVALID_SOCKET) {
+ int retval = f_send(socket, (const char*) buf, (unsigned int) count, flags);
+ if (retval == -1) {
+ errno = GetLastError();
+ if (errno == WSAEWOULDBLOCK) {
+ errno = EAGAIN;
+ }
+ }
+ return retval;
+ }
+ } CATCH_AND_REPORT();
+ errno = EBADF;
+ return -1;
+}
+
ssize_t FDAPI_write(int rfd, const void *buf, size_t count) {
try {
SOCKET socket = RFDMap::getInstance().lookupSocket(rfd);
@@ -1195,12 +1252,14 @@ private:
bind = FDAPI_bind;
connect = FDAPI_connect;
fcntl = FDAPI_fcntl;
+ ioctl = FDAPI_ioctl;
fdapi_fstat64 = (fdapi_fstat) FDAPI_fstat64;
freeaddrinfo = FDAPI_freeaddrinfo;
fsync = FDAPI_fsync;
ftruncate = FDAPI_ftruncate;
getaddrinfo = FDAPI_getaddrinfo;
getsockopt = FDAPI_getsockopt;
+ gethostbyname = FDAPI_gethostbyname;
getpeername = FDAPI_getpeername;
getsockname = FDAPI_getsockname;
htonl = FDAPI_htonl;
@@ -1216,9 +1275,11 @@ private:
pipe = FDAPI_pipe;
poll = FDAPI_poll;
read = FDAPI_read;
+ recv = FDAPI_recv;
select = FDAPI_select;
setsockopt = FDAPI_setsockopt;
socket = FDAPI_socket;
+ send = FDAPI_send;
write = FDAPI_write;
}
diff --git a/src/Win32_Interop/Win32_FDAPI.h b/src/Win32_Interop/Win32_FDAPI.h
index 8fae9c7..6e09596
--- a/src/Win32_Interop/Win32_FDAPI.h
+++ b/src/Win32_Interop/Win32_FDAPI.h
@@ -116,9 +116,12 @@ typedef int (*fdapi_open)(const char * _Filename, int _OpenFlag, int flags);
typedef int (*fdapi_accept)(int sockfd, struct sockaddr *addr, socklen_t *addrlen);
typedef int (*fdapi_setsockopt)(int sockfd, int level, int optname,const void *optval, socklen_t optlen);
typedef int (*fdapi_fcntl)(int fd, int cmd, int flags);
+typedef int (*fdapi_ioctl)(int fd, int cmd, char *buf);
typedef int (*fdapi_poll)(struct pollfd *fds, nfds_t nfds, int timeout);
typedef int (*fdapi_getsockopt)(int sockfd, int level, int optname, void *optval, socklen_t *optlen);
typedef int (*fdapi_connect)(int sockfd, const struct sockaddr *addr, size_t addrlen);
+typedef ssize_t (*fdapi_recv)(int fd, void *buf, size_t count, int flags);
+typedef ssize_t (*fdapi_send)(int rfd, void const *buf, size_t count, int flags);
typedef ssize_t (*fdapi_read)(int fd, void *buf, size_t count);
typedef ssize_t (*fdapi_write)(int fd, const void *buf, size_t count);
typedef int (*fdapi_fsync)(int fd);
@@ -128,6 +131,7 @@ typedef int (*fdapi_bind)(int sockfd, const struct sockaddr *addr, socklen_t add
typedef u_short (*fdapi_htons)(u_short hostshort);
typedef u_long (*fdapi_htonl)(u_long hostlong);
typedef u_short (*fdapi_ntohs)(u_short netshort);
+typedef struct hostent* (*fdapi_gethostbyname)(const char *name);
typedef int (*fdapi_getpeername)(int sockfd, struct sockaddr *addr, socklen_t * addrlen);
typedef int (*fdapi_getsockname)(int sockfd, struct sockaddr* addrsock, int* addrlen );
typedef void (*fdapi_freeaddrinfo)(struct addrinfo *ai);
@@ -159,12 +163,14 @@ extern fdapi_access access;
extern fdapi_bind bind;
extern fdapi_connect connect;
extern fdapi_fcntl fcntl;
+extern fdapi_ioctl ioctl;
extern fdapi_fstat fdapi_fstat64;
extern fdapi_freeaddrinfo freeaddrinfo;
extern fdapi_fsync fsync;
extern fdapi_ftruncate ftruncate;
extern fdapi_getaddrinfo getaddrinfo;
extern fdapi_getsockopt getsockopt;
+extern fdapi_gethostbyname gethostbyname;
extern fdapi_getpeername getpeername;
extern fdapi_getsockname getsockname;
extern fdapi_htonl htonl;
@@ -180,7 +186,9 @@ extern fdapi_open open;
extern fdapi_pipe pipe;
extern fdapi_poll poll;
extern fdapi_read read;
+extern fdapi_recv recv;
extern fdapi_select select;
+extern fdapi_send send;
extern fdapi_setsockopt setsockopt;
extern fdapi_socket socket;
extern fdapi_write write;
diff --git a/src/Win32_Interop/Win32_Interop.vcxproj b/src/Win32_Interop/Win32_Interop.vcxproj
index 93fc44b..b75d89b
--- a/src/Win32_Interop/Win32_Interop.vcxproj
+++ b/src/Win32_Interop/Win32_Interop.vcxproj
@@ -74,35 +74,6 @@
<ClInclude Include="win32_wsiocp2.h" />
<ClInclude Include="WS2tcpip.h" />
</ItemGroup>
- <ItemGroup>
- <CustomBuild Include="EventLog.mc">
- <FileType>Document</FileType>
- <Command Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">md resources
-mc.exe -A -b -c -h . -r resources EventLog.mc
-rc.exe -foresources/EventLog.res resources/EventLog.rc
-link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
-</Command>
- <Command Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">md resources
-mc.exe -A -b -c -h . -r resources EventLog.mc
-rc.exe -foresources/EventLog.res resources/EventLog.rc
-link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
-</Command>
- <Outputs Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">EventLog.h</Outputs>
- <Outputs Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">EventLog.h</Outputs>
- <Command Condition="'$(Configuration)|$(Platform)'=='Release|x64'">md resources
-mc.exe -A -b -c -h . -r resources EventLog.mc
-rc.exe -foresources/EventLog.res resources/EventLog.rc
-link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
-</Command>
- <Command Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">md resources
-mc.exe -A -b -c -h . -r resources EventLog.mc
-rc.exe -foresources/EventLog.res resources/EventLog.rc
-link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
-</Command>
- <Outputs Condition="'$(Configuration)|$(Platform)'=='Release|x64'">EventLog.h</Outputs>
- <Outputs Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">EventLog.h</Outputs>
- </CustomBuild>
- </ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{8C07F811-C81C-432C-B334-1AE6FAECF951}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
@@ -113,27 +84,25 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v120</PlatformToolset>
+ <PlatformToolset>v140_xp</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v120</PlatformToolset>
- <WholeProgramOptimization>true</WholeProgramOptimization>
+ <PlatformToolset>v140_xp</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v120</PlatformToolset>
- <WholeProgramOptimization>true</WholeProgramOptimization>
+ <PlatformToolset>v140_xp</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
@@ -152,13 +121,9 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
- <IntDir>$(Platform)\$(Configuration)\</IntDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
- <IntDir>$(Platform)\$(Configuration)\</IntDir>
+ <PropertyGroup>
+ <OutDir>$(SolutionDir)build\$(Platform)\$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)build\$(Platform)\$(Configuration)\$(MSBuildProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
@@ -166,9 +131,10 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1</PreprocessorDefinitions>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;_NO_CRT_STDIO_INLINE;_CRT_SECURE_NO_DEPRECATE;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(ProjectDir)..\..\deps\lua\src;$(ProjectDir)..\..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ <MinimalRebuild>false</MinimalRebuild>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
@@ -186,9 +152,10 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1;_WIN32_WINNT=0x0501</PreprocessorDefinitions>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;_NO_CRT_STDIO_INLINE;_CRT_SECURE_NO_DEPRECATE;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1;_WIN32_WINNT=0x0501</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(ProjectDir)..\..\deps\lua\src;$(ProjectDir)..\..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ <MinimalRebuild>false</MinimalRebuild>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
@@ -211,10 +178,9 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
<Optimization>Full</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1</PreprocessorDefinitions>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;_NO_CRT_STDIO_INLINE;_CRT_SECURE_NO_DEPRECATE;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(ProjectDir)..\..\deps\lua\src;$(ProjectDir)..\..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
- <WholeProgramOptimization>true</WholeProgramOptimization>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
@@ -235,9 +201,9 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1;_WIN32_WINNT=0x0501</PreprocessorDefinitions>
+ <PreprocessorDefinitions>_WIN32_WINNT=0x0502;USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;_NO_CRT_STDIO_INLINE;_CRT_SECURE_NO_DEPRECATE;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1;_WIN32_WINNT=0x0501</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(ProjectDir)..\..\deps\lua\src;$(ProjectDir)..\..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
diff --git a/src/Win32_Interop/Win32_service.cpp b/src/Win32_Interop/Win32_service.cpp
index 488538e..1c33f53
--- a/src/Win32_Interop/Win32_service.cpp
+++ b/src/Win32_Interop/Win32_service.cpp
@@ -59,7 +59,6 @@ this should preceed the other arguments passed to redis. For instance:
#include <windowsx.h>
#include <shlobj.h>
#include <tchar.h>
-#include <strsafe.h>
#include <aclapi.h>
#include "Win32_EventLog.h"
#include <algorithm>
diff --git a/src/ziplist.c b/src/ziplist.c
index 24b0a7c..29d445d
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -920,7 +920,7 @@ void ziplistRepr(unsigned char *zl) {
entry = zipEntry(p);
printf(
"{"
- "addr 0x%08lx, " /* TODO" verify 0x%08lx */
+ "addr %p, "
"index %2d, "
"offset %5ld, "
"rl: %5u, "
@@ -929,9 +929,9 @@ void ziplistRepr(unsigned char *zl) {
"pls: %2u, "
"payload %5u"
"} ",
- (PORT_ULONG)p,
+ (void *)p,
index,
- (PORT_ULONG)(p-zl),
+ (long)(p-zl),
entry.headersize+entry.len,
entry.headersize,
entry.prevrawlen,
src/global_scheduler/CMakeLists.txt
-14
View File
@@ -1,14 +0,0 @@
cmake_minimum_required(VERSION 3.4)
project(global_scheduler)
include_directories(${CMAKE_CURRENT_LIST_DIR})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall")
add_executable(global_scheduler global_scheduler.cc global_scheduler_algorithm.cc)
# Make sure ${HIREDIS_LIB} is ready before linking.
add_dependencies(global_scheduler hiredis common)
target_link_libraries(global_scheduler common ${HIREDIS_LIB} ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} pthread)
src/global_scheduler/global_scheduler.cc
-492
View File
@@ -1,492 +0,0 @@
#include <getopt.h>
#include <signal.h>
#include <stdlib.h>
#include "common.h"
#include "event_loop.h"
#include "global_scheduler.h"
#include "global_scheduler_algorithm.h"
#include "net.h"
#include "ray/util/util.h"
#include "state/db_client_table.h"
#include "state/local_scheduler_table.h"
#include "state/object_table.h"
#include "state/table.h"
#include "state/task_table.h"
/**
* Retry the task assignment. If the local scheduler that the task is assigned
* to is no longer active, do not retry the assignment.
* TODO(rkn): We currently only retry the method if the global scheduler
* publishes a task to a local scheduler before the local scheduler has
* subscribed to the channel. If we enforce that ordering, we can remove this
* retry method.
*
* @param id The task ID.
* @param user_context The global scheduler state.
* @param user_data The Task that failed to be assigned.
* @return Void.
*/
void assign_task_to_local_scheduler_retry(UniqueID id,
void *user_context,
void *user_data) {
GlobalSchedulerState *state = (GlobalSchedulerState *) user_context;
Task *task = (Task *) user_data;
RAY_CHECK(Task_state(task) == TaskStatus::SCHEDULED);
// If the local scheduler has died since we requested the task assignment, do
// not retry again.
DBClientID local_scheduler_id = Task_local_scheduler(task);
auto it = state->local_schedulers.find(local_scheduler_id);
if (it == state->local_schedulers.end()) {
return;
}
// The local scheduler is still alive. The failure is most likely due to the
// task assignment getting published before the local scheduler subscribed to
// the channel. Retry the assignment.
auto retryInfo = RetryInfo{
.num_retries = 0, // This value is unused.
.timeout = 0, // This value is unused.
.fail_callback = assign_task_to_local_scheduler_retry,
};
task_table_update(state->db, Task_copy(task), &retryInfo, NULL, user_context);
}
/**
* Assign the given task to the local scheduler, update Redis and scheduler data
* structures.
*
* @param state Global scheduler state.
* @param task Task to be assigned to the local scheduler.
* @param local_scheduler_id DB client ID for the local scheduler.
* @return Void.
*/
void assign_task_to_local_scheduler(GlobalSchedulerState *state,
Task *task,
DBClientID local_scheduler_id) {
TaskSpec *spec = Task_task_execution_spec(task)->Spec();
RAY_LOG(DEBUG) << "assigning task to local_scheduler_id = "
<< local_scheduler_id;
Task_set_state(task, TaskStatus::SCHEDULED);
Task_set_local_scheduler(task, local_scheduler_id);
RAY_LOG(DEBUG) << "Issuing a task table update for task = "
<< Task_task_id(task);
auto retryInfo = RetryInfo{
.num_retries = 0, // This value is unused.
.timeout = 0, // This value is unused.
.fail_callback = assign_task_to_local_scheduler_retry,
};
task_table_update(state->db, Task_copy(task), &retryInfo, NULL, state);
/* Update the object table info to reflect the fact that the results of this
* task will be created on the machine that the task was assigned to. This can
* be used to improve locality-aware scheduling. */
for (int64_t i = 0; i < TaskSpec_num_returns(spec); ++i) {
ObjectID return_id = TaskSpec_return(spec, i);
if (state->scheduler_object_info_table.find(return_id) ==
state->scheduler_object_info_table.end()) {
SchedulerObjectInfo &obj_info_entry =
state->scheduler_object_info_table[return_id];
/* The value -1 indicates that the size of the object is not known yet. */
obj_info_entry.data_size = -1;
}
RAY_CHECK(state->local_scheduler_plasma_map.count(local_scheduler_id) == 1);
state->scheduler_object_info_table[return_id].object_locations.push_back(
state->local_scheduler_plasma_map[local_scheduler_id]);
}
/* TODO(rkn): We should probably pass around local_scheduler struct pointers
* instead of db_client_id objects. */
/* Update the local scheduler info. */
auto it = state->local_schedulers.find(local_scheduler_id);
RAY_CHECK(it != state->local_schedulers.end());
LocalScheduler &local_scheduler = it->second;
local_scheduler.num_tasks_sent += 1;
local_scheduler.num_recent_tasks_sent += 1;
// Resource accounting update for this local scheduler.
for (auto const &resource_pair : TaskSpec_get_required_resources(spec)) {
std::string resource_name = resource_pair.first;
double resource_quantity = resource_pair.second;
// The local scheduler must have this resource because otherwise we wouldn't
// be assigning the task to this local scheduler.
RAY_CHECK(local_scheduler.info.dynamic_resources.count(resource_name) ==
1 ||
resource_quantity == 0);
// Subtract task's resource from the cached dynamic resource capacity for
// this local scheduler. This will be overwritten on the next heartbeat.
local_scheduler.info.dynamic_resources[resource_name] =
MAX(0, local_scheduler.info.dynamic_resources[resource_name] -
resource_quantity);
}
}
GlobalSchedulerState *GlobalSchedulerState_init(event_loop *loop,
const char *node_ip_address,
const char *redis_primary_addr,
int redis_primary_port) {
GlobalSchedulerState *state = new GlobalSchedulerState();
state->loop = loop;
state->db = db_connect(std::string(redis_primary_addr), redis_primary_port,
"global_scheduler", node_ip_address,
std::vector<std::string>());
db_attach(state->db, loop, false);
state->policy_state = GlobalSchedulerPolicyState_init();
return state;
}
void GlobalSchedulerState_free(GlobalSchedulerState *state) {
db_disconnect(state->db);
state->local_schedulers.clear();
GlobalSchedulerPolicyState_free(state->policy_state);
/* Delete the plasma to local scheduler association map. */
state->plasma_local_scheduler_map.clear();
/* Delete the local scheduler to plasma association map. */
state->local_scheduler_plasma_map.clear();
/* Free the scheduler object info table. */
state->scheduler_object_info_table.clear();
/* Free the array of unschedulable tasks. */
int64_t num_pending_tasks = state->pending_tasks.size();
if (num_pending_tasks > 0) {
RAY_LOG(WARNING) << "There are " << num_pending_tasks
<< " remaining tasks in the pending tasks array.";
}
for (int i = 0; i < num_pending_tasks; ++i) {
Task *pending_task = state->pending_tasks[i];
Task_free(pending_task);
}
state->pending_tasks.clear();
/* Destroy the event loop. */
destroy_outstanding_callbacks(state->loop);
event_loop_destroy(state->loop);
state->loop = NULL;
/* Free the global scheduler state. */
delete state;
}
/* We need this code so we can clean up when we get a SIGTERM signal. */
GlobalSchedulerState *g_state;
void signal_handler(int signal) {
if (signal == SIGTERM) {
GlobalSchedulerState_free(g_state);
exit(0);
}
}
/* End of the cleanup code. */
void process_task_waiting(Task *waiting_task, void *user_context) {
GlobalSchedulerState *state = (GlobalSchedulerState *) user_context;
RAY_LOG(DEBUG) << "Task waiting callback is called.";
bool successfully_assigned =
handle_task_waiting(state, state->policy_state, waiting_task);
/* If the task was not successfully submitted to a local scheduler, add the
* task to the array of pending tasks. The global scheduler will periodically
* resubmit the tasks in this array. */
if (!successfully_assigned) {
Task *task_copy = Task_copy(waiting_task);
state->pending_tasks.push_back(task_copy);
}
}
void add_local_scheduler(GlobalSchedulerState *state,
DBClientID db_client_id,
const char *manager_address) {
/* Add plasma_manager ip:port -> local_scheduler_db_client_id association to
* state. */
state->plasma_local_scheduler_map[std::string(manager_address)] =
db_client_id;
/* Add local_scheduler_db_client_id -> plasma_manager ip:port association to
* state. */
state->local_scheduler_plasma_map[db_client_id] =
std::string(manager_address);
/* Add new local scheduler to the state. */
LocalScheduler &local_scheduler = state->local_schedulers[db_client_id];
local_scheduler.id = db_client_id;
local_scheduler.num_heartbeats_missed = 0;
local_scheduler.num_tasks_sent = 0;
local_scheduler.num_recent_tasks_sent = 0;
local_scheduler.info.task_queue_length = 0;
local_scheduler.info.available_workers = 0;
/* Allow the scheduling algorithm to process this event. */
handle_new_local_scheduler(state, state->policy_state, db_client_id);
}
std::unordered_map<DBClientID, LocalScheduler>::iterator remove_local_scheduler(
GlobalSchedulerState *state,
std::unordered_map<DBClientID, LocalScheduler>::iterator it) {
RAY_CHECK(it != state->local_schedulers.end());
DBClientID local_scheduler_id = it->first;
it = state->local_schedulers.erase(it);
/* Remove the local scheduler from the mappings. This code only makes sense if
* there is a one-to-one mapping between local schedulers and plasma managers.
*/
std::string manager_address =
state->local_scheduler_plasma_map[local_scheduler_id];
state->local_scheduler_plasma_map.erase(local_scheduler_id);
state->plasma_local_scheduler_map.erase(manager_address);
handle_local_scheduler_removed(state, state->policy_state,
local_scheduler_id);
return it;
}
/**
* Process a notification about a new DB client connecting to Redis.
*
* @param manager_address An ip:port pair for the plasma manager associated with
* this db client.
* @return Void.
*/
void process_new_db_client(DBClient *db_client, void *user_context) {
GlobalSchedulerState *state = (GlobalSchedulerState *) user_context;
RAY_LOG(DEBUG) << "db client table callback for db client = "
<< db_client->id;
if (strncmp(db_client->client_type.c_str(), "local_scheduler",
strlen("local_scheduler")) == 0) {
bool local_scheduler_present =
(state->local_schedulers.find(db_client->id) !=
state->local_schedulers.end());
if (db_client->is_alive) {
/* This is a notification for an insert. We may receive duplicate
* notifications since we read the entire table before processing
* notifications. Filter out local schedulers that we already added. */
if (!local_scheduler_present) {
add_local_scheduler(state, db_client->id,
db_client->manager_address.c_str());
}
} else {
if (local_scheduler_present) {
remove_local_scheduler(state,
state->local_schedulers.find(db_client->id));
}
}
}
}
/**
* Process notification about the new object information.
*
* @param object_id ID of the object that the notification is about.
* @param data_size The object size.
* @param manager_count The number of locations for this object.
* @param manager_ids The vector of Plasma Manager client IDs.
* @param user_context The user context.
* @return Void.
*/
void object_table_subscribe_callback(ObjectID object_id,
int64_t data_size,
const std::vector<DBClientID> &manager_ids,
void *user_context) {
/* Extract global scheduler state from the callback context. */
GlobalSchedulerState *state = (GlobalSchedulerState *) user_context;
RAY_LOG(DEBUG) << "object table subscribe callback for OBJECT = "
<< object_id;
const std::vector<std::string> managers =
db_client_table_get_ip_addresses(state->db, manager_ids);
RAY_LOG(DEBUG) << "\tManagers<" << managers.size() << ">:";
for (size_t i = 0; i < managers.size(); i++) {
RAY_LOG(DEBUG) << "\t\t" << managers[i];
}
if (state->scheduler_object_info_table.find(object_id) ==
state->scheduler_object_info_table.end()) {
/* Construct a new object info hash table entry. */
SchedulerObjectInfo &obj_info_entry =
state->scheduler_object_info_table[object_id];
obj_info_entry.data_size = data_size;
RAY_LOG(DEBUG) << "New object added to object_info_table with id = "
<< object_id;
RAY_LOG(DEBUG) << "\tmanager locations:";
for (size_t i = 0; i < managers.size(); i++) {
RAY_LOG(DEBUG) << "\t\t" << managers[i];
}
}
SchedulerObjectInfo &obj_info_entry =
state->scheduler_object_info_table[object_id];
/* In all cases, replace the object location vector on each callback. */
obj_info_entry.object_locations.clear();
for (size_t i = 0; i < managers.size(); i++) {
obj_info_entry.object_locations.push_back(managers[i]);
}
}
void local_scheduler_table_handler(DBClientID client_id,
LocalSchedulerInfo info,
void *user_context) {
/* Extract global scheduler state from the callback context. */
GlobalSchedulerState *state = (GlobalSchedulerState *) user_context;
ARROW_UNUSED(state);
RAY_LOG(DEBUG) << "Local scheduler heartbeat from db_client_id " << client_id;
RAY_LOG(DEBUG) << "total workers = " << info.total_num_workers
<< ", task queue length = " << info.task_queue_length
<< ", available workers = " << info.available_workers;
/* Update the local scheduler info struct. */
auto it = state->local_schedulers.find(client_id);
if (it != state->local_schedulers.end()) {
if (info.is_dead) {
/* The local scheduler is exiting. Increase the number of heartbeats
* missed to the timeout threshold. This will trigger removal of the
* local scheduler the next time the timeout handler fires. */
it->second.num_heartbeats_missed =
RayConfig::instance().num_heartbeats_timeout();
} else {
/* Reset the number of tasks sent since the last heartbeat. */
LocalScheduler &local_scheduler = it->second;
local_scheduler.num_heartbeats_missed = 0;
local_scheduler.num_recent_tasks_sent = 0;
local_scheduler.info = info;
}
} else {
RAY_LOG(WARNING) << "client_id didn't match any cached local scheduler "
<< "entries";
}
}
int task_cleanup_handler(event_loop *loop, timer_id id, void *context) {
GlobalSchedulerState *state = (GlobalSchedulerState *) context;
/* Loop over the pending tasks in reverse order and resubmit them. */
auto it = state->pending_tasks.end();
while (it != state->pending_tasks.begin()) {
it--;
Task *pending_task = *it;
/* Pretend that the task has been resubmitted. */
bool successfully_assigned =
handle_task_waiting(state, state->policy_state, pending_task);
if (successfully_assigned) {
/* The task was successfully assigned, so remove it from this list and
* free it. This uses the fact that pending_tasks is a vector and so erase
* returns an iterator to the next element in the vector. */
it = state->pending_tasks.erase(it);
Task_free(pending_task);
}
}
return GLOBAL_SCHEDULER_TASK_CLEANUP_MILLISECONDS;
}
int heartbeat_timeout_handler(event_loop *loop, timer_id id, void *context) {
GlobalSchedulerState *state = (GlobalSchedulerState *) context;
/* Check for local schedulers that have missed a number of heartbeats. If any
* local schedulers have died, notify others so that the state can be cleaned
* up. */
/* TODO(swang): If the local scheduler hasn't actually died, then it should
* clean up its state and exit upon receiving this notification. */
auto it = state->local_schedulers.begin();
while (it != state->local_schedulers.end()) {
if (it->second.num_heartbeats_missed >=
RayConfig::instance().num_heartbeats_timeout()) {
RAY_LOG(WARNING) << "Missed too many heartbeats from local scheduler, "
<< "marking as dead.";
/* Notify others by updating the global state. */
db_client_table_remove(state->db, it->second.id, NULL, NULL, NULL);
/* Remove the scheduler from the local state. The call to
* remove_local_scheduler modifies the container in place and returns the
* next iterator. */
it = remove_local_scheduler(state, it);
} else {
it->second.num_heartbeats_missed += 1;
it++;
}
}
/* Reset the timer. */
return RayConfig::instance().heartbeat_timeout_milliseconds();
}
void start_server(const char *node_ip_address,
const char *redis_primary_addr,
int redis_primary_port) {
event_loop *loop = event_loop_create();
g_state = GlobalSchedulerState_init(loop, node_ip_address, redis_primary_addr,
redis_primary_port);
/* TODO(rkn): subscribe to notifications from the object table. */
/* Subscribe to notifications about new local schedulers. TODO(rkn): this
* needs to also get all of the clients that registered with the database
* before this call to subscribe. */
db_client_table_subscribe(g_state->db, process_new_db_client,
(void *) g_state, NULL, NULL, NULL);
/* Subscribe to notifications about waiting tasks. If a local scheduler
* submits tasks to the global scheduler before the global scheduler
* successfully subscribes, then the local scheduler that submitted the tasks
* will retry. */
task_table_subscribe(g_state->db, UniqueID::nil(), TaskStatus::WAITING,
process_task_waiting, (void *) g_state, NULL, NULL,
NULL);
object_table_subscribe_to_notifications(g_state->db, true,
object_table_subscribe_callback,
g_state, NULL, NULL, NULL);
/* Subscribe to notifications from local schedulers. These notifications serve
* as heartbeats and contain informaion about the load on the local
* schedulers. */
local_scheduler_table_subscribe(g_state->db, local_scheduler_table_handler,
g_state, NULL);
/* Start a timer that periodically checks if there are queued tasks that can
* be scheduled. Currently this is only used to handle the special case in
* which a task is waiting and no node meets its static resource requirements.
* If a new node joins the cluster that does have enough resources, then this
* timer should notice and schedule the task. */
event_loop_add_timer(loop, GLOBAL_SCHEDULER_TASK_CLEANUP_MILLISECONDS,
task_cleanup_handler, g_state);
event_loop_add_timer(loop,
RayConfig::instance().heartbeat_timeout_milliseconds(),
heartbeat_timeout_handler, g_state);
/* Start the event loop. */
event_loop_run(loop);
}
int main(int argc, char *argv[]) {
InitShutdownRAII ray_log_shutdown_raii(
ray::RayLog::StartRayLog, ray::RayLog::ShutDownRayLog, argv[0],
ray::RayLogLevel::INFO, /*log_dir=*/"");
ray::RayLog::InstallFailureSignalHandler();
signal(SIGTERM, signal_handler);
/* IP address and port of the primary redis instance. */
char *redis_primary_addr_port = NULL;
/* The IP address of the node that this global scheduler is running on. */
char *node_ip_address = NULL;
int c;
while ((c = getopt(argc, argv, "h:r:")) != -1) {
switch (c) {
case 'r':
redis_primary_addr_port = optarg;
break;
case 'h':
node_ip_address = optarg;
break;
default:
RAY_LOG(FATAL) << "unknown option " << c;
}
}
char redis_primary_addr[16];
int redis_primary_port = -1;
if (!redis_primary_addr_port ||
parse_ip_addr_port(redis_primary_addr_port, redis_primary_addr,
&redis_primary_port) == -1) {
RAY_LOG(FATAL) << "specify the primary redis address like 127.0.0.1:6379 "
<< "with the -r switch";
}
if (!node_ip_address) {
RAY_LOG(FATAL) << "specify the node IP address with the -h switch";
}
start_server(node_ip_address, redis_primary_addr, redis_primary_port);
}
src/global_scheduler/global_scheduler.h
-94
View File
@@ -1,94 +0,0 @@
#ifndef GLOBAL_SCHEDULER_H
#define GLOBAL_SCHEDULER_H
#include "task.h"
#include <unordered_map>
#include "ray/gcs/client.h"
#include "state/db.h"
#include "state/local_scheduler_table.h"
/* The frequency with which the global scheduler checks if there are any tasks
* that haven't been scheduled yet. */
#define GLOBAL_SCHEDULER_TASK_CLEANUP_MILLISECONDS 100
/** Contains all information that is associated with a local scheduler. */
typedef struct {
/** The ID of the local scheduler in Redis. */
DBClientID id;
/** The number of heartbeat intervals that have passed since we last heard
* from this local scheduler. */
int64_t num_heartbeats_missed;
/** The number of tasks sent from the global scheduler to this local
* scheduler. */
int64_t num_tasks_sent;
/** The number of tasks sent from the global scheduler to this local scheduler
* since the last heartbeat arrived. */
int64_t num_recent_tasks_sent;
/** The latest information about the local scheduler capacity. This is updated
* every time a new local scheduler heartbeat arrives. */
LocalSchedulerInfo info;
} LocalScheduler;
typedef class GlobalSchedulerPolicyState GlobalSchedulerPolicyState;
/**
* This defines a hash table used to cache information about different objects.
*/
typedef struct {
/** The size in bytes of the object. */
int64_t data_size;
/** A vector of object locations for this object. */
std::vector<std::string> object_locations;
} SchedulerObjectInfo;
/**
* Global scheduler state structure.
*/
typedef struct {
/** The global scheduler event loop. */
event_loop *loop;
/** The global state store database. */
DBHandle *db;
/** A hash table mapping local scheduler ID to the local schedulers that are
* connected to Redis. */
std::unordered_map<DBClientID, LocalScheduler> local_schedulers;
/** The state managed by the scheduling policy. */
GlobalSchedulerPolicyState *policy_state;
/** The plasma_manager ip:port -> local_scheduler_db_client_id association. */
std::unordered_map<std::string, DBClientID> plasma_local_scheduler_map;
/** The local_scheduler_db_client_id -> plasma_manager ip:port association. */
std::unordered_map<DBClientID, std::string> local_scheduler_plasma_map;
/** Objects cached by this global scheduler instance. */
std::unordered_map<ObjectID, SchedulerObjectInfo> scheduler_object_info_table;
/** An array of tasks that haven't been scheduled yet. */
std::vector<Task *> pending_tasks;
} GlobalSchedulerState;
/**
* This is a helper method to look up the local scheduler struct that
* corresponds to a particular local_scheduler_id.
*
* @param state The state of the global scheduler.
* @param The local_scheduler_id of the local scheduler.
* @return The corresponding local scheduler struct. If the global scheduler is
* not aware of the local scheduler, then this will be NULL.
*/
LocalScheduler *get_local_scheduler(GlobalSchedulerState *state,
DBClientID local_scheduler_id);
/**
* Assign the given task to the local scheduler, update Redis and scheduler data
* structures.
*
* @param state Global scheduler state.
* @param task Task to be assigned to the local scheduler.
* @param local_scheduler_id DB client ID for the local scheduler.
* @return Void.
*/
void assign_task_to_local_scheduler(GlobalSchedulerState *state,
Task *task,
DBClientID local_scheduler_id);
#endif /* GLOBAL_SCHEDULER_H */
src/global_scheduler/global_scheduler_algorithm.cc
-257
View File
@@ -1,257 +0,0 @@
#include <limits.h>
#include "task.h"
#include "state/task_table.h"
#include "global_scheduler_algorithm.h"
GlobalSchedulerPolicyState *GlobalSchedulerPolicyState_init(void) {
GlobalSchedulerPolicyState *policy_state = new GlobalSchedulerPolicyState();
return policy_state;
}
void GlobalSchedulerPolicyState_free(GlobalSchedulerPolicyState *policy_state) {
delete policy_state;
}
/**
* Checks if the given local scheduler satisfies the task's hard constraints.
*
* @param scheduler Local scheduler.
* @param spec Task specification.
* @return True if all tasks's resource constraints are satisfied. False
* otherwise.
*/
bool constraints_satisfied_hard(const LocalScheduler *scheduler,
const TaskSpec *spec) {
if (scheduler->info.static_resources.count("CPU") == 1 &&
scheduler->info.static_resources.at("CPU") == 0) {
// Don't give tasks to local schedulers that have 0 CPUs. This can be an
// issue for actor creation tasks that require 0 CPUs (but the subsequent
// actor methods require some CPUs).
return false;
}
for (auto const &resource_pair : TaskSpec_get_required_resources(spec)) {
std::string resource_name = resource_pair.first;
double resource_quantity = resource_pair.second;
// Continue on if the task doesn't actually require this resource.
if (resource_quantity == 0) {
continue;
}
// Check if the local scheduler has this resource.
if (scheduler->info.static_resources.count(resource_name) == 0) {
return false;
}
// Check if the local scheduler has enough of the resource.
if (scheduler->info.static_resources.at(resource_name) <
resource_quantity) {
return false;
}
}
return true;
}
int64_t locally_available_data_size(const GlobalSchedulerState *state,
DBClientID local_scheduler_id,
TaskSpec *task_spec) {
/* This function will compute the total size of all the object dependencies
* for the given task that are already locally available to the specified
* local scheduler. */
int64_t task_data_size = 0;
RAY_CHECK(state->local_scheduler_plasma_map.count(local_scheduler_id) == 1);
const std::string &plasma_manager =
state->local_scheduler_plasma_map.at(local_scheduler_id);
/* TODO(rkn): Note that if the same object ID appears as multiple arguments,
* then it will be overcounted. */
for (int64_t i = 0; i < TaskSpec_num_args(task_spec); ++i) {
int count = TaskSpec_arg_id_count(task_spec, i);
for (int j = 0; j < count; ++j) {
ObjectID object_id = TaskSpec_arg_id(task_spec, i, j);
if (state->scheduler_object_info_table.count(object_id) == 0) {
/* If this global scheduler is not aware of this object ID, then ignore
* it. */
continue;
}
const SchedulerObjectInfo &object_size_info =
state->scheduler_object_info_table.at(object_id);
if (std::find(object_size_info.object_locations.begin(),
object_size_info.object_locations.end(), plasma_manager) ==
object_size_info.object_locations.end()) {
/* This local scheduler does not have access to this object, so don't
* count this object. */
continue;
}
/* Look at the size of the object. */
int64_t object_size = object_size_info.data_size;
if (object_size == -1) {
/* This means that this global scheduler does not know the object size
* yet, so assume that the object is one megabyte. TODO(rkn): Maybe we
* should instead use the average object size. */
object_size = 1000000;
}
/* If we get here, then this local scheduler has access to this object, so
* count the contribution of this object. */
task_data_size += object_size;
}
}
return task_data_size;
}
double calculate_cost_pending(const GlobalSchedulerState *state,
const LocalScheduler *scheduler,
TaskSpec *task_spec) {
/* Calculate how much data is already present on this machine. TODO(rkn): Note
* that this information is not being used yet. Fix this. */
locally_available_data_size(state, scheduler->id, task_spec);
/* TODO(rkn): This logic does not load balance properly when the different
* machines have different sizes. Fix this. */
double cost_pending = scheduler->num_recent_tasks_sent +
scheduler->info.task_queue_length -
scheduler->info.available_workers;
return cost_pending;
}
bool handle_task_waiting_random(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
Task *task) {
TaskSpec *task_spec = Task_task_execution_spec(task)->Spec();
RAY_CHECK(task_spec != NULL)
<< "task wait handler encounted a task with NULL spec";
std::vector<DBClientID> feasible_nodes;
for (const auto &it : state->local_schedulers) {
// Local scheduler map iterator yields <DBClientID, LocalScheduler> pairs.
const LocalScheduler &local_scheduler = it.second;
if (!constraints_satisfied_hard(&local_scheduler, task_spec)) {
continue;
}
// Add this local scheduler as a candidate for random selection.
feasible_nodes.push_back(it.first);
}
if (feasible_nodes.size() == 0) {
RAY_LOG(ERROR) << "Infeasible task. No nodes satisfy hard constraints for "
<< "task = " << Task_task_id(task);
return false;
}
// Randomly select the local scheduler. TODO(atumanov): replace with
// std::discrete_distribution<int>.
std::uniform_int_distribution<> dis(0, feasible_nodes.size() - 1);
DBClientID local_scheduler_id =
feasible_nodes[dis(policy_state->getRandomGenerator())];
RAY_CHECK(!local_scheduler_id.is_nil())
<< "Task is feasible, but doesn't have a local scheduler assigned.";
// A local scheduler ID was found, so assign the task.
assign_task_to_local_scheduler(state, task, local_scheduler_id);
return true;
}
bool handle_task_waiting_cost(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
Task *task) {
TaskSpec *task_spec = Task_task_execution_spec(task)->Spec();
int64_t curtime = current_time_ms();
RAY_CHECK(task_spec != NULL)
<< "task wait handler encounted a task with NULL spec";
// For tasks already seen by the global scheduler (spillback > 1),
// adjust scheduled task counts for the source local scheduler.
if (task->execution_spec->SpillbackCount() > 1) {
auto it = state->local_schedulers.find(task->local_scheduler_id);
// Task's previous local scheduler must be present and known.
RAY_CHECK(it != state->local_schedulers.end());
LocalScheduler &src_local_scheduler = it->second;
src_local_scheduler.num_recent_tasks_sent -= 1;
}
bool task_feasible = false;
// Go through all the nodes, calculate the score for each, pick max score.
double best_local_scheduler_score = INT32_MIN;
RAY_CHECK(best_local_scheduler_score < 0)
<< "We might have a floating point underflow";
RAY_LOG(INFO) << "ct[" << curtime << "] task from "
<< task->local_scheduler_id << " spillback "
<< task->execution_spec->SpillbackCount();
// The best node to send this task.
DBClientID best_local_scheduler_id = DBClientID::nil();
for (auto it = state->local_schedulers.begin();
it != state->local_schedulers.end(); it++) {
// For each local scheduler, calculate its score. Check hard constraints
// first.
LocalScheduler *scheduler = &(it->second);
if (!constraints_satisfied_hard(scheduler, task_spec)) {
continue;
}
// Skip the local scheduler the task came from.
if (task->local_scheduler_id == scheduler->id) {
continue;
}
task_feasible = true;
// This node satisfies the hard capacity constraint. Calculate its score.
double score = -1 * calculate_cost_pending(state, scheduler, task_spec);
RAY_LOG(INFO) << "ct[" << curtime << "][" << scheduler->id << "][q"
<< scheduler->info.task_queue_length << "][w"
<< scheduler->info.available_workers << "]: score " << score
<< " bestscore " << best_local_scheduler_score;
if (score >= best_local_scheduler_score) {
best_local_scheduler_score = score;
best_local_scheduler_id = scheduler->id;
}
}
if (!task_feasible) {
RAY_LOG(ERROR) << "Infeasible task. No nodes satisfy hard constraints for "
<< "task = " << Task_task_id(task);
// TODO(atumanov): propagate this error to the task's driver and/or
// cache the task in case new local schedulers satisfy it in the future.
return false;
}
RAY_CHECK(!best_local_scheduler_id.is_nil())
<< "Task is feasible, but doesn't have a local scheduler assigned.";
// A local scheduler ID was found, so assign the task.
assign_task_to_local_scheduler(state, task, best_local_scheduler_id);
return true;
}
bool handle_task_waiting(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
Task *task) {
return handle_task_waiting_random(state, policy_state, task);
}
void handle_object_available(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
ObjectID object_id) {
/* Do nothing for now. */
}
void handle_new_local_scheduler(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
DBClientID db_client_id) {
/* Do nothing for now. */
}
void handle_local_scheduler_removed(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
DBClientID db_client_id) {
/* Do nothing for now. */
}
src/global_scheduler/global_scheduler_algorithm.h
-126
View File
@@ -1,126 +0,0 @@
#ifndef GLOBAL_SCHEDULER_ALGORITHM_H
#define GLOBAL_SCHEDULER_ALGORITHM_H
#include <chrono>
#include <random>
#include "common.h"
#include "global_scheduler.h"
#include "task.h"
/* ==== The scheduling algorithm ====
*
* This file contains declaration for all functions and data structures that
* need to be provided if you want to implement a new algorithm for the global
* scheduler.
*
*/
enum class GlobalSchedulerAlgorithm {
SCHED_ALGORITHM_ROUND_ROBIN = 1,
SCHED_ALGORITHM_TRANSFER_AWARE = 2,
SCHED_ALGORITHM_MAX
};
/// The class encapsulating state managed by the global scheduling policy.
class GlobalSchedulerPolicyState {
public:
GlobalSchedulerPolicyState(int64_t round_robin_index)
: round_robin_index_(round_robin_index),
gen_(std::chrono::high_resolution_clock::now()
.time_since_epoch()
.count()) {}
GlobalSchedulerPolicyState()
: round_robin_index_(0),
gen_(std::chrono::high_resolution_clock::now()
.time_since_epoch()
.count()) {}
/// Return the policy's random number generator.
///
/// @return The policy's random number generator.
std::mt19937_64 &getRandomGenerator() { return gen_; }
/// Return the round robin index maintained by policy state.
///
/// @return The round robin index.
int64_t getRoundRobinIndex() const { return round_robin_index_; }
private:
/// The index of the next local scheduler to assign a task to.
int64_t round_robin_index_;
/// Internally maintained random number generator.
std::mt19937_64 gen_;
};
/**
* Create the state of the global scheduler policy. This state must be freed by
* the caller.
*
* @return The state of the scheduling policy.
*/
GlobalSchedulerPolicyState *GlobalSchedulerPolicyState_init(void);
/**
* Free the global scheduler policy state.
*
* @param policy_state The policy state to free.
* @return Void.
*/
void GlobalSchedulerPolicyState_free(GlobalSchedulerPolicyState *policy_state);
/**
* Main new task handling function in the global scheduler.
*
* @param state Global scheduler state.
* @param policy_state State specific to the scheduling policy.
* @param task New task to be scheduled.
* @return True if the task was assigned to a local scheduler and false
* otherwise.
*/
bool handle_task_waiting(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
Task *task);
/**
* Handle the fact that a new object is available.
*
* @param state The global scheduler state.
* @param policy_state The state managed by the scheduling policy.
* @param object_id The ID of the object that is now available.
* @return Void.
*/
void handle_object_available(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
ObjectID object_id);
/**
* Handle a heartbeat message from a local scheduler. TODO(rkn): this is a
* placeholder for now.
*
* @param state The global scheduler state.
* @param policy_state The state managed by the scheduling policy.
* @return Void.
*/
void handle_local_scheduler_heartbeat(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state);
/**
* Handle the presence of a new local scheduler. Currently, this just adds the
* local scheduler to a queue of local schedulers.
*
* @param state The global scheduler state.
* @param policy_state The state managed by the scheduling policy.
* @param The db client ID of the new local scheduler.
* @return Void.
*/
void handle_new_local_scheduler(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
DBClientID db_client_id);
void handle_local_scheduler_removed(GlobalSchedulerState *state,
GlobalSchedulerPolicyState *policy_state,
DBClientID db_client_id);
#endif /* GLOBAL_SCHEDULER_ALGORITHM_H */
src/local_scheduler/CMakeLists.txt
-104
View File
@@ -1,104 +0,0 @@
cmake_minimum_required(VERSION 3.4)
project(local_scheduler)
add_definitions(-fPIC)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${PYTHON_INCLUDE_DIRS}")
include_directories("${NUMPY_INCLUDE_DIR}")
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall")
if(UNIX AND NOT APPLE)
link_libraries(rt)
endif()
include_directories("${CMAKE_CURRENT_LIST_DIR}/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../")
# TODO(pcm): get rid of this:
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/")
endif()
include_directories("${ARROW_INCLUDE_DIR}")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/format/")
# Compile flatbuffers
set(LOCAL_SCHEDULER_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/local_scheduler.fbs")
set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
set(LOCAL_SCHEDULER_FBS_OUTPUT_FILES
"${OUTPUT_DIR}/local_scheduler_generated.h")
add_custom_command(
OUTPUT ${LOCAL_SCHEDULER_FBS_OUTPUT_FILES}
COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${LOCAL_SCHEDULER_FBS_SRC} --gen-object-api --scoped-enums
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${LOCAL_SCHEDULER_FBS_SRC}"
VERBATIM)
add_custom_target(gen_local_scheduler_fbs DEPENDS ${LOCAL_SCHEDULER_FBS_OUTPUT_FILES})
add_dependencies(gen_local_scheduler_fbs arrow)
add_library(local_scheduler_client STATIC local_scheduler_client.cc)
# local_scheduler_shared.h includes ray/gcs/client.h which requires gen_gcs_fbs & gen_node_manager_fbs.
add_dependencies(local_scheduler_client common hiredis gen_local_scheduler_fbs ${COMMON_FBS_OUTPUT_FILES} gen_gcs_fbs gen_node_manager_fbs)
add_executable(local_scheduler local_scheduler.cc local_scheduler_algorithm.cc)
add_dependencies(local_scheduler hiredis)
target_link_libraries(local_scheduler local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY})
add_executable(local_scheduler_tests test/local_scheduler_tests.cc local_scheduler.cc local_scheduler_algorithm.cc)
add_dependencies(local_scheduler_tests hiredis)
target_link_libraries(local_scheduler_tests local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY})
target_compile_options(local_scheduler_tests PUBLIC "-DLOCAL_SCHEDULER_TEST")
macro(get_local_scheduler_library LANG VAR)
set(${VAR} "local_scheduler_library_${LANG}")
endmacro()
macro(set_local_scheduler_library LANG)
get_local_scheduler_library(${LANG} LOCAL_SCHEDULER_LIBRARY_${LANG})
set(LOCAL_SCHEDULER_LIBRARY_LANG ${LOCAL_SCHEDULER_LIBRARY_${LANG}})
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/")
file(GLOB LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC
lib/${LANG}/*.cc
${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/*.cc)
add_library(${LOCAL_SCHEDULER_LIBRARY_LANG} SHARED
${LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC})
if(APPLE)
if ("${LANG}" STREQUAL "python")
SET_TARGET_PROPERTIES(${LOCAL_SCHEDULER_LIBRARY_LANG} PROPERTIES SUFFIX .so)
endif()
target_link_libraries(${LOCAL_SCHEDULER_LIBRARY_LANG} "-undefined dynamic_lookup" local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
else(APPLE)
target_link_libraries(${LOCAL_SCHEDULER_LIBRARY_LANG} local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
endif(APPLE)
add_dependencies(${LOCAL_SCHEDULER_LIBRARY_LANG} gen_local_scheduler_fbs)
install(TARGETS ${LOCAL_SCHEDULER_LIBRARY_LANG} DESTINATION ${CMAKE_SOURCE_DIR}/local_scheduler)
endmacro()
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
set_local_scheduler_library("python")
endif()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
add_compile_options("-I$ENV{JAVA_HOME}/include/")
if(WIN32)
add_compile_options("-I$ENV{JAVA_HOME}/include/win32")
elseif(APPLE)
add_compile_options("-I$ENV{JAVA_HOME}/include/darwin")
else() # linux
add_compile_options("-I$ENV{JAVA_HOME}/include/linux")
endif()
set_local_scheduler_library("java")
endif()
src/local_scheduler/build/.gitkeep
View File
src/local_scheduler/format/local_scheduler.fbs
-130
View File
@@ -1,130 +0,0 @@
// Local scheduler protocol specification
namespace ray.local_scheduler.protocol;
enum MessageType:int {
// Task is submitted to the local scheduler. This is sent from a worker to a
// local scheduler.
SubmitTask = 1,
// Notify the local scheduler that a task has finished. This is sent from a
// worker to a local scheduler.
TaskDone,
// Log a message to the event table. This is sent from a worker to a local
// scheduler.
EventLogMessage,
// Send an initial connection message to the local scheduler. This is sent
// from a worker or driver to a local scheduler.
RegisterClientRequest,
// Send a reply confirming the successful registration of a worker or driver.
// This is sent from the local scheduler to a worker or driver.
RegisterClientReply,
// Notify the local scheduler that this client disconnected unexpectedly.
// This is sent from a worker to a local scheduler.
DisconnectClient,
// Notify the local scheduler that this client is disconnecting gracefully.
// This is sent from a worker to a local scheduler.
IntentionalDisconnectClient,
// Get a new task from the local scheduler. This is sent from a worker to a
// local scheduler.
GetTask,
// Tell a worker to execute a task. This is sent from a local scheduler to a
// worker.
ExecuteTask,
// Reconstruct or fetch possibly lost objects. This is sent from a worker to
// a local scheduler.
ReconstructObjects,
// For a worker that was blocked on some object(s), tell the local scheduler
// that the worker is now unblocked. This is sent from a worker to a local
// scheduler.
NotifyUnblocked,
// Add a result table entry for an object put.
PutObject,
// A request to get the task frontier for an actor, called by the actor when
// saving a checkpoint.
GetActorFrontierRequest,
// The ActorFrontier response to a GetActorFrontierRequest. The local
// scheduler returns the actor's per-handle task counts and execution
// dependencies, which can later be used as the argument to SetActorFrontier
// when resuming from the checkpoint.
GetActorFrontierReply,
// A request to set the task frontier for an actor, called when resuming from
// a checkpoint. The local scheduler will update the actor's per-handle task
// counts and execution dependencies, discard any tasks that already executed
// before the checkpoint, and make any tasks on the frontier runnable by
// making their execution dependencies available.
SetActorFrontier
}
table SubmitTaskRequest {
execution_dependencies: [string];
task_spec: string;
}
// This message is sent from the local scheduler to a worker.
table GetTaskReply {
// A string of bytes representing the task specification.
task_spec: string;
// The IDs of the GPUs that the worker is allowed to use for this task.
gpu_ids: [int];
}
table EventLogMessage {
key: string;
value: string;
timestamp: double;
}
// This struct is used to register a new worker with the local scheduler.
// It is shipped as part of local_scheduler_connect.
table RegisterClientRequest {
// True if the client is a worker and false if the client is a driver.
is_worker: bool;
// The ID of the worker or driver.
client_id: string;
// The process ID of this worker.
worker_pid: long;
// The driver ID. This is non-nil if the client is a driver.
driver_id: string;
}
table DisconnectClient {
}
table ReconstructObjects {
// List of object IDs of the objects that we want to reconstruct or fetch.
object_ids: [string];
// Do we only want to fetch the objects or also reconstruct them?
fetch_only: bool;
}
table PutObject {
// Task ID of the task that performed the put.
task_id: string;
// Object ID of the object that is being put.
object_id: string;
}
// The ActorFrontier is used to represent the current frontier of tasks that
// the local scheduler has marked as runnable for a particular actor. It is
// used to save the point in an actor's lifetime at which a checkpoint was
// taken, so that the same frontier of tasks can be made runnable again if the
// actor is resumed from that checkpoint.
table ActorFrontier {
// Actor ID of the actor whose frontier is described.
actor_id: string;
// A list of handle IDs, representing the callers of the actor that have
// submitted a runnable task to the local scheduler. A nil ID represents the
// creator of the actor.
handle_ids: [string];
// A list representing the number of tasks executed so far, per handle. Each
// count in task_counters corresponds to the handle at the same in index in
// handle_ids.
task_counters: [long];
// A list representing the execution dependency for the next runnable task,
// per handle. Each execution dependency in frontier_dependencies corresponds
// to the handle at the same in index in handle_ids.
frontier_dependencies: [string];
}
table GetActorFrontierRequest {
actor_id: string;
}
src/local_scheduler/local_scheduler.cc
-1555
View File
File diff suppressed because it is too large Load Diff
src/local_scheduler/local_scheduler.h
-176
View File
@@ -1,176 +0,0 @@
#ifndef LOCAL_SCHEDULER_H
#define LOCAL_SCHEDULER_H
#include "event_loop.h"
#include "local_scheduler_shared.h"
#include "task.h"
/**
* Establish a connection to a new client.
*
* @param loop Event loop of the local scheduler.
* @param listener_socket Socket the local scheduler is listening on for new
* client requests.
* @param context State of the local scheduler.
* @param events Flag for events that are available on the listener socket.
* @return Void.
*/
void new_client_connection(event_loop *loop,
int listener_sock,
void *context,
int events);
/**
* Check if a driver is still alive.
*
* @param driver_id The ID of the driver.
* @return True if the driver is still alive and false otherwise.
*/
bool is_driver_alive(WorkerID driver_id);
/**
* This function can be called by the scheduling algorithm to assign a task
* to a worker.
*
* @param info
* @param task The task that is submitted to the worker.
* @param worker The worker to assign the task to.
* @return Void.
*/
void assign_task_to_worker(LocalSchedulerState *state,
TaskExecutionSpec &task,
LocalSchedulerClient *worker);
/*
* This function is called whenever a task has finished on one of the workers.
* It updates the resource accounting and the global state store.
*
* @param state The local scheduler state.
* @param worker The worker that finished the task.
* @return Void.
*/
void finish_task(LocalSchedulerState *state, LocalSchedulerClient *worker);
/**
* This is the callback that is used to process a notification from the Plasma
* store that an object has been sealed.
*
* @param loop The local scheduler's event loop.
* @param client_sock The file descriptor to read the notification from.
* @param context The local scheduler state.
* @param events
* @return Void.
*/
void process_plasma_notification(event_loop *loop,
int client_sock,
void *context,
int events);
/**
* Reconstruct an object. If the object does not exist on any nodes, according
* to the state tables, and if the object is not already being reconstructed,
* this triggers a single reexecution of the task that originally created the
* object.
*
* @param state The local scheduler state.
* @param object_id The ID of the object to reconstruct.
* @return Void.
*/
void reconstruct_object(LocalSchedulerState *state, ObjectID object_id);
void print_resource_info(const LocalSchedulerState *s, const TaskSpec *spec);
/**
* Kill a worker, if it is a child process, and clean up all of its associated
* state. Note that this function is also called on drivers, but it should not
* actually send a kill signal to drivers.
*
* @param state The local scheduler state.
* @param worker The local scheduler client to kill.
* @param wait A boolean representing whether to wait for the killed worker to
* exit.
* @param suppress_warning A bool that is true if we should not warn the driver,
* and false otherwise. This should only be true when a driver is
* removed.
* @return Void.
*/
void kill_worker(LocalSchedulerState *state,
LocalSchedulerClient *worker,
bool wait,
bool suppress_warning);
/**
* Start a worker. This forks a new worker process that can be added to the
* pool of available workers, pending registration of its PID with the local
* scheduler.
*
* @param state The local scheduler state.
* @param Void.
*/
void start_worker(LocalSchedulerState *state);
/**
* Check if a certain quantity of dynamic resources are available. If num_cpus
* is 0, we ignore the dynamic number of available CPUs (which may be negative).
*
* @param state The state of the local scheduler.
* @param resources The resources to check.
* @return True if there are enough CPUs and GPUs and false otherwise.
*/
bool check_dynamic_resources(
LocalSchedulerState *state,
const std::unordered_map<std::string, double> &resources);
/**
* Acquire additional resources (CPUs and GPUs) for a worker.
*
* @param state The local scheduler state.
* @param worker The worker who is acquiring resources.
* @param resources The resources to acquire.
* @return Void.
*/
void acquire_resources(
LocalSchedulerState *state,
LocalSchedulerClient *worker,
const std::unordered_map<std::string, double> &resources);
/**
* Return resources (CPUs and GPUs) being used by a worker to the local
* scheduler.
*
* @param state The local scheduler state.
* @param worker The worker who is returning resources.
* @param resources The resources to release.
* @return Void.
*/
void release_resources(
LocalSchedulerState *state,
LocalSchedulerClient *worker,
const std::unordered_map<std::string, double> &resources);
/** The following methods are for testing purposes only. */
#ifdef LOCAL_SCHEDULER_TEST
LocalSchedulerState *LocalSchedulerState_init(
const char *node_ip_address,
event_loop *loop,
const char *redis_addr,
int redis_port,
const char *local_scheduler_socket_name,
const char *plasma_manager_socket_name,
const char *plasma_store_socket_name,
const char *plasma_manager_address,
bool global_scheduler_exists,
const std::unordered_map<std::string, double> &static_resource_vector,
const char *worker_path,
int num_workers);
SchedulingAlgorithmState *get_algorithm_state(LocalSchedulerState *state);
void process_message(event_loop *loop,
int client_sock,
void *context,
int events);
#endif
#endif /* LOCAL_SCHEDULER_H */
src/local_scheduler/local_scheduler_algorithm.cc
-1851
View File
File diff suppressed because it is too large Load Diff
src/local_scheduler/local_scheduler_algorithm.h
-438
View File
@@ -1,438 +0,0 @@
#ifndef LOCAL_SCHEDULER_ALGORITHM_H
#define LOCAL_SCHEDULER_ALGORITHM_H
#include "local_scheduler_shared.h"
#include "common/task.h"
#include "state/local_scheduler_table.h"
/* ==== The scheduling algorithm ====
*
* This file contains declaration for all functions and data structures
* that need to be provided if you want to implement a new algorithms
* for the local scheduler.
*
*/
/**
* Initialize the scheduler state.
*
* @return State managed by the scheduling algorithm.
*/
SchedulingAlgorithmState *SchedulingAlgorithmState_init(void);
/**
* Free the scheduler state.
*
* @param algorithm_state State maintained by the scheduling algorithm.
* @return Void.
*/
void SchedulingAlgorithmState_free(SchedulingAlgorithmState *algorithm_state);
/**
*
*/
void provide_scheduler_info(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerInfo *info);
/**
* This function will be called when a new task is submitted by a worker for
* execution. The task will either be:
* 1. Put into the waiting queue, where it will wait for its dependencies to
* become available.
* 2. Put into the dispatch queue, where it will wait for an available worker.
* 3. Given to the global scheduler to be scheduled.
*
* Currently, the local scheduler policy is to keep the task if its
* dependencies are ready and there is an available worker.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param task Task that is submitted by the worker.
* @return Void.
*/
void handle_task_submitted(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
TaskExecutionSpec &execution_spec);
/**
* This version of handle_task_submitted is used when the task being submitted
* is a method of an actor.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param task Task that is submitted by the worker.
* @return Void.
*/
void handle_actor_task_submitted(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
TaskExecutionSpec &execution_spec);
/**
* This function will be called when the local scheduler receives a notification
* about the creation of a new actor. This can be used by the scheduling
* algorithm to resubmit cached actor tasks.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param actor_id The ID of the actor being created.
* @return Void.
*/
void handle_actor_creation_notification(
LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
ActorID actor_id);
/**
* This function will be called when a task is assigned by the global scheduler
* for execution on this local scheduler.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param task Task that is assigned by the global scheduler.
* @return Void.
*/
void handle_task_scheduled(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
TaskExecutionSpec &execution_spec);
/**
* This function will be called when an actor task is assigned by the global
* scheduler or by another local scheduler for execution on this local
* scheduler.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param task Task that is assigned by the global scheduler.
* @return Void.
*/
void handle_actor_task_scheduled(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
TaskExecutionSpec &execution_spec);
/**
* This function is called if a new object becomes available in the local
* plasma store.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param object_id ID of the object that became available.
* @return Void.
*/
void handle_object_available(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
ObjectID object_id);
/**
* This function is called if an object is removed from the local plasma store.
*
* @param state The state of the local scheduler.
* @param object_id ID of the object that was removed.
* @return Void.
*/
void handle_object_removed(LocalSchedulerState *state, ObjectID object_id);
/**
* This function is called when a new worker becomes available.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param worker The worker that is available.
* @return Void.
*/
void handle_worker_available(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker);
/**
* This function is called when a worker is removed.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param worker The worker that is removed.
* @return Void.
*/
void handle_worker_removed(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker);
/**
* This version of handle_worker_available is called whenever the worker that is
* available is running an actor.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param worker The worker that is available.
* @return Void.
*/
void handle_actor_worker_available(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker);
/**
* Handle the fact that a new worker is available for running an actor.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param actor_id The ID of the actor running on the worker.
* @param initial_execution_dependency The dummy object ID of the actor
* creation task.
* @param worker The worker that was converted to an actor.
* @return Void.
*/
void handle_convert_worker_to_actor(
LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
const ActorID &actor_id,
const ObjectID &initial_execution_dependency,
LocalSchedulerClient *worker);
/**
* Handle the fact that a worker running an actor has disconnected.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param worker The worker that was disconnected.
* @param cleanup Whether the disconnect was during cleanup.
* @return Void.
*/
void handle_actor_worker_disconnect(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker,
bool cleanup);
/**
* This function is called when a worker that was executing a task becomes
* blocked on an object that isn't available locally yet.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param worker The worker that is blocked.
* @return Void.
*/
void handle_worker_blocked(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker);
/**
* This function is called when an actor that was executing a task becomes
* blocked on an object that isn't available locally yet.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param worker The worker that is blocked.
* @return Void.
*/
void handle_actor_worker_blocked(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker);
/**
* This function is called when a worker that was blocked on an object that
* wasn't available locally yet becomes unblocked.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param worker The worker that is now unblocked.
* @return Void.
*/
void handle_worker_unblocked(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker);
/**
* This function is called when an actor that was blocked on an object that
* wasn't available locally yet becomes unblocked.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param worker The worker that is now unblocked.
* @return Void.
*/
void handle_actor_worker_unblocked(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker);
/**
* Process the fact that a driver has been removed. This will remove all of the
* tasks for that driver from the scheduling algorithm's internal data
* structures.
*
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param driver_id The ID of the driver that was removed.
* @return Void.
*/
void handle_driver_removed(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
WorkerID driver_id);
/**
* This function fetches queued task's missing object dependencies. It is
* called every local_scheduler_fetch_timeout_milliseconds.
*
* @param loop The local scheduler's event loop.
* @param id The ID of the timer that triggers this function.
* @param context The function's context.
* @return An integer representing the time interval in seconds before the
* next invocation of the function.
*/
int fetch_object_timeout_handler(event_loop *loop, timer_id id, void *context);
/**
* This function initiates reconstruction for task's missing object
* dependencies. It is called every
* local_scheduler_reconstruction_timeout_milliseconds, but it may not initiate
* reconstruction for every missing object.
*
* @param loop The local scheduler's event loop.
* @param id The ID of the timer that triggers this function.
* @param context The function's context.
* @return An integer representing the time interval in seconds before the
* next invocation of the function.
*/
int reconstruct_object_timeout_handler(event_loop *loop,
timer_id id,
void *context);
/// This function initiates reconstruction for the actor creation tasks
/// corresponding to the actor tasks cached in the local scheduler.
///
/// \param loop The local scheduler's event loop.
/// \param id The ID of the timer that triggers this function.
/// \param context The function's context.
/// \return An integer representing the time interval in seconds before the
/// next invocation of the function.
int rerun_actor_creation_tasks_timeout_handler(event_loop *loop,
timer_id id,
void *context);
/**
* Check whether an object, including actor dummy objects, is locally
* available.
*
* @param algorithm_state State maintained by the scheduling algorithm.
* @param object_id The ID of the object to check for.
* @return A bool representing whether the object is locally available.
*/
bool object_locally_available(SchedulingAlgorithmState *algorithm_state,
ObjectID object_id);
/// Spill some tasks back to the global scheduler. This function implements the
/// spillback policy.
///
/// @param state The scheduler state.
/// @return Void.
void spillback_tasks_handler(LocalSchedulerState *state);
/**
* A helper function to print debug information about the current state and
* number of workers.
*
* @param message A message to identify the log message.
* @param algorithm_state State maintained by the scheduling algorithm.
* @return Void.
*/
void print_worker_info(const char *message,
SchedulingAlgorithmState *algorithm_state);
/*
* The actor frontier consists of the number of tasks executed so far and the
* execution dependencies required by the current runnable tasks, according to
* the actor's local scheduler. Since an actor may have multiple handles, the
* tasks submitted to the actor form a DAG, where nodes are tasks and edges are
* execution dependencies. The frontier is a cut across this DAG. The number of
* tasks so far is the number of nodes included in the DAG root's partition.
*
* The actor gets the current frontier of tasks from the local scheduler during
* a checkpoint save, so that it can save the point in the actor's lifetime at
* which the checkpoint was taken. If the actor later resumes from that
* checkpoint, the actor can set the current frontier of tasks in the local
* scheduler so that the same frontier of tasks can be made runnable again
* during reconstruction, and so that we do not duplicate execution of tasks
* that already executed before the checkpoint.
*/
/**
* Get the number of tasks, per actor handle, that have been executed on an
* actor so far.
*
* @param algorithm_state State maintained by the scheduling algorithm.
* @param actor_id The ID of the actor whose task counters are returned.
* @return A map from handle ID to the number of tasks submitted by that handle
* that have executed so far.
*/
std::unordered_map<ActorHandleID, int64_t> get_actor_task_counters(
SchedulingAlgorithmState *algorithm_state,
ActorID actor_id);
/**
* Set the number of tasks, per actor handle, that have been executed on an
* actor so far. All previous counts will be overwritten. Tasks that are
* waiting or runnable on the local scheduler that have a lower task count will
* be discarded, so that we don't duplicate execution.
*
* @param algorithm_state State maintained by the scheduling algorithm.
* @param actor_id The ID of the actor whose task counters are returned.
* @param task_counters A map from handle ID to the number of tasks submitted
* by that handle that have executed so far.
* @return Void.
*/
void set_actor_task_counters(
SchedulingAlgorithmState *algorithm_state,
ActorID actor_id,
const std::unordered_map<ActorHandleID, int64_t> &task_counters);
/**
* Get the actor's frontier of task dependencies.
* NOTE(swang): The returned frontier only includes handles known by the local
* scheduler. It does not include handles for which the local scheduler has not
* seen a runnable task yet.
*
* @param algorithm_state State maintained by the scheduling algorithm.
* @param actor_id The ID of the actor whose task counters are returned.
* @return A map from handle ID to execution dependency for the earliest
* runnable task submitted through that handle.
*/
std::unordered_map<ActorHandleID, ObjectID> get_actor_frontier(
SchedulingAlgorithmState *algorithm_state,
ActorID actor_id);
/**
* Set the actor's frontier of task dependencies. The previous frontier will be
* overwritten. Any tasks that have an execution dependency on the new frontier
* (and that have all other dependencies fulfilled) will become runnable.
*
* @param algorithm_state State maintained by the scheduling algorithm.
* @param actor_id The ID of the actor whose task counters are returned.
* @param frontier_dependencies A map from handle ID to execution dependency
* for the earliest runnable task submitted through that handle.
* @return Void.
*/
void set_actor_frontier(
LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
ActorID actor_id,
const std::unordered_map<ActorHandleID, ObjectID> &frontier_dependencies);
/** The following methods are for testing purposes only. */
#ifdef LOCAL_SCHEDULER_TEST
/**
* Get the number of tasks currently waiting for object dependencies to become
* available locally.
*
* @param algorithm_state State maintained by the scheduling algorithm.
* @return The number of tasks queued.
*/
int num_waiting_tasks(SchedulingAlgorithmState *algorithm_state);
/**
* Get the number of tasks currently waiting for a worker to become available.
*
* @param algorithm_state State maintained by the scheduling algorithm.
* @return The number of tasks queued.
*/
int num_dispatch_tasks(SchedulingAlgorithmState *algorithm_state);
#endif
#endif /* LOCAL_SCHEDULER_ALGORITHM_H */
src/local_scheduler/local_scheduler_client.cc
-385
View File
@@ -1,385 +0,0 @@
#include "local_scheduler_client.h"
#include "common_protocol.h"
#include "format/local_scheduler_generated.h"
#include "ray/raylet/format/node_manager_generated.h"
#include "common/io.h"
#include "common/task.h"
#include <stdlib.h>
#include <sys/types.h>
#include <unistd.h>
using MessageType = ray::local_scheduler::protocol::MessageType;
LocalSchedulerConnection *LocalSchedulerConnection_init(
const char *local_scheduler_socket,
const UniqueID &client_id,
bool is_worker,
const JobID &driver_id,
bool use_raylet,
const Language &language) {
LocalSchedulerConnection *result = new LocalSchedulerConnection();
result->use_raylet = use_raylet;
result->conn = connect_ipc_sock_retry(local_scheduler_socket, -1, -1);
/* Register with the local scheduler.
* NOTE(swang): If the local scheduler exits and we are registered as a
* worker, we will get killed. */
flatbuffers::FlatBufferBuilder fbb;
if (use_raylet) {
auto message = ray::protocol::CreateRegisterClientRequest(
fbb, is_worker, to_flatbuf(fbb, client_id), getpid(),
to_flatbuf(fbb, driver_id), language);
fbb.Finish(message);
} else {
auto message = ray::local_scheduler::protocol::CreateRegisterClientRequest(
fbb, is_worker, to_flatbuf(fbb, client_id), getpid(),
to_flatbuf(fbb, driver_id));
fbb.Finish(message);
}
/* Register the process ID with the local scheduler. */
int success = write_message(
result->conn, static_cast<int64_t>(MessageType::RegisterClientRequest),
fbb.GetSize(), fbb.GetBufferPointer(), &result->write_mutex);
RAY_CHECK(success == 0) << "Unable to register worker with local scheduler";
return result;
}
void LocalSchedulerConnection_free(LocalSchedulerConnection *conn) {
close(conn->conn);
delete conn;
}
void local_scheduler_disconnect_client(LocalSchedulerConnection *conn) {
flatbuffers::FlatBufferBuilder fbb;
auto message = ray::local_scheduler::protocol::CreateDisconnectClient(fbb);
fbb.Finish(message);
if (conn->use_raylet) {
write_message(conn->conn, static_cast<int64_t>(
MessageType::IntentionalDisconnectClient),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
} else {
write_message(conn->conn,
static_cast<int64_t>(MessageType::DisconnectClient),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
}
}
void local_scheduler_log_event(LocalSchedulerConnection *conn,
uint8_t *key,
int64_t key_length,
uint8_t *value,
int64_t value_length,
double timestamp) {
flatbuffers::FlatBufferBuilder fbb;
auto key_string = fbb.CreateString((char *) key, key_length);
auto value_string = fbb.CreateString((char *) value, value_length);
auto message = ray::local_scheduler::protocol::CreateEventLogMessage(
fbb, key_string, value_string, timestamp);
fbb.Finish(message);
write_message(conn->conn, static_cast<int64_t>(MessageType::EventLogMessage),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
}
void local_scheduler_submit(LocalSchedulerConnection *conn,
const TaskExecutionSpec &execution_spec) {
flatbuffers::FlatBufferBuilder fbb;
auto execution_dependencies =
to_flatbuf(fbb, execution_spec.ExecutionDependencies());
auto task_spec =
fbb.CreateString(reinterpret_cast<char *>(execution_spec.Spec()),
execution_spec.SpecSize());
auto message = ray::local_scheduler::protocol::CreateSubmitTaskRequest(
fbb, execution_dependencies, task_spec);
fbb.Finish(message);
write_message(conn->conn, static_cast<int64_t>(MessageType::SubmitTask),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
}
void local_scheduler_submit_raylet(
LocalSchedulerConnection *conn,
const std::vector<ObjectID> &execution_dependencies,
const ray::raylet::TaskSpecification &task_spec) {
flatbuffers::FlatBufferBuilder fbb;
auto execution_dependencies_message = to_flatbuf(fbb, execution_dependencies);
auto message = ray::local_scheduler::protocol::CreateSubmitTaskRequest(
fbb, execution_dependencies_message, task_spec.ToFlatbuffer(fbb));
fbb.Finish(message);
write_message(conn->conn, static_cast<int64_t>(MessageType::SubmitTask),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
}
TaskSpec *local_scheduler_get_task(LocalSchedulerConnection *conn,
int64_t *task_size) {
int64_t type;
int64_t reply_size;
uint8_t *reply;
{
std::unique_lock<std::mutex> guard(conn->mutex);
write_message(conn->conn, static_cast<int64_t>(MessageType::GetTask), 0,
NULL, &conn->write_mutex);
/* Receive a task from the local scheduler. This will block until the local
* scheduler gives this client a task. */
read_message(conn->conn, &type, &reply_size, &reply);
}
if (type == static_cast<int64_t>(CommonMessageType::DISCONNECT_CLIENT)) {
RAY_LOG(DEBUG) << "Exiting because local scheduler closed connection.";
exit(1);
}
RAY_CHECK(static_cast<MessageType>(type) == MessageType::ExecuteTask);
/* Parse the flatbuffer object. */
auto reply_message =
flatbuffers::GetRoot<ray::local_scheduler::protocol::GetTaskReply>(reply);
/* Create a copy of the task spec so we can free the reply. */
*task_size = reply_message->task_spec()->size();
TaskSpec *data = (TaskSpec *) reply_message->task_spec()->data();
TaskSpec *spec = TaskSpec_copy(data, *task_size);
// Set the GPU IDs for this task. We only do this for non-actor tasks because
// for actors the GPUs are associated with the actor itself and not with the
// actor methods. Note that this also processes GPUs for actor creation tasks.
if (!TaskSpec_is_actor_task(spec)) {
conn->gpu_ids.clear();
for (size_t i = 0; i < reply_message->gpu_ids()->size(); ++i) {
conn->gpu_ids.push_back(reply_message->gpu_ids()->Get(i));
}
}
/* Free the original message from the local scheduler. */
free(reply);
/* Return the copy of the task spec and pass ownership to the caller. */
return spec;
}
// This is temporarily duplicated from local_scheduler_get_task while we have
// the raylet and non-raylet code paths.
TaskSpec *local_scheduler_get_task_raylet(LocalSchedulerConnection *conn,
int64_t *task_size) {
int64_t type;
int64_t reply_size;
uint8_t *reply;
{
std::unique_lock<std::mutex> guard(conn->mutex);
write_message(conn->conn, static_cast<int64_t>(MessageType::GetTask), 0,
NULL, &conn->write_mutex);
// Receive a task from the local scheduler. This will block until the local
// scheduler gives this client a task.
read_message(conn->conn, &type, &reply_size, &reply);
}
if (type == static_cast<int64_t>(CommonMessageType::DISCONNECT_CLIENT)) {
RAY_LOG(DEBUG) << "Exiting because local scheduler closed connection.";
exit(1);
}
RAY_CHECK(type == static_cast<int64_t>(MessageType::ExecuteTask));
// Parse the flatbuffer object.
auto reply_message = flatbuffers::GetRoot<ray::protocol::GetTaskReply>(reply);
// Create a copy of the task spec so we can free the reply.
*task_size = reply_message->task_spec()->size();
const TaskSpec *data =
reinterpret_cast<const TaskSpec *>(reply_message->task_spec()->data());
TaskSpec *spec = TaskSpec_copy(const_cast<TaskSpec *>(data), *task_size);
// Set the resource IDs for this task.
conn->resource_ids_.clear();
for (size_t i = 0; i < reply_message->fractional_resource_ids()->size();
++i) {
auto const &fractional_resource_ids =
reply_message->fractional_resource_ids()->Get(i);
auto &acquired_resources = conn->resource_ids_[string_from_flatbuf(
*fractional_resource_ids->resource_name())];
size_t num_resource_ids = fractional_resource_ids->resource_ids()->size();
size_t num_resource_fractions =
fractional_resource_ids->resource_fractions()->size();
RAY_CHECK(num_resource_ids == num_resource_fractions);
RAY_CHECK(num_resource_ids > 0);
for (size_t j = 0; j < num_resource_ids; ++j) {
int64_t resource_id = fractional_resource_ids->resource_ids()->Get(j);
double resource_fraction =
fractional_resource_ids->resource_fractions()->Get(j);
if (num_resource_ids > 1) {
int64_t whole_fraction = resource_fraction;
RAY_CHECK(whole_fraction == resource_fraction);
}
acquired_resources.push_back(
std::make_pair(resource_id, resource_fraction));
}
}
// Free the original message from the local scheduler.
free(reply);
// Return the copy of the task spec and pass ownership to the caller.
return spec;
}
void local_scheduler_task_done(LocalSchedulerConnection *conn) {
write_message(conn->conn, static_cast<int64_t>(MessageType::TaskDone), 0,
NULL, &conn->write_mutex);
}
void local_scheduler_reconstruct_objects(
LocalSchedulerConnection *conn,
const std::vector<ObjectID> &object_ids,
bool fetch_only) {
flatbuffers::FlatBufferBuilder fbb;
auto object_ids_message = to_flatbuf(fbb, object_ids);
auto message = ray::local_scheduler::protocol::CreateReconstructObjects(
fbb, object_ids_message, fetch_only);
fbb.Finish(message);
write_message(conn->conn,
static_cast<int64_t>(MessageType::ReconstructObjects),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
/* TODO(swang): Propagate the error. */
}
void local_scheduler_log_message(LocalSchedulerConnection *conn) {
write_message(conn->conn, static_cast<int64_t>(MessageType::EventLogMessage),
0, NULL, &conn->write_mutex);
}
void local_scheduler_notify_unblocked(LocalSchedulerConnection *conn) {
write_message(conn->conn, static_cast<int64_t>(MessageType::NotifyUnblocked),
0, NULL, &conn->write_mutex);
}
void local_scheduler_put_object(LocalSchedulerConnection *conn,
TaskID task_id,
ObjectID object_id) {
flatbuffers::FlatBufferBuilder fbb;
auto message = ray::local_scheduler::protocol::CreatePutObject(
fbb, to_flatbuf(fbb, task_id), to_flatbuf(fbb, object_id));
fbb.Finish(message);
write_message(conn->conn, static_cast<int64_t>(MessageType::PutObject),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
}
const std::vector<uint8_t> local_scheduler_get_actor_frontier(
LocalSchedulerConnection *conn,
ActorID actor_id) {
flatbuffers::FlatBufferBuilder fbb;
auto message = ray::local_scheduler::protocol::CreateGetActorFrontierRequest(
fbb, to_flatbuf(fbb, actor_id));
fbb.Finish(message);
int64_t type;
std::vector<uint8_t> reply;
{
std::unique_lock<std::mutex> guard(conn->mutex);
write_message(conn->conn,
static_cast<int64_t>(MessageType::GetActorFrontierRequest),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
read_vector(conn->conn, &type, reply);
}
if (static_cast<CommonMessageType>(type) ==
CommonMessageType::DISCONNECT_CLIENT) {
RAY_LOG(DEBUG) << "Exiting because local scheduler closed connection.";
exit(1);
}
RAY_CHECK(static_cast<MessageType>(type) ==
MessageType::GetActorFrontierReply);
return reply;
}
void local_scheduler_set_actor_frontier(LocalSchedulerConnection *conn,
const std::vector<uint8_t> &frontier) {
write_message(conn->conn, static_cast<int64_t>(MessageType::SetActorFrontier),
frontier.size(), const_cast<uint8_t *>(frontier.data()),
&conn->write_mutex);
}
std::pair<std::vector<ObjectID>, std::vector<ObjectID>> local_scheduler_wait(
LocalSchedulerConnection *conn,
const std::vector<ObjectID> &object_ids,
int num_returns,
int64_t timeout_milliseconds,
bool wait_local) {
// Write request.
flatbuffers::FlatBufferBuilder fbb;
auto message = ray::protocol::CreateWaitRequest(
fbb, to_flatbuf(fbb, object_ids), num_returns, timeout_milliseconds,
wait_local);
fbb.Finish(message);
int64_t type;
int64_t reply_size;
uint8_t *reply;
{
std::unique_lock<std::mutex> guard(conn->mutex);
write_message(conn->conn,
static_cast<int64_t>(ray::protocol::MessageType::WaitRequest),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
// Read result.
read_message(conn->conn, &type, &reply_size, &reply);
}
RAY_CHECK(static_cast<ray::protocol::MessageType>(type) ==
ray::protocol::MessageType::WaitReply);
auto reply_message = flatbuffers::GetRoot<ray::protocol::WaitReply>(reply);
// Convert result.
std::pair<std::vector<ObjectID>, std::vector<ObjectID>> result;
auto found = reply_message->found();
for (uint i = 0; i < found->size(); i++) {
ObjectID object_id = ObjectID::from_binary(found->Get(i)->str());
result.first.push_back(object_id);
}
auto remaining = reply_message->remaining();
for (uint i = 0; i < remaining->size(); i++) {
ObjectID object_id = ObjectID::from_binary(remaining->Get(i)->str());
result.second.push_back(object_id);
}
/* Free the original message from the local scheduler. */
free(reply);
return result;
}
void local_scheduler_push_error(LocalSchedulerConnection *conn,
const JobID &job_id,
const std::string &type,
const std::string &error_message,
double timestamp) {
flatbuffers::FlatBufferBuilder fbb;
auto message = ray::protocol::CreatePushErrorRequest(
fbb, to_flatbuf(fbb, job_id), fbb.CreateString(type),
fbb.CreateString(error_message), timestamp);
fbb.Finish(message);
write_message(conn->conn, static_cast<int64_t>(
ray::protocol::MessageType::PushErrorRequest),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
}
void local_scheduler_push_profile_events(
LocalSchedulerConnection *conn,
const ProfileTableDataT &profile_events) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreateProfileTableData(fbb, &profile_events);
fbb.Finish(message);
write_message(conn->conn,
static_cast<int64_t>(
ray::protocol::MessageType::PushProfileEventsRequest),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
}
void local_scheduler_free_objects_in_object_store(
LocalSchedulerConnection *conn,
const std::vector<ray::ObjectID> &object_ids,
bool local_only) {
flatbuffers::FlatBufferBuilder fbb;
auto message = ray::protocol::CreateFreeObjectsRequest(
fbb, local_only, to_flatbuf(fbb, object_ids));
fbb.Finish(message);
int success = write_message(
conn->conn,
static_cast<int64_t>(
ray::protocol::MessageType::FreeObjectsInObjectStoreRequest),
fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex);
RAY_CHECK(success == 0) << "Failed to write message to raylet.";
}
src/local_scheduler/local_scheduler_client.h
-260
View File
@@ -1,260 +0,0 @@
#ifndef LOCAL_SCHEDULER_CLIENT_H
#define LOCAL_SCHEDULER_CLIENT_H
#include <mutex>
#include "common/task.h"
#include "local_scheduler_shared.h"
#include "ray/raylet/task_spec.h"
struct LocalSchedulerConnection {
/// True if we should use the raylet code path and false otherwise.
bool use_raylet;
/** File descriptor of the Unix domain socket that connects to local
* scheduler. */
int conn;
/** The IDs of the GPUs that this client can use. NOTE(rkn): This is only used
* by legacy Ray and will be deprecated. */
std::vector<int> gpu_ids;
/// A map from resource name to the resource IDs that are currently reserved
/// for this worker. Each pair consists of the resource ID and the fraction
/// of that resource allocated for this worker.
std::unordered_map<std::string, std::vector<std::pair<int64_t, double>>>
resource_ids_;
/// A mutex to protect stateful operations of the local scheduler client.
std::mutex mutex;
/// A mutext to protect write operations of the local scheduler client.
std::mutex write_mutex;
};
/**
* Connect to the local scheduler.
*
* @param local_scheduler_socket The name of the socket to use to connect to the
* local scheduler.
* @param worker_id A unique ID to represent the worker.
* @param is_worker Whether this client is a worker. If it is a worker, an
* additional message will be sent to register as one.
* @param driver_id The ID of the driver. This is non-nil if the client is a
* driver.
* @param use_raylet True if we should use the raylet code path and false
* otherwise.
* @return The connection information.
*/
LocalSchedulerConnection *LocalSchedulerConnection_init(
const char *local_scheduler_socket,
const UniqueID &worker_id,
bool is_worker,
const JobID &driver_id,
bool use_raylet,
const Language &language);
/**
* Disconnect from the local scheduler.
*
* @param conn Local scheduler connection information returned by
* LocalSchedulerConnection_init.
* @return Void.
*/
void LocalSchedulerConnection_free(LocalSchedulerConnection *conn);
/**
* Submit a task to the local scheduler.
*
* @param conn The connection information.
* @param execution_spec The execution spec for the task to submit.
* @return Void.
*/
void local_scheduler_submit(LocalSchedulerConnection *conn,
const TaskExecutionSpec &execution_spec);
/// Submit a task using the raylet code path.
///
/// \param The connection information.
/// \param The execution dependencies.
/// \param The task specification.
/// \return Void.
void local_scheduler_submit_raylet(
LocalSchedulerConnection *conn,
const std::vector<ObjectID> &execution_dependencies,
const ray::raylet::TaskSpecification &task_spec);
/**
* Notify the local scheduler that this client is disconnecting gracefully. This
* is used by actors to exit gracefully so that the local scheduler doesn't
* propagate an error message to the driver.
*
* @param conn The connection information.
* @return Void.
*/
void local_scheduler_disconnect_client(LocalSchedulerConnection *conn);
/**
* Log an event to the event log. This will call RPUSH key value. We use RPUSH
* instead of SET so that it is possible to flush the log multiple times with
* the same key (for example the key might be shared across logging calls in the
* same task on a worker).
*
* @param conn The connection information.
* @param key The key to store the event in.
* @param key_length The length of the key.
* @param value The value to store.
* @param value_length The length of the value.
* @param timestamp The time that the event is logged.
* @return Void.
*/
void local_scheduler_log_event(LocalSchedulerConnection *conn,
uint8_t *key,
int64_t key_length,
uint8_t *value,
int64_t value_length,
double timestamp);
/**
* Get next task for this client. This will block until the scheduler assigns
* a task to this worker. This allocates and returns a task, and so the task
* must be freed by the caller.
*
* @todo When does this actually get freed?
*
* @param conn The connection information.
* @param task_size A pointer to fill out with the task size.
* @return The address of the assigned task.
*/
TaskSpec *local_scheduler_get_task(LocalSchedulerConnection *conn,
int64_t *task_size);
/// Get next task for this client. This will block until the scheduler assigns
/// a task to this worker. This allocates and returns a task, and so the task
/// must be freed by the caller.
///
/// \param conn The connection information.
/// \param task_size A pointer to fill out with the task size.
/// \return The address of the assigned task.
TaskSpec *local_scheduler_get_task_raylet(LocalSchedulerConnection *conn,
int64_t *task_size);
/**
* Tell the local scheduler that the client has finished executing a task.
*
* @param conn The connection information.
* @return Void.
*/
void local_scheduler_task_done(LocalSchedulerConnection *conn);
/**
* Tell the local scheduler to reconstruct or fetch objects.
*
* @param conn The connection information.
* @param object_ids The IDs of the objects to reconstruct.
* @param fetch_only Only fetch objects, do not reconstruct them.
* @return Void.
*/
void local_scheduler_reconstruct_objects(
LocalSchedulerConnection *conn,
const std::vector<ObjectID> &object_ids,
bool fetch_only = false);
/**
* Send a log message to the local scheduler.
*
* @param conn The connection information.
* @return Void.
*/
void local_scheduler_log_message(LocalSchedulerConnection *conn);
/**
* Notify the local scheduler that this client (worker) is no longer blocked.
*
* @param conn The connection information.
* @return Void.
*/
void local_scheduler_notify_unblocked(LocalSchedulerConnection *conn);
/**
* Record the mapping from object ID to task ID for put events.
*
* @param conn The connection information.
* @param task_id The ID of the task that called put.
* @param object_id The ID of the object being stored.
* @return Void.
*/
void local_scheduler_put_object(LocalSchedulerConnection *conn,
TaskID task_id,
ObjectID object_id);
/**
* Get an actor's current task frontier.
*
* @param conn The connection information.
* @param actor_id The ID of the actor whose frontier is returned.
* @return A byte vector that can be traversed as an ActorFrontier flatbuffer.
*/
const std::vector<uint8_t> local_scheduler_get_actor_frontier(
LocalSchedulerConnection *conn,
ActorID actor_id);
/**
* Set an actor's current task frontier.
*
* @param conn The connection information.
* @param frontier An ActorFrontier flatbuffer to set the frontier to.
* @return Void.
*/
void local_scheduler_set_actor_frontier(LocalSchedulerConnection *conn,
const std::vector<uint8_t> &frontier);
/// Wait for the given objects until timeout expires or num_return objects are
/// found.
///
/// \param conn The connection information.
/// \param object_ids The objects to wait for.
/// \param num_returns The number of objects to wait for.
/// \param timeout_milliseconds Duration, in milliseconds, to wait before
/// returning.
/// \param wait_local Whether to wait for objects to appear on this node.
/// \return A pair with the first element containing the object ids that were
/// found, and the second element the objects that were not found.
std::pair<std::vector<ObjectID>, std::vector<ObjectID>> local_scheduler_wait(
LocalSchedulerConnection *conn,
const std::vector<ObjectID> &object_ids,
int num_returns,
int64_t timeout_milliseconds,
bool wait_local);
/// Push an error to the relevant driver.
///
/// \param conn The connection information.
/// \param The ID of the job that the error is for.
/// \param The type of the error.
/// \param The error message.
/// \param The timestamp of the error.
/// \return Void.
void local_scheduler_push_error(LocalSchedulerConnection *conn,
const JobID &job_id,
const std::string &type,
const std::string &error_message,
double timestamp);
/// Store some profile events in the GCS.
///
/// \param conn The connection information.
/// \param profile_events A batch of profiling event information.
/// \return Void.
void local_scheduler_push_profile_events(
LocalSchedulerConnection *conn,
const ProfileTableDataT &profile_events);
/// Free a list of objects from object stores.
///
/// \param conn The connection information.
/// \param object_ids A list of ObjectsIDs to be deleted.
/// \param local_only Whether keep this request with local object store
/// or send it to all the object stores.
/// \return Void.
void local_scheduler_free_objects_in_object_store(
LocalSchedulerConnection *conn,
const std::vector<ray::ObjectID> &object_ids,
bool local_only);
#endif
src/local_scheduler/local_scheduler_shared.h
-137
View File
@@ -1,137 +0,0 @@
#ifndef LOCAL_SCHEDULER_SHARED_H
#define LOCAL_SCHEDULER_SHARED_H
#include "common/task.h"
#include "common/state/table.h"
#include "common/state/db.h"
#include "plasma/client.h"
#include "ray/gcs/client.h"
#include <list>
#include <unordered_map>
#include <unordered_set>
#include <vector>
/** This struct is used to maintain a mapping from actor IDs to the ID of the
* local scheduler that is responsible for the actor. */
struct ActorMapEntry {
/** The ID of the driver that created the actor. */
WorkerID driver_id;
/** The ID of the local scheduler that is responsible for the actor. */
DBClientID local_scheduler_id;
};
/** Internal state of the scheduling algorithm. */
typedef struct SchedulingAlgorithmState SchedulingAlgorithmState;
struct LocalSchedulerClient;
/** A struct storing the configuration state of the local scheduler. This should
* consist of values that don't change over the lifetime of the local
* scheduler. */
typedef struct {
/** The script to use when starting a new worker. */
const char **start_worker_command;
/** Whether there is a global scheduler. */
bool global_scheduler_exists;
} local_scheduler_config;
/** The state of the local scheduler. */
struct LocalSchedulerState {
/** The configuration for the local scheduler. */
local_scheduler_config config;
/** The local scheduler event loop. */
event_loop *loop;
/** List of workers available to this node. This is used to free the worker
* structs when we free the scheduler state and also to access the worker
* structs in the tests. */
std::list<LocalSchedulerClient *> workers;
/** A set of driver IDs corresponding to drivers that have been removed. This
* is used to make sure we don't execute any tasks belong to dead drivers. */
std::unordered_set<WorkerID> removed_drivers;
/** A set of actors IDs corresponding to local actors that have been removed.
* This ensures we can reject any tasks destined for dead actors. */
std::unordered_set<ActorID> removed_actors;
/** List of the process IDs for child processes (workers) started by the
* local scheduler that have not sent a REGISTER_PID message yet. */
std::vector<pid_t> child_pids;
/** A hash table mapping actor IDs to the db_client_id of the local scheduler
* that is responsible for the actor. */
std::unordered_map<ActorID, ActorMapEntry> actor_mapping;
/** The handle to the database. */
DBHandle *db;
/** The Plasma client. */
plasma::PlasmaClient *plasma_conn;
/** State for the scheduling algorithm. */
SchedulingAlgorithmState *algorithm_state;
/** Input buffer, used for reading input in process_message to avoid
* allocation for each call to process_message. */
std::vector<uint8_t> input_buffer;
/** Vector of static attributes associated with the node owned by this local
* scheduler. */
std::unordered_map<std::string, double> static_resources;
/** Vector of dynamic attributes associated with the node owned by this local
* scheduler. */
std::unordered_map<std::string, double> dynamic_resources;
/** The IDs of the available GPUs. There is redundancy here in that
* available_gpus.size() == dynamic_resources[ResourceIndex_GPU] should
* always be true. */
std::vector<int> available_gpus;
/** The time (in milliseconds since the Unix epoch) when the most recent
* heartbeat was sent. */
int64_t previous_heartbeat_time;
};
/** Contains all information associated with a local scheduler client. */
struct LocalSchedulerClient {
/** The socket used to communicate with the client. */
int sock;
/** True if the client has registered and false otherwise. */
bool registered;
/** True if the client has sent a disconnect message to the local scheduler
* and false otherwise. If this is true, then the local scheduler will not
* propagate an error message to the driver when the client exits. */
bool disconnected;
/** True if the client is a worker and false if it is a driver. */
bool is_worker;
/** The worker ID if the client is a worker and the driver ID if the client is
* a driver. */
WorkerID client_id;
/** A pointer to the task object that is currently running on this client. If
* no task is running on the worker, this will be NULL. This is used to
* update the task table. */
Task *task_in_progress;
/** An array of resource counts currently in use by the worker. */
std::unordered_map<std::string, double> resources_in_use;
/** A vector of the IDs of the GPUs that the worker is currently using. If the
* worker is an actor, this will be constant throughout the lifetime of the
* actor (and will be equal to the number of GPUs requested by the actor). If
* the worker is not an actor, this will be constant for the duration of a
* task and will have length equal to the number of GPUs requested by the
* task (in particular it will not change if the task blocks). */
std::vector<int> gpus_in_use;
/** A flag to indicate whether this worker is currently blocking on an
* object(s) that isn't available locally yet. */
bool is_blocked;
/** The process ID of the client. If this is set to zero, the client has not
* yet registered a process ID. */
pid_t pid;
/** Whether the client is a child process of the local scheduler. */
bool is_child;
/** The ID of the actor on this worker. If there is no actor running on this
* worker, this should be NIL_ACTOR_ID. */
ActorID actor_id;
/** A pointer to the local scheduler state. */
LocalSchedulerState *local_scheduler_state;
};
/**
* Free the local scheduler state. This disconnects all clients and notifies
* the global scheduler of the local scheduler's exit.
*
* @param state The state to free.
* @return Void
*/
void LocalSchedulerState_free(LocalSchedulerState *state);
#endif /* LOCAL_SCHEDULER_SHARED_H */
src/local_scheduler/test/local_scheduler_tests.cc
-704
View File
@@ -1,704 +0,0 @@
#include "greatest.h"
#include <assert.h>
#include <unistd.h>
#include <poll.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <cstdio>
#include <sstream>
#include <string>
#include <thread>
#include "common.h"
#include "test/test_common.h"
#include "test/example_task.h"
#include "event_loop.h"
#include "io.h"
#include "task.h"
#include "state/object_table.h"
#include "state/task_table.h"
#include "state/redis.h"
#include "local_scheduler_shared.h"
#include "local_scheduler.h"
#include "local_scheduler_algorithm.h"
#include "local_scheduler_client.h"
SUITE(local_scheduler_tests);
TaskBuilder *g_task_builder = NULL;
const char *plasma_store_socket_name = "/tmp/plasma_store_socket_1";
const char *plasma_manager_socket_name_format = "/tmp/plasma_manager_socket_%d";
const char *local_scheduler_socket_name_format =
"/tmp/local_scheduler_socket_%d";
int64_t timeout_handler(event_loop *loop, int64_t id, void *context) {
event_loop_stop(loop);
return EVENT_LOOP_TIMER_DONE;
}
typedef struct {
/** A socket to mock the Plasma manager. Clients (such as workers) that
* connect to this file descriptor must be accepted. */
int plasma_manager_fd;
/** A socket to communicate with the Plasma store. */
int plasma_store_fd;
/** Local scheduler's socket for IPC requests. */
int local_scheduler_fd;
/** Local scheduler's local scheduler state. */
LocalSchedulerState *local_scheduler_state;
/** Local scheduler's event loop. */
event_loop *loop;
/** Number of local scheduler client connections, or mock workers. */
int num_local_scheduler_conns;
/** Local scheduler client connections. */
LocalSchedulerConnection **conns;
} LocalSchedulerMock;
/**
* Register clients of the local scheduler. This function is started in a
* separate thread so enable a blocking call to register the clients.
*/
static void register_clients(int num_mock_workers, LocalSchedulerMock *mock) {
for (int i = 0; i < num_mock_workers; ++i) {
new_client_connection(mock->loop, mock->local_scheduler_fd,
(void *) mock->local_scheduler_state, 0);
LocalSchedulerClient *worker = mock->local_scheduler_state->workers.back();
process_message(mock->local_scheduler_state->loop, worker->sock, worker, 0);
}
}
LocalSchedulerMock *LocalSchedulerMock_init(int num_workers,
int num_mock_workers) {
const char *node_ip_address = "127.0.0.1";
const char *redis_addr = node_ip_address;
int redis_port = 6379;
std::unordered_map<std::string, double> static_resource_conf;
static_resource_conf["CPU"] = INT16_MAX;
static_resource_conf["GPU"] = 0;
LocalSchedulerMock *mock =
(LocalSchedulerMock *) malloc(sizeof(LocalSchedulerMock));
memset(mock, 0, sizeof(LocalSchedulerMock));
mock->loop = event_loop_create();
/* Bind to the local scheduler port and initialize the local scheduler. */
std::string plasma_manager_socket_name = bind_ipc_sock_retry(
plasma_manager_socket_name_format, &mock->plasma_manager_fd);
mock->plasma_store_fd =
connect_ipc_sock_retry(plasma_store_socket_name, 5, 100);
std::string local_scheduler_socket_name = bind_ipc_sock_retry(
local_scheduler_socket_name_format, &mock->local_scheduler_fd);
RAY_CHECK(mock->plasma_store_fd >= 0 && mock->local_scheduler_fd >= 0);
/* Construct worker command */
std::stringstream worker_command_ss;
worker_command_ss << "python ../python/ray/workers/default_worker.py"
<< " --node-ip-address=" << node_ip_address
<< " --object-store-name=" << plasma_store_socket_name
<< " --object-store-manager-name="
<< plasma_manager_socket_name
<< " --local-scheduler-name=" << local_scheduler_socket_name
<< " --redis-address=" << redis_addr << ":" << redis_port;
std::string worker_command = worker_command_ss.str();
mock->local_scheduler_state = LocalSchedulerState_init(
"127.0.0.1", mock->loop, redis_addr, redis_port,
local_scheduler_socket_name.c_str(), plasma_store_socket_name,
plasma_manager_socket_name.c_str(), NULL, false, static_resource_conf,
worker_command.c_str(), num_workers);
/* Accept the workers as clients to the plasma manager. */
for (int i = 0; i < num_workers; ++i) {
accept_client(mock->plasma_manager_fd);
}
/* Connect a local scheduler client. */
mock->num_local_scheduler_conns = num_mock_workers;
mock->conns = (LocalSchedulerConnection **) malloc(
sizeof(LocalSchedulerConnection *) * num_mock_workers);
std::thread background_thread =
std::thread(register_clients, num_mock_workers, mock);
for (int i = 0; i < num_mock_workers; ++i) {
mock->conns[i] = LocalSchedulerConnection_init(
local_scheduler_socket_name.c_str(), WorkerID::nil(), true,
JobID::nil(), false, Language::PYTHON);
}
background_thread.join();
return mock;
}
void LocalSchedulerMock_free(LocalSchedulerMock *mock) {
/* Disconnect clients. */
for (int i = 0; i < mock->num_local_scheduler_conns; ++i) {
LocalSchedulerConnection_free(mock->conns[i]);
}
free(mock->conns);
/* Kill all the workers and run the event loop again so that the task table
* updates propagate and the tasks in progress are freed. */
while (mock->local_scheduler_state->workers.size() > 0) {
LocalSchedulerClient *worker = mock->local_scheduler_state->workers.front();
kill_worker(mock->local_scheduler_state, worker, true, false);
}
event_loop_add_timer(mock->loop, 500,
(event_loop_timer_handler) timeout_handler, NULL);
event_loop_run(mock->loop);
/* This also frees mock->loop. */
LocalSchedulerState_free(mock->local_scheduler_state);
close(mock->plasma_store_fd);
close(mock->plasma_manager_fd);
free(mock);
}
void reset_worker(LocalSchedulerMock *mock, LocalSchedulerClient *worker) {
if (worker->task_in_progress) {
Task_free(worker->task_in_progress);
worker->task_in_progress = NULL;
}
}
/**
* Test that object reconstruction gets called. If a task gets submitted,
* assigned to a worker, and then reconstruction is triggered for its return
* value, the task should get assigned to a worker again.
*/
TEST object_reconstruction_test(void) {
LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1);
LocalSchedulerConnection *worker = local_scheduler->conns[0];
/* Create a task with zero dependencies and one return value. */
TaskExecutionSpec execution_spec = example_task_execution_spec(0, 1);
TaskSpec *spec = execution_spec.Spec();
int64_t task_size = execution_spec.SpecSize();
ObjectID return_id = TaskSpec_return(spec, 0);
/* Add an empty object table entry for the object we want to reconstruct, to
* simulate it having been created and evicted. */
const char *client_id = "clientid";
/* Lookup the shard locations for the object table. */
std::vector<std::string> db_shards_addresses;
std::vector<int> db_shards_ports;
redisContext *context = redisConnect("127.0.0.1", 6379);
get_redis_shards(context, db_shards_addresses, db_shards_ports);
redisFree(context);
/* There should only be one shard, so we can safely add the empty object
* table entry to the first one. */
ASSERT(db_shards_addresses.size() == 1);
context = redisConnect(db_shards_addresses[0].c_str(), db_shards_ports[0]);
redisReply *reply = (redisReply *) redisCommand(
context, "RAY.OBJECT_TABLE_ADD %b %ld %b %s", return_id.data(),
sizeof(return_id), 1, NIL_DIGEST, (size_t) DIGEST_SIZE, client_id);
freeReplyObject(reply);
reply = (redisReply *) redisCommand(context, "RAY.OBJECT_TABLE_REMOVE %b %s",
return_id.data(), sizeof(return_id),
client_id);
freeReplyObject(reply);
redisFree(context);
pid_t pid = fork();
if (pid == 0) {
/* Make sure we receive the task twice. First from the initial submission,
* and second from the reconstruct request. */
int64_t task_assigned_size;
local_scheduler_submit(worker, execution_spec);
TaskSpec *task_assigned =
local_scheduler_get_task(worker, &task_assigned_size);
ASSERT_EQ(memcmp(task_assigned, spec, task_size), 0);
ASSERT_EQ(task_assigned_size, task_size);
int64_t reconstruct_task_size;
TaskSpec *reconstruct_task =
local_scheduler_get_task(worker, &reconstruct_task_size);
ASSERT_EQ(memcmp(reconstruct_task, spec, task_size), 0);
ASSERT_EQ(reconstruct_task_size, task_size);
/* Clean up. */
free(reconstruct_task);
free(task_assigned);
LocalSchedulerMock_free(local_scheduler);
exit(0);
} else {
/* Run the event loop. NOTE: OSX appears to require the parent process to
* listen for events on the open file descriptors. */
event_loop_add_timer(local_scheduler->loop, 500,
(event_loop_timer_handler) timeout_handler, NULL);
event_loop_run(local_scheduler->loop);
/* Set the task's status to TaskStatus::DONE to prevent the race condition
* that would suppress object reconstruction. */
Task *task = Task_alloc(
execution_spec, TaskStatus::DONE,
get_db_client_id(local_scheduler->local_scheduler_state->db));
task_table_add_task(local_scheduler->local_scheduler_state->db, task, NULL,
NULL, NULL);
/* Trigger reconstruction, and run the event loop again. */
ObjectID return_id = TaskSpec_return(spec, 0);
local_scheduler_reconstruct_objects(worker,
std::vector<ObjectID>({return_id}));
event_loop_add_timer(local_scheduler->loop, 500,
(event_loop_timer_handler) timeout_handler, NULL);
event_loop_run(local_scheduler->loop);
/* Wait for the child process to exit and check that there are no tasks
* left in the local scheduler's task queue. Then, clean up. */
wait(NULL);
ASSERT_EQ(num_waiting_tasks(
local_scheduler->local_scheduler_state->algorithm_state),
0);
ASSERT_EQ(num_dispatch_tasks(
local_scheduler->local_scheduler_state->algorithm_state),
0);
LocalSchedulerMock_free(local_scheduler);
PASS();
}
}
/**
* Test that object reconstruction gets recursively called. In a chain of
* tasks, if all inputs are lost, then reconstruction of the final object
* should trigger reconstruction of all previous tasks in the lineage.
*/
TEST object_reconstruction_recursive_test(void) {
LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1);
LocalSchedulerConnection *worker = local_scheduler->conns[0];
/* Create a chain of tasks, each one dependent on the one before it. Mark
* each object as available so that tasks will run immediately. */
const int NUM_TASKS = 10;
std::vector<TaskExecutionSpec> specs;
specs.push_back(example_task_execution_spec(0, 1));
for (int i = 1; i < NUM_TASKS; ++i) {
ObjectID arg_id = TaskSpec_return(specs[i - 1].Spec(), 0);
specs.push_back(example_task_execution_spec_with_args(1, 1, &arg_id));
}
/* Lookup the shard locations for the object table. */
const char *client_id = "clientid";
std::vector<std::string> db_shards_addresses;
std::vector<int> db_shards_ports;
redisContext *context = redisConnect("127.0.0.1", 6379);
get_redis_shards(context, db_shards_addresses, db_shards_ports);
redisFree(context);
/* There should only be one shard, so we can safely add the empty object
* table entry to the first one. */
ASSERT(db_shards_addresses.size() == 1);
context = redisConnect(db_shards_addresses[0].c_str(), db_shards_ports[0]);
for (int i = 0; i < NUM_TASKS; ++i) {
ObjectID return_id = TaskSpec_return(specs[i].Spec(), 0);
redisReply *reply = (redisReply *) redisCommand(
context, "RAY.OBJECT_TABLE_ADD %b %ld %b %s", return_id.data(),
sizeof(return_id), 1, NIL_DIGEST, (size_t) DIGEST_SIZE, client_id);
freeReplyObject(reply);
reply = (redisReply *) redisCommand(
context, "RAY.OBJECT_TABLE_REMOVE %b %s", return_id.data(),
sizeof(return_id), client_id);
freeReplyObject(reply);
}
redisFree(context);
pid_t pid = fork();
if (pid == 0) {
/* Submit the tasks, and make sure each one gets assigned to a worker. */
for (int i = 0; i < NUM_TASKS; ++i) {
local_scheduler_submit(worker, specs[i]);
}
/* Make sure we receive each task from the initial submission. */
for (int i = 0; i < NUM_TASKS; ++i) {
int64_t task_size;
TaskSpec *task_assigned = local_scheduler_get_task(worker, &task_size);
ASSERT_EQ(memcmp(task_assigned, specs[i].Spec(), specs[i].SpecSize()), 0);
ASSERT_EQ(task_size, specs[i].SpecSize());
free(task_assigned);
}
/* Check that the workers receive all tasks in the final return object's
* lineage during reconstruction. */
for (int i = 0; i < NUM_TASKS; ++i) {
int64_t task_assigned_size;
TaskSpec *task_assigned =
local_scheduler_get_task(worker, &task_assigned_size);
for (auto it = specs.begin(); it != specs.end(); it++) {
if (memcmp(task_assigned, it->Spec(), task_assigned_size) == 0) {
specs.erase(it);
break;
}
}
free(task_assigned);
}
ASSERT(specs.size() == 0);
LocalSchedulerMock_free(local_scheduler);
exit(0);
} else {
/* Simulate each task putting its return values in the object store so that
* the next task can run. */
for (int i = 0; i < NUM_TASKS; ++i) {
ObjectID return_id = TaskSpec_return(specs[i].Spec(), 0);
handle_object_available(
local_scheduler->local_scheduler_state,
local_scheduler->local_scheduler_state->algorithm_state, return_id);
}
/* Run the event loop. All tasks should now be dispatched. NOTE: OSX
* appears to require the parent process to listen for events on the open
* file descriptors. */
event_loop_add_timer(local_scheduler->loop, 500,
(event_loop_timer_handler) timeout_handler, NULL);
event_loop_run(local_scheduler->loop);
/* Set the final task's status to TaskStatus::DONE to prevent the race
* condition that would suppress object reconstruction. */
Task *last_task = Task_alloc(
specs[NUM_TASKS - 1], TaskStatus::DONE,
get_db_client_id(local_scheduler->local_scheduler_state->db));
task_table_add_task(local_scheduler->local_scheduler_state->db, last_task,
NULL, NULL, NULL);
/* Simulate eviction of the objects, so that reconstruction is required. */
for (int i = 0; i < NUM_TASKS; ++i) {
ObjectID return_id = TaskSpec_return(specs[i].Spec(), 0);
handle_object_removed(local_scheduler->local_scheduler_state, return_id);
}
/* Trigger reconstruction for the last object. */
ObjectID return_id = TaskSpec_return(specs[NUM_TASKS - 1].Spec(), 0);
local_scheduler_reconstruct_objects(worker,
std::vector<ObjectID>({return_id}));
/* Run the event loop again. All tasks should be resubmitted. */
event_loop_add_timer(local_scheduler->loop, 500,
(event_loop_timer_handler) timeout_handler, NULL);
event_loop_run(local_scheduler->loop);
/* Simulate each task putting its return values in the object store so that
* the next task can run. */
for (int i = 0; i < NUM_TASKS; ++i) {
ObjectID return_id = TaskSpec_return(specs[i].Spec(), 0);
handle_object_available(
local_scheduler->local_scheduler_state,
local_scheduler->local_scheduler_state->algorithm_state, return_id);
}
/* Run the event loop again. All tasks should be dispatched again. */
event_loop_add_timer(local_scheduler->loop, 500,
(event_loop_timer_handler) timeout_handler, NULL);
event_loop_run(local_scheduler->loop);
/* Wait for the child process to exit and check that there are no tasks
* left in the local scheduler's task queue. Then, clean up. */
wait(NULL);
ASSERT_EQ(num_waiting_tasks(
local_scheduler->local_scheduler_state->algorithm_state),
0);
ASSERT_EQ(num_dispatch_tasks(
local_scheduler->local_scheduler_state->algorithm_state),
0);
specs.clear();
LocalSchedulerMock_free(local_scheduler);
PASS();
}
}
/**
* Test that object reconstruction gets suppressed when there is a location
* listed for the object in the object table.
*/
TaskExecutionSpec *object_reconstruction_suppression_spec;
void object_reconstruction_suppression_callback(ObjectID object_id,
bool success,
void *user_context) {
RAY_CHECK(success);
/* Submit the task after adding the object to the object table. */
LocalSchedulerConnection *worker = (LocalSchedulerConnection *) user_context;
local_scheduler_submit(worker, *object_reconstruction_suppression_spec);
}
TEST object_reconstruction_suppression_test(void) {
LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1);
LocalSchedulerConnection *worker = local_scheduler->conns[0];
TaskExecutionSpec execution_spec = example_task_execution_spec(0, 1);
object_reconstruction_suppression_spec = &execution_spec;
ObjectID return_id =
TaskSpec_return(object_reconstruction_suppression_spec->Spec(), 0);
pid_t pid = fork();
if (pid == 0) {
/* Make sure we receive the task once. This will block until the
* object_table_add callback completes. */
int64_t task_assigned_size;
TaskSpec *task_assigned =
local_scheduler_get_task(worker, &task_assigned_size);
ASSERT_EQ(
memcmp(task_assigned, object_reconstruction_suppression_spec->Spec(),
object_reconstruction_suppression_spec->SpecSize()),
0);
/* Trigger a reconstruction. We will check that no tasks get queued as a
* result of this line in the event loop process. */
local_scheduler_reconstruct_objects(worker,
std::vector<ObjectID>({return_id}));
/* Clean up. */
free(task_assigned);
LocalSchedulerMock_free(local_scheduler);
exit(0);
} else {
/* Connect a plasma manager client so we can call object_table_add. */
std::vector<std::string> db_connect_args;
db_connect_args.push_back("manager_address");
db_connect_args.push_back("127.0.0.1:12346");
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", db_connect_args);
db_attach(db, local_scheduler->loop, false);
/* Add the object to the object table. */
object_table_add(db, return_id, 1, (unsigned char *) NIL_DIGEST, NULL,
object_reconstruction_suppression_callback,
(void *) worker);
/* Run the event loop. NOTE: OSX appears to require the parent process to
* listen for events on the open file descriptors. */
event_loop_add_timer(local_scheduler->loop, 1000,
(event_loop_timer_handler) timeout_handler, NULL);
event_loop_run(local_scheduler->loop);
/* Wait for the child process to exit and check that there are no tasks
* left in the local scheduler's task queue. Then, clean up. */
wait(NULL);
ASSERT_EQ(num_waiting_tasks(
local_scheduler->local_scheduler_state->algorithm_state),
0);
ASSERT_EQ(num_dispatch_tasks(
local_scheduler->local_scheduler_state->algorithm_state),
0);
db_disconnect(db);
LocalSchedulerMock_free(local_scheduler);
PASS();
}
}
TEST task_dependency_test(void) {
LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1);
LocalSchedulerState *state = local_scheduler->local_scheduler_state;
SchedulingAlgorithmState *algorithm_state = state->algorithm_state;
/* Get the first worker. */
LocalSchedulerClient *worker = state->workers.front();
TaskExecutionSpec execution_spec = example_task_execution_spec(1, 1);
TaskSpec *spec = execution_spec.Spec();
ObjectID oid = TaskSpec_arg_id(spec, 0, 0);
/* Check that the task gets queued in the waiting queue if the task is
* submitted, but the input and workers are not available. */
handle_task_submitted(state, algorithm_state, execution_spec);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
/* Once the input is available, the task gets moved to the dispatch queue. */
handle_object_available(state, algorithm_state, oid);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1);
/* Once a worker is available, the task gets assigned. */
handle_worker_available(state, algorithm_state, worker);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
reset_worker(local_scheduler, worker);
/* Check that the task gets queued in the waiting queue if the task is
* submitted and a worker is available, but the input is not. */
handle_object_removed(state, oid);
handle_task_submitted(state, algorithm_state, execution_spec);
handle_worker_available(state, algorithm_state, worker);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
/* Once the input is available, the task gets assigned. */
handle_object_available(state, algorithm_state, oid);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
reset_worker(local_scheduler, worker);
/* Check that the task gets queued in the dispatch queue if the task is
* submitted and the input is available, but no worker is available yet. */
handle_task_submitted(state, algorithm_state, execution_spec);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1);
/* Once a worker is available, the task gets assigned. */
handle_worker_available(state, algorithm_state, worker);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
reset_worker(local_scheduler, worker);
/* If an object gets removed, check the first scenario again, where the task
* gets queued in the waiting task if the task is submitted and a worker is
* available, but the input is not. */
handle_task_submitted(state, algorithm_state, execution_spec);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1);
/* If the input is removed while a task is in the dispatch queue, the task
* gets moved back to the waiting queue. */
handle_object_removed(state, oid);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
/* Once the input is available, the task gets moved back to the dispatch
* queue. */
handle_object_available(state, algorithm_state, oid);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1);
/* Once a worker is available, the task gets assigned. */
handle_worker_available(state, algorithm_state, worker);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
LocalSchedulerMock_free(local_scheduler);
PASS();
}
TEST task_multi_dependency_test(void) {
LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1);
LocalSchedulerState *state = local_scheduler->local_scheduler_state;
SchedulingAlgorithmState *algorithm_state = state->algorithm_state;
/* Get the first worker. */
LocalSchedulerClient *worker = state->workers.front();
TaskExecutionSpec execution_spec = example_task_execution_spec(2, 1);
TaskSpec *spec = execution_spec.Spec();
ObjectID oid1 = TaskSpec_arg_id(spec, 0, 0);
ObjectID oid2 = TaskSpec_arg_id(spec, 1, 0);
/* Check that the task gets queued in the waiting queue if the task is
* submitted, but the inputs and workers are not available. */
handle_task_submitted(state, algorithm_state, execution_spec);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
/* Check that the task stays in the waiting queue if only one input becomes
* available. */
handle_object_available(state, algorithm_state, oid2);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
/* Once all inputs are available, the task is moved to the dispatch queue. */
handle_object_available(state, algorithm_state, oid1);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1);
/* Once a worker is available, the task gets assigned. */
handle_worker_available(state, algorithm_state, worker);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
reset_worker(local_scheduler, worker);
/* Check that the task gets queued in the dispatch queue if the task is
* submitted and the inputs are available, but no worker is available yet. */
handle_task_submitted(state, algorithm_state, execution_spec);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1);
/* If any input is removed while a task is in the dispatch queue, the task
* gets moved back to the waiting queue. */
handle_object_removed(state, oid1);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
handle_object_removed(state, oid2);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
/* Check that the task stays in the waiting queue if only one input becomes
* available. */
handle_object_available(state, algorithm_state, oid2);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
/* Check that the task stays in the waiting queue if the one input is
* unavailable again. */
handle_object_removed(state, oid2);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
/* Check that the task stays in the waiting queue if the other input becomes
* available. */
handle_object_available(state, algorithm_state, oid1);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 1);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
/* Once all inputs are available, the task is moved to the dispatch queue. */
handle_object_available(state, algorithm_state, oid2);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1);
/* Once a worker is available, the task gets assigned. */
handle_worker_available(state, algorithm_state, worker);
ASSERT_EQ(num_waiting_tasks(algorithm_state), 0);
ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0);
reset_worker(local_scheduler, worker);
LocalSchedulerMock_free(local_scheduler);
PASS();
}
TEST start_kill_workers_test(void) {
/* Start some workers. */
int num_workers = 4;
LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(num_workers, 0);
/* We start off with num_workers children processes, but no workers
* registered yet. */
ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(),
static_cast<size_t>(num_workers));
ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(), 0);
/* Make sure that each worker connects to the local_scheduler scheduler. This
* for loop will hang if one of the workers does not connect. */
for (int i = 0; i < num_workers; ++i) {
new_client_connection(local_scheduler->loop,
local_scheduler->local_scheduler_fd,
(void *) local_scheduler->local_scheduler_state, 0);
}
/* After handling each worker's initial connection, we should now have all
* workers accounted for, but we haven't yet matched up process IDs with our
* children processes. */
ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(),
static_cast<size_t>(num_workers));
ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(),
static_cast<size_t>(num_workers));
/* Each worker should register its process ID. */
for (auto const &worker : local_scheduler->local_scheduler_state->workers) {
process_message(local_scheduler->local_scheduler_state->loop, worker->sock,
worker, 0);
}
ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 0);
ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(),
static_cast<size_t>(num_workers));
/* After killing a worker, its state is cleaned up. */
LocalSchedulerClient *worker =
local_scheduler->local_scheduler_state->workers.front();
kill_worker(local_scheduler->local_scheduler_state, worker, false, false);
ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 0);
ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(),
static_cast<size_t>(num_workers - 1));
/* Start a worker after the local scheduler has been initialized. */
start_worker(local_scheduler->local_scheduler_state);
/* Accept the workers as clients to the plasma manager. */
int new_worker_fd = accept_client(local_scheduler->plasma_manager_fd);
/* The new worker should register its process ID. */
ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 1);
ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(),
static_cast<size_t>(num_workers - 1));
/* Make sure the new worker connects to the local_scheduler scheduler. */
new_client_connection(local_scheduler->loop,
local_scheduler->local_scheduler_fd,
(void *) local_scheduler->local_scheduler_state, 0);
ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 1);
ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(),
static_cast<size_t>(num_workers));
/* Make sure that the new worker registers its process ID. */
worker = local_scheduler->local_scheduler_state->workers.back();
process_message(local_scheduler->local_scheduler_state->loop, worker->sock,
worker, 0);
ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 0);
ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(),
static_cast<size_t>(num_workers));
/* Clean up. */
close(new_worker_fd);
LocalSchedulerMock_free(local_scheduler);
PASS();
}
SUITE(local_scheduler_tests) {
RUN_REDIS_TEST(object_reconstruction_test);
RUN_REDIS_TEST(object_reconstruction_recursive_test);
RUN_REDIS_TEST(object_reconstruction_suppression_test);
RUN_REDIS_TEST(task_dependency_test);
RUN_REDIS_TEST(task_multi_dependency_test);
RUN_REDIS_TEST(start_kill_workers_test);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
g_task_builder = make_task_builder();
GREATEST_MAIN_BEGIN();
RUN_SUITE(local_scheduler_tests);
GREATEST_MAIN_END();
}
src/local_scheduler/test/run_tests.sh
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env bash
# This needs to be run in the build tree, which is normally ray/build
# Cause the script to exit if a single command fails.
set -e
LaunchRedis() {
port=$1
if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
./src/credis/redis/src/redis-server \
--loglevel warning \
--loadmodule ./src/credis/build/src/libmember.so \
--loadmodule ./src/common/redis_module/libray_redis_module.so \
--port $port &
else
./src/common/thirdparty/redis/src/redis-server \
--loglevel warning \
--loadmodule ./src/common/redis_module/libray_redis_module.so \
--port $port &
fi
}
# Start the Redis shards.
LaunchRedis 6379
LaunchRedis 6380
sleep 1s
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 &
sleep 0.5s
./src/local_scheduler/local_scheduler_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
killall plasma_store_server
src/local_scheduler/test/run_valgrind.sh
-41
View File
@@ -1,41 +0,0 @@
#!/usr/bin/env bash
# This needs to be run in the build tree, which is normally ray/build
set -x
# Cause the script to exit if a single command fails.
set -e
LaunchRedis() {
port=$1
if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
./src/credis/redis/src/redis-server \
--loglevel warning \
--loadmodule ./src/credis/build/src/libmember.so \
--loadmodule ./src/common/redis_module/libray_redis_module.so \
--port $port &
else
./src/common/thirdparty/redis/src/redis-server \
--loglevel warning \
--loadmodule ./src/common/redis_module/libray_redis_module.so \
--port $port &
fi
}
# Start the Redis shards.
LaunchRedis 6379
LaunchRedis 6380
sleep 1s
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 &
sleep 0.5s
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/local_scheduler/local_scheduler_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
killall plasma_store_server
src/plasma/CMakeLists.txt
-61
View File
@@ -1,61 +0,0 @@
cmake_minimum_required(VERSION 3.4)
project(plasma)
include_directories(${CMAKE_CURRENT_LIST_DIR})
include_directories(${CMAKE_CURRENT_LIST_DIR}/thirdparty)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --std=c99 -O3")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++11 -O3 -Werror -Wall")
if(UNIX AND NOT APPLE)
link_libraries(rt)
endif()
include_directories("${ARROW_INCLUDE_DIR}")
set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
set(PLASMA_FBS_OUTPUT_FILES
"${OUTPUT_DIR}/plasma_generated.h"
"${OUTPUT_DIR}/common_generated.h")
add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
add_dependencies(gen_plasma_fbs arrow_ep)
# Copy the fbs files from Arrow project to local directory.
add_custom_command(
OUTPUT ${PLASMA_FBS_SRC}
COMMAND mkdir -p ${CMAKE_CURRENT_LIST_DIR}/format/
COMMAND cp ${ARROW_SOURCE_DIR}/cpp/src/plasma/format/plasma.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
COMMAND cp ${ARROW_SOURCE_DIR}/cpp/src/plasma/format/common.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
COMMENT "Copying ${PLASMA_FBS_SRC} to local"
VERBATIM)
# Compile flatbuffers
add_custom_command(
OUTPUT ${PLASMA_FBS_OUTPUT_FILES}
# The --gen-object-api flag generates a C++ class MessageT for each
# flatbuffers message Message, which can be used to store deserialized
# messages in data structures. This is currently used for ObjectInfo for
# example.
COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} --gen-object-api --scoped-enums
DEPENDS ${PLASMA_FBS_SRC}
COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}"
VERBATIM)
include_directories("${FLATBUFFERS_INCLUDE_DIR}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
add_executable(plasma_manager
plasma_manager.cc)
add_dependencies(plasma_manager gen_plasma_fbs)
target_link_libraries(plasma_manager common ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY})
define_test(client_tests "")
define_test(manager_tests "" plasma_manager.cc)
target_link_libraries(manager_tests ${Boost_SYSTEM_LIBRARY})
add_dependencies(manager_tests gen_plasma_fbs)
src/plasma/doc/plasma-doxy-config
-2473
View File
File diff suppressed because it is too large Load Diff
src/plasma/plasma_manager.cc
-1692
View File
File diff suppressed because it is too large Load Diff
src/plasma/plasma_manager.h
-277
View File
@@ -1,277 +0,0 @@
#ifndef PLASMA_MANAGER_H
#define PLASMA_MANAGER_H
#include "protocol.h"
#ifndef RAY_NUM_RETRIES
#define NUM_RETRIES 5
#else
#define NUM_RETRIES RAY_NUM_RETRIES
#endif
typedef struct PlasmaManagerState PlasmaManagerState;
typedef struct ClientConnection ClientConnection;
/**
* Initializes the plasma manager state. This connects the manager to the local
* plasma store, starts the manager listening for client connections, and
* connects the manager to a database if there is one. The returned manager
* state should be freed using the provided PlasmaManagerState_destroy
* function.
*
* @param store_socket_name The socket name used to connect to the local store.
* @param manager_socket_name The socket name used to connect to the manager.
* @param manager_addr Our IP address.
* @param manager_port The IP port that we listen on.
* @param db_addr The IP address of the database to connect to. If this is NULL,
* then the manager will be initialized without a database
* connection.
* @param db_port The IP port of the database to connect to.
* @return A pointer to the initialized plasma manager state.
*/
PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name,
const char *manager_socket_name,
const char *manager_addr,
int manager_port,
const char *db_addr,
int db_port);
/**
* Destroys the plasma manager state and its connections.
*
* @param state A pointer to the plasma manager state to destroy.
* @return Void.
*/
void PlasmaManagerState_free(PlasmaManagerState *state);
/**
* Process a request from another object store manager to transfer an object.
*
* @param loop This is the event loop of the plasma manager.
* @param object_id The object_id of the object we will be sending.
* @param addr The IP address of the plasma manager to send the object to.
* @param port The port of the plasma manager we are sending the object to.
* @param conn The ClientConnection to the other plasma manager.
* @return Void.
*
* This establishes a connection to the remote manager if one doesn't already
* exist, and queues up the request to transfer the data to the other object
* manager.
*/
void process_transfer(event_loop *loop,
ray::ObjectID object_id,
uint8_t addr[4],
int port,
ClientConnection *conn);
/**
* Process a request from another object store manager to receive data.
*
* @param loop This is the event loop of the plasma manager.
* @param client_sock The connection to the other plasma manager.
* @param object_id The object_id of the object we will be reading.
* @param data_size Size of the object.
* @param metadata_size Size of the metadata.
* @param conn The ClientConnection to the other plasma manager.
* @return Void.
*
* Initializes the object we are going to write to in the local plasma store
* and then switches the data socket to read the raw object bytes instead of
* plasma requests.
*/
void process_data(event_loop *loop,
int client_sock,
ray::ObjectID object_id,
int64_t data_size,
int64_t metadata_size,
ClientConnection *conn);
/**
* Read the next chunk of the object in transit from the plasma manager
* connected to the given socket. Once all data for this object has been read,
* the socket switches to listening for the next plasma request.
*
* @param loop This is the event loop of the plasma manager.
* @param data_sock The connection to the other plasma manager.
* @param context The ClientConnection to the other plasma manager.
* @return Void.
*/
void process_data_chunk(event_loop *loop,
int data_sock,
void *context,
int events);
/**
* Callback that will be called when a new object becomes available.
*
* @param loop This is the event loop of the plasma manager.
* @param client_sock The connection to the plasma store.
* @param context Plasma manager state.
* @param events (unused).
* @return Void.
*/
void process_object_notification(event_loop *loop,
int client_sock,
void *context,
int events);
/**
* Send the next request queued for the other plasma manager connected to the
* socket "data_sock". This could be a request to either write object data or
* request object data. If the request is to write object data and no data has
* been sent yet, the initial handshake to transfer the object size is
* performed.
*
* @param loop This is the event loop of the plasma manager.
* @param data_sock This is the socket the other plasma manager is listening on.
* @param context The ClientConnection to the other plasma manager, contains a
* queue of objects that will be sent.
* @return Void.
*/
void send_queued_request(event_loop *loop,
int data_sock,
void *context,
int events);
/**
* Register a new client connection with the plasma manager. A client can
* either be a worker or another plasma manager.
*
* @param loop This is the event loop of the plasma manager.
* @param listener_socket The socket the plasma manager is listening on.
* @param context The plasma manager state.
* @return Void.
*/
ClientConnection *ClientConnection_listen(event_loop *loop,
int listener_sock,
void *context,
int events);
/**
* The following definitions are internal to the plasma manager code but are
* needed by the unit tests in test/manager_tests.c. This includes structs
* instantiated by the unit tests and forward declarations for functions used
* internally by the plasma manager code.
*/
/* Buffer for requests between plasma managers. */
typedef struct PlasmaRequestBuffer {
plasma::flatbuf::MessageType type;
ray::ObjectID object_id;
uint8_t *data;
int64_t data_size;
uint8_t *metadata;
int64_t metadata_size;
} PlasmaRequestBuffer;
/**
* Call the request_transfer method, which well attempt to get an object from
* a remote Plasma manager. If it is unable to get it from another Plasma
* manager, it will cycle through a list of Plasma managers that have the
* object. This method is only called from the tests.
*
* @param object_id The object ID of the object to transfer.
* @param manager_vector The Plasma managers that have the object.
* @param context The plasma manager state.
* @return Void.
*/
void call_request_transfer(ray::ObjectID object_id,
const std::vector<std::string> &manager_vector,
void *context);
/*
* This runs periodically (every manager_timeout_milliseconds milliseconds) and
* reissues transfer requests for all outstanding fetch requests. This is only
* exposed so that it can be called from the tests.
*/
int fetch_timeout_handler(event_loop *loop, timer_id id, void *context);
/**
* Get a connection to the remote manager at the specified address. Creates a
* new connection to this manager if one doesn't already exist.
*
* @param state Our plasma manager state.
* @param ip_addr The IP address of the remote manager we want to connect to.
* @param port The port that the remote manager is listening on.
* @return A pointer to the connection to the remote manager.
*/
ClientConnection *get_manager_connection(PlasmaManagerState *state,
const char *ip_addr,
int port);
/**
* Reads an object chunk sent by the given client into a buffer. This is the
* complement to write_object_chunk.
*
* @param conn The connection to the client who's sending the data. The
* connection's cursor will be reset if this is the last read for the
* current object.
* @param buf The buffer to write the data into.
* @return The errno set, if the read wasn't successful.
*/
int read_object_chunk(ClientConnection *conn, PlasmaRequestBuffer *buf);
/**
* Writes an object chunk from a buffer to the given client. This is the
* complement to read_object_chunk.
*
* @param conn The connection to the client who's receiving the data. The
* connection's cursor will be reset if this is the last write for the
* current object.
* @param buf The buffer to read data from.
* @return The errno set, if the write wasn't successful.
*/
int write_object_chunk(ClientConnection *conn, PlasmaRequestBuffer *buf);
/**
* Start a new request on this connection.
*
* @param conn The connection on which the request is being sent.
* @return Void.
*/
void ClientConnection_start_request(ClientConnection *client_conn);
/**
* Finish the current request on this connection.
*
* @param conn The connection on which the request is being sent.
* @return Void.
*/
void ClientConnection_finish_request(ClientConnection *client_conn);
/**
* Check whether the current request on this connection is finished.
*
* @param conn The connection on which the request is being sent.
* @return Whether the request has finished.
*/
bool ClientConnection_request_finished(ClientConnection *client_conn);
/**
* Get the event loop of the given plasma manager state.
*
* @param state The state of the plasma manager whose loop we want.
* @return A pointer to the manager's event loop.
*/
event_loop *get_event_loop(PlasmaManagerState *state);
/**
* Get the file descriptor for the given client's socket. This is the socket
* that the client sends or reads data through.
*
* @param conn The connection to the client who's sending or reading data.
* @return A file descriptor for the socket.
*/
int get_client_sock(ClientConnection *conn);
/**
* Return whether or not the object is local.
*
* @param state The state of the plasma manager.
* @param object_id The ID of the object we want to find.
* @return A bool that is true if the requested object is local and false
* otherwise.
*/
bool is_object_local(PlasmaManagerState *state, ray::ObjectID object_id);
#endif /* PLASMA_MANAGER_H */
src/plasma/plasma_protocol.cc
-576
View File
@@ -1,576 +0,0 @@
#include "flatbuffers/flatbuffers.h"
#include "format/plasma_generated.h"
#include "plasma_common.h"
#include "plasma_protocol.h"
#include "plasma_io.h"
flatbuffers::Offset<
flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>>
to_flatbuffer(flatbuffers::FlatBufferBuilder &fbb,
ObjectID object_ids[],
int64_t num_objects) {
std::vector<flatbuffers::Offset<flatbuffers::String>> results;
for (size_t i = 0; i < num_objects; i++) {
results.push_back(fbb.CreateString(object_ids[i].binary()));
}
return fbb.CreateVector(results);
}
Status PlasmaReceive(int sock,
int64_t message_type,
std::vector<uint8_t> &buffer) {
int64_t type;
RETURN_NOT_OK(ReadMessage(sock, &type, buffer));
RAY_CHECK(type == message_type) << "type = " << type
<< ", message_type = " << message_type;
return Status::OK();
}
/* Create messages. */
Status SendCreateRequest(int sock,
ObjectID object_id,
int64_t data_size,
int64_t metadata_size) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaCreateRequest(
fbb, fbb.CreateString(object_id.binary()), data_size, metadata_size);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaCreateRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadCreateRequest(uint8_t *data,
ObjectID *object_id,
int64_t *data_size,
int64_t *metadata_size) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
*data_size = message->data_size();
*metadata_size = message->metadata_size();
*object_id = ObjectID::from_binary(message->object_id()->str());
return Status::OK();
}
Status SendCreateReply(int sock,
ObjectID object_id,
PlasmaObject *object,
int error_code) {
flatbuffers::FlatBufferBuilder fbb;
PlasmaObjectSpec plasma_object(
object->handle.store_fd, object->handle.mmap_size, object->data_offset,
object->data_size, object->metadata_offset, object->metadata_size);
auto message =
CreatePlasmaCreateReply(fbb, fbb.CreateString(object_id.binary()),
&plasma_object, (PlasmaError) error_code);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaCreateReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadCreateReply(uint8_t *data,
ObjectID *object_id,
PlasmaObject *object) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaCreateReply>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
object->handle.store_fd = message->plasma_object()->segment_index();
object->handle.mmap_size = message->plasma_object()->mmap_size();
object->data_offset = message->plasma_object()->data_offset();
object->data_size = message->plasma_object()->data_size();
object->metadata_offset = message->plasma_object()->metadata_offset();
object->metadata_size = message->plasma_object()->metadata_size();
return plasma_error_status(message->error());
}
/* Seal messages. */
Status SendSealRequest(int sock, ObjectID object_id, unsigned char *digest) {
flatbuffers::FlatBufferBuilder fbb;
auto digest_string = fbb.CreateString((char *) digest, kDigestSize);
auto message = CreatePlasmaSealRequest(
fbb, fbb.CreateString(object_id.binary()), digest_string);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaSealRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadSealRequest(uint8_t *data,
ObjectID *object_id,
unsigned char *digest) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaSealRequest>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
RAY_CHECK(message->digest()->size() == kDigestSize);
memcpy(digest, message->digest()->data(), kDigestSize);
return Status::OK();
}
Status SendSealReply(int sock, ObjectID object_id, int error) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaSealReply(
fbb, fbb.CreateString(object_id.binary()), (PlasmaError) error);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaSealReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadSealReply(uint8_t *data, ObjectID *object_id) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaSealReply>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
return plasma_error_status(message->error());
}
/* Release messages. */
Status SendReleaseRequest(int sock, ObjectID object_id) {
flatbuffers::FlatBufferBuilder fbb;
auto message =
CreatePlasmaReleaseRequest(fbb, fbb.CreateString(object_id.binary()));
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaReleaseRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadReleaseRequest(uint8_t *data, ObjectID *object_id) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaReleaseRequest>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
return Status::OK();
}
Status SendReleaseReply(int sock, ObjectID object_id, int error) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaReleaseReply(
fbb, fbb.CreateString(object_id.binary()), (PlasmaError) error);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaReleaseReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadReleaseReply(uint8_t *data, ObjectID *object_id) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
return plasma_error_status(message->error());
}
/* Delete messages. */
Status SendDeleteRequest(int sock, ObjectID object_id) {
flatbuffers::FlatBufferBuilder fbb;
auto message =
CreatePlasmaDeleteRequest(fbb, fbb.CreateString(object_id.binary()));
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaDeleteRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadDeleteRequest(uint8_t *data, ObjectID *object_id) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
return Status::OK();
}
Status SendDeleteReply(int sock, ObjectID object_id, int error) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaDeleteReply(
fbb, fbb.CreateString(object_id.binary()), (PlasmaError) error);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaDeleteReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadDeleteReply(uint8_t *data, ObjectID *object_id) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaDeleteReply>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
return plasma_error_status(message->error());
}
/* Satus messages. */
Status SendStatusRequest(int sock, ObjectID object_ids[], int64_t num_objects) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaStatusRequest(
fbb, to_flatbuffer(fbb, object_ids, num_objects));
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaStatusRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadStatusRequest(uint8_t *data,
ObjectID object_ids[],
int64_t num_objects) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaStatusRequest>(data);
for (int64_t i = 0; i < num_objects; ++i) {
object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
}
return Status::OK();
}
Status SendStatusReply(int sock,
ObjectID object_ids[],
ObjectStatus object_status[],
int64_t num_objects) {
flatbuffers::FlatBufferBuilder fbb;
auto message =
CreatePlasmaStatusReply(fbb, to_flatbuffer(fbb, object_ids, num_objects),
fbb.CreateVector(object_status, num_objects));
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaStatusReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
int64_t ReadStatusReply_num_objects(uint8_t *data) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
return message->object_ids()->size();
}
Status ReadStatusReply(uint8_t *data,
ObjectID object_ids[],
int object_status[],
int64_t num_objects) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
for (int64_t i = 0; i < num_objects; ++i) {
object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
}
for (int64_t i = 0; i < num_objects; ++i) {
object_status[i] = message->status()->data()[i];
}
return Status::OK();
}
/* Contains messages. */
Status SendContainsRequest(int sock, ObjectID object_id) {
flatbuffers::FlatBufferBuilder fbb;
auto message =
CreatePlasmaContainsRequest(fbb, fbb.CreateString(object_id.binary()));
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaContainsRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadContainsRequest(uint8_t *data, ObjectID *object_id) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaContainsRequest>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
return Status::OK();
}
Status SendContainsReply(int sock, ObjectID object_id, int has_object) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaContainsReply(
fbb, fbb.CreateString(object_id.binary()), has_object);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaContainsReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadContainsReply(uint8_t *data, ObjectID *object_id, int *has_object) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaContainsReply>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
*has_object = message->has_object();
return Status::OK();
}
/* Connect messages. */
Status SendConnectRequest(int sock) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaConnectRequest(fbb);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaConnectRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadConnectRequest(uint8_t *data) {
return Status::OK();
}
Status SendConnectReply(int sock, int64_t memory_capacity) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaConnectReply(fbb, memory_capacity);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaConnectReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadConnectReply(uint8_t *data, int64_t *memory_capacity) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaConnectReply>(data);
*memory_capacity = message->memory_capacity();
return Status::OK();
}
/* Evict messages. */
Status SendEvictRequest(int sock, int64_t num_bytes) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaEvictRequest(fbb, num_bytes);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaEvictRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadEvictRequest(uint8_t *data, int64_t *num_bytes) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaEvictRequest>(data);
*num_bytes = message->num_bytes();
return Status::OK();
}
Status SendEvictReply(int sock, int64_t num_bytes) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaEvictReply(fbb, num_bytes);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaEvictReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadEvictReply(uint8_t *data, int64_t &num_bytes) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaEvictReply>(data);
num_bytes = message->num_bytes();
return Status::OK();
}
/* Get messages. */
Status SendGetRequest(int sock,
ObjectID object_ids[],
int64_t num_objects,
int64_t timeout_ms) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaGetRequest(
fbb, to_flatbuffer(fbb, object_ids, num_objects), timeout_ms);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaGetRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadGetRequest(uint8_t *data,
std::vector<ObjectID> &object_ids,
int64_t *timeout_ms) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaGetRequest>(data);
for (int64_t i = 0; i < message->object_ids()->size(); ++i) {
auto object_id = message->object_ids()->Get(i)->str();
object_ids.push_back(ObjectID::from_binary(object_id));
}
*timeout_ms = message->timeout_ms();
return Status::OK();
}
Status SendGetReply(int sock,
ObjectID object_ids[],
std::unordered_map<ObjectID, PlasmaObject> &plasma_objects,
int64_t num_objects) {
flatbuffers::FlatBufferBuilder fbb;
std::vector<PlasmaObjectSpec> objects;
for (int i = 0; i < num_objects; ++i) {
const PlasmaObject &object = plasma_objects[object_ids[i]];
objects.push_back(PlasmaObjectSpec(
object.handle.store_fd, object.handle.mmap_size, object.data_offset,
object.data_size, object.metadata_offset, object.metadata_size));
}
auto message = CreatePlasmaGetReply(
fbb, to_flatbuffer(fbb, object_ids, num_objects),
fbb.CreateVectorOfStructs(objects.data(), num_objects));
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaGetReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadGetReply(uint8_t *data,
ObjectID object_ids[],
PlasmaObject plasma_objects[],
int64_t num_objects) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
for (int64_t i = 0; i < num_objects; ++i) {
object_ids[i] = ObjectID::from_binary(message->object_ids()->Get(i)->str());
}
for (int64_t i = 0; i < num_objects; ++i) {
const PlasmaObjectSpec *object = message->plasma_objects()->Get(i);
plasma_objects[i].handle.store_fd = object->segment_index();
plasma_objects[i].handle.mmap_size = object->mmap_size();
plasma_objects[i].data_offset = object->data_offset();
plasma_objects[i].data_size = object->data_size();
plasma_objects[i].metadata_offset = object->metadata_offset();
plasma_objects[i].metadata_size = object->metadata_size();
}
return Status::OK();
}
/* Fetch messages. */
Status SendFetchRequest(int sock, ObjectID object_ids[], int64_t num_objects) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaFetchRequest(
fbb, to_flatbuffer(fbb, object_ids, num_objects));
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaFetchRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadFetchRequest(uint8_t *data, std::vector<ObjectID> &object_ids) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaFetchRequest>(data);
for (int64_t i = 0; i < message->object_ids()->size(); ++i) {
object_ids.push_back(
ObjectID::from_binary(message->object_ids()->Get(i)->str()));
}
return Status::OK();
}
/* Wait messages. */
Status SendWaitRequest(int sock,
ObjectRequest object_requests[],
int num_requests,
int num_ready_objects,
int64_t timeout_ms) {
flatbuffers::FlatBufferBuilder fbb;
std::vector<flatbuffers::Offset<ObjectRequestSpec>> object_request_specs;
for (int i = 0; i < num_requests; i++) {
object_request_specs.push_back(CreateObjectRequestSpec(
fbb, fbb.CreateString(object_requests[i].object_id.binary()),
object_requests[i].type));
}
auto message =
CreatePlasmaWaitRequest(fbb, fbb.CreateVector(object_request_specs),
num_ready_objects, timeout_ms);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaWaitRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadWaitRequest(uint8_t *data,
ObjectRequestMap &object_requests,
int64_t *timeout_ms,
int *num_ready_objects) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaWaitRequest>(data);
*num_ready_objects = message->num_ready_objects();
*timeout_ms = message->timeout();
for (int i = 0; i < message->object_requests()->size(); i++) {
ObjectID object_id = ObjectID::from_binary(
message->object_requests()->Get(i)->object_id()->str());
ObjectRequest object_request({object_id,
message->object_requests()->Get(i)->type(),
ObjectStatus_Nonexistent});
object_requests[object_id] = object_request;
}
return Status::OK();
}
Status SendWaitReply(int sock,
const ObjectRequestMap &object_requests,
int num_ready_objects) {
flatbuffers::FlatBufferBuilder fbb;
std::vector<flatbuffers::Offset<ObjectReply>> object_replies;
for (const auto &entry : object_requests) {
const auto &object_request = entry.second;
object_replies.push_back(CreateObjectReply(
fbb, fbb.CreateString(object_request.object_id.binary()),
object_request.status));
}
auto message = CreatePlasmaWaitReply(
fbb, fbb.CreateVector(object_replies.data(), num_ready_objects),
num_ready_objects);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaWaitReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadWaitReply(uint8_t *data,
ObjectRequest object_requests[],
int *num_ready_objects) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaWaitReply>(data);
*num_ready_objects = message->num_ready_objects();
for (int i = 0; i < *num_ready_objects; i++) {
object_requests[i].object_id = ObjectID::from_binary(
message->object_requests()->Get(i)->object_id()->str());
object_requests[i].status = message->object_requests()->Get(i)->status();
}
return Status::OK();
}
/* Subscribe messages. */
Status SendSubscribeRequest(int sock) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaSubscribeRequest(fbb);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaSubscribeRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
/* Data messages. */
Status SendDataRequest(int sock,
ObjectID object_id,
const char *address,
int port) {
flatbuffers::FlatBufferBuilder fbb;
auto addr = fbb.CreateString((char *) address, strlen(address));
auto message = CreatePlasmaDataRequest(
fbb, fbb.CreateString(object_id.binary()), addr, port);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaDataRequest, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadDataRequest(uint8_t *data,
ObjectID *object_id,
char **address,
int *port) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaDataRequest>(data);
RAY_DCHECK(message->object_id()->size() == sizeof(ObjectID));
*object_id = ObjectID::from_binary(message->object_id()->str());
*address = strdup(message->address()->c_str());
*port = message->port();
return Status::OK();
}
Status SendDataReply(int sock,
ObjectID object_id,
int64_t object_size,
int64_t metadata_size) {
flatbuffers::FlatBufferBuilder fbb;
auto message = CreatePlasmaDataReply(
fbb, fbb.CreateString(object_id.binary()), object_size, metadata_size);
fbb.Finish(message);
return WriteMessage(sock, MessageType::PlasmaDataReply, fbb.GetSize(),
fbb.GetBufferPointer());
}
Status ReadDataReply(uint8_t *data,
ObjectID *object_id,
int64_t *object_size,
int64_t *metadata_size) {
RAY_DCHECK(data);
auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
*object_id = ObjectID::from_binary(message->object_id()->str());
*object_size = (int64_t) message->object_size();
*metadata_size = (int64_t) message->metadata_size();
return Status::OK();
}
src/plasma/protocol.h
-77
View File
@@ -1,77 +0,0 @@
#ifndef PLASMA_PROTOCOL_H
#define PLASMA_PROTOCOL_H
#include "./format/common_generated.h"
#include "./format/plasma_generated.h"
namespace plasma {
namespace flatbuf {
enum class MessageType : int64_t;
};
using arrow::Status;
typedef std::unordered_map<ObjectID, ObjectRequest> ObjectRequestMap;
Status PlasmaReceive(int sock,
flatbuf::MessageType message_type,
std::vector<uint8_t> *buffer);
Status SendWaitReply(int sock,
const ObjectRequestMap &object_requests,
int num_ready_objects);
Status SendStatusReply(int sock,
ObjectID object_ids[],
int object_status[],
int64_t num_objects);
Status SendDataRequest(int sock,
ObjectID object_id,
const char *address,
int port);
Status SendDataReply(int sock,
ObjectID object_id,
int64_t object_size,
int64_t metadata_size);
Status ReadDataRequest(uint8_t *data,
size_t size,
ObjectID *object_id,
char **address,
int *port);
Status ReadDataReply(uint8_t *data,
size_t size,
ObjectID *object_id,
int64_t *object_size,
int64_t *metadata_size);
Status ReadFetchRequest(uint8_t *data,
size_t size,
std::vector<ObjectID> &object_ids);
Status ReadStatusRequest(uint8_t *data,
size_t size,
ObjectID object_ids[],
int64_t num_objects);
Status ReadWaitRequest(uint8_t *data,
size_t size,
ObjectRequestMap &object_requests,
int64_t *timeout_ms,
int *num_ready_objects);
Status ReadStatusRequest(uint8_t *data,
size_t size,
ObjectID object_ids[],
int64_t num_objects);
std::unique_ptr<uint8_t[]> CreateObjectInfoBuffer(
flatbuf::ObjectInfoT *object_info);
} // namespace plasma
#endif
src/plasma/setup-env.sh
-5
View File
@@ -1,5 +0,0 @@
echo "Adding Plasma to PYTHONPATH" 1>&2
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
export PYTHONPATH="$ROOT_DIR/lib/python/:$PYTHONPATH"
src/plasma/setup.py
-35
View File
@@ -1,35 +0,0 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from setuptools import setup, find_packages
import setuptools.command.install as _install
import subprocess
class install(_install.install):
def run(self):
subprocess.check_call(["make"])
subprocess.check_call(["cp", "build/plasma_store",
"plasma/plasma_store"])
subprocess.check_call(["cp", "build/plasma_manager",
"plasma/plasma_manager"])
subprocess.check_call(["cmake", ".."], cwd="./build")
subprocess.check_call(["make", "install"], cwd="./build")
# Calling _install.install.run(self) does not fetch required packages
# and instead performs an old-style install. See command/install.py in
# setuptools. So, calling do_egg_install() manually here.
self.do_egg_install()
setup(name="Plasma",
version="0.0.1",
description="Plasma client for Python",
packages=find_packages(),
package_data={"plasma": ["plasma_store",
"plasma_manager",
"libplasma.so"]},
cmdclass={"install": install},
include_package_data=True,
zip_safe=False)
src/plasma/test/client_tests.cc
-337
View File
@@ -1,337 +0,0 @@
#include "greatest.h"
#include <assert.h>
#include <unistd.h>
#include <sys/time.h>
#include "plasma/test-util.h"
#include "plasma/common.h"
#include "plasma/client.h"
using namespace plasma;
SUITE(plasma_client_tests);
TEST plasma_status_tests(void) {
PlasmaClient client1;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
plasma::kPlasmaDefaultReleaseDelay));
PlasmaClient client2;
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
plasma::kPlasmaDefaultReleaseDelay));
ObjectID oid1 = random_object_id();
/* Test for object non-existence. */
int status;
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT(status == static_cast<int>(ObjectLocation::Nonexistent));
/* Test for the object being in local Plasma store. */
/* First create object. */
int64_t data_size = 100;
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
std::shared_ptr<Buffer> data;
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
ARROW_CHECK_OK(client1.Seal(oid1));
/* Sleep to avoid race condition of Plasma Manager waiting for notification.
*/
sleep(1);
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT(status == static_cast<int>(ObjectLocation::Local));
/* Test for object being remote. */
ARROW_CHECK_OK(client2.Info(oid1, &status));
ASSERT(status == static_cast<int>(ObjectLocation::Remote));
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
TEST plasma_fetch_tests(void) {
PlasmaClient client1;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
plasma::kPlasmaDefaultReleaseDelay));
PlasmaClient client2;
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
plasma::kPlasmaDefaultReleaseDelay));
ObjectID oid1 = random_object_id();
/* Test for object non-existence. */
int status;
/* No object in the system */
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT(status == static_cast<int>(ObjectLocation::Nonexistent));
/* Test for the object being in local Plasma store. */
/* First create object. */
int64_t data_size = 100;
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
std::shared_ptr<Buffer> data;
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
ARROW_CHECK_OK(client1.Seal(oid1));
/* Object with ID oid1 has been just inserted. On the next fetch we might
* either find the object or not, depending on whether the Plasma Manager has
* received the notification from the Plasma Store or not. */
ObjectID oid_array1[1] = {oid1};
ARROW_CHECK_OK(client1.Fetch(1, oid_array1));
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT(status == static_cast<int>(ObjectLocation::Local) ||
status == static_cast<int>(ObjectLocation::Nonexistent));
/* Sleep to make sure Plasma Manager got the notification. */
sleep(1);
ARROW_CHECK_OK(client1.Info(oid1, &status));
ASSERT(status == static_cast<int>(ObjectLocation::Local));
/* Test for object being remote. */
ARROW_CHECK_OK(client2.Info(oid1, &status));
ASSERT(status == static_cast<int>(ObjectLocation::Remote));
/* Sleep to make sure the object has been fetched and it is now stored in the
* local Plasma Store. */
ARROW_CHECK_OK(client2.Fetch(1, oid_array1));
sleep(1);
ARROW_CHECK_OK(client2.Info(oid1, &status));
ASSERT(status == static_cast<int>(ObjectLocation::Local));
sleep(1);
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
void init_data_123(uint8_t *data, uint64_t size, uint8_t base) {
for (size_t i = 0; i < size; i++) {
data[i] = base + i;
}
}
bool is_equal_data_123(const uint8_t *data1,
const uint8_t *data2,
uint64_t size) {
for (size_t i = 0; i < size; i++) {
if (data1[i] != data2[i]) {
return false;
};
}
return true;
}
TEST plasma_nonblocking_get_tests(void) {
PlasmaClient client;
ARROW_CHECK_OK(client.Connect("/tmp/store1", "/tmp/manager1",
plasma::kPlasmaDefaultReleaseDelay));
ObjectID oid = random_object_id();
ObjectID oid_array[1] = {oid};
ObjectBuffer obj_buffer;
/* Test for object non-existence. */
ARROW_CHECK_OK(client.Get(oid_array, 1, 0, &obj_buffer));
ASSERT(obj_buffer.data == nullptr);
/* Test for the object being in local Plasma store. */
/* First create object. */
int64_t data_size = 4;
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
std::shared_ptr<Buffer> data;
ARROW_CHECK_OK(client.Create(oid, data_size, metadata, metadata_size, &data));
init_data_123(data->mutable_data(), data_size, 0);
ARROW_CHECK_OK(client.Seal(oid));
sleep(1);
ARROW_CHECK_OK(client.Get(oid_array, 1, 0, &obj_buffer));
ASSERT(is_equal_data_123(data->data(), obj_buffer.data->data(), data_size) ==
true);
sleep(1);
ARROW_CHECK_OK(client.Disconnect());
PASS();
}
TEST plasma_wait_for_objects_tests(void) {
PlasmaClient client1;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
plasma::kPlasmaDefaultReleaseDelay));
PlasmaClient client2;
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
plasma::kPlasmaDefaultReleaseDelay));
ObjectID oid1 = random_object_id();
ObjectID oid2 = random_object_id();
#define NUM_OBJ_REQUEST 2
#define WAIT_TIMEOUT_MS 1000
ObjectRequest obj_requests[NUM_OBJ_REQUEST];
obj_requests[0].object_id = oid1;
obj_requests[0].type = ObjectRequestType::PLASMA_QUERY_ANYWHERE;
obj_requests[1].object_id = oid2;
obj_requests[1].type = ObjectRequestType::PLASMA_QUERY_ANYWHERE;
struct timeval start, end;
gettimeofday(&start, NULL);
int n;
ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, &n));
ASSERT(n == 0);
gettimeofday(&end, NULL);
float diff_ms = (end.tv_sec - start.tv_sec);
diff_ms = (((diff_ms * 1000000.) + end.tv_usec) - (start.tv_usec)) / 1000.;
/* Reduce threshold by 10% to make sure we pass consistently. */
ASSERT(diff_ms > WAIT_TIMEOUT_MS * 0.9);
/* Create and insert an object in plasma_conn1. */
int64_t data_size = 4;
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
std::shared_ptr<Buffer> data;
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
ARROW_CHECK_OK(client1.Seal(oid1));
ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, &n));
ASSERT(n == 1);
/* Create and insert an object in client2. */
ARROW_CHECK_OK(
client2.Create(oid2, data_size, metadata, metadata_size, &data));
ARROW_CHECK_OK(client2.Seal(oid2));
ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, &n));
ASSERT(n == 2);
ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, &n));
ASSERT(n == 2);
obj_requests[0].type = ObjectRequestType::PLASMA_QUERY_LOCAL;
obj_requests[1].type = ObjectRequestType::PLASMA_QUERY_LOCAL;
ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, &n));
ASSERT(n == 1);
ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
WAIT_TIMEOUT_MS, &n));
ASSERT(n == 1);
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
TEST plasma_get_tests(void) {
PlasmaClient client1, client2;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
plasma::kPlasmaDefaultReleaseDelay));
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
plasma::kPlasmaDefaultReleaseDelay));
ObjectID oid1 = random_object_id();
ObjectID oid2 = random_object_id();
ObjectBuffer obj_buffer1;
ObjectID oid_array1[1] = {oid1};
ObjectID oid_array2[1] = {oid2};
int64_t data_size = 4;
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
std::shared_ptr<Buffer> data;
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
init_data_123(data->mutable_data(), data_size, 1);
ARROW_CHECK_OK(client1.Seal(oid1));
ARROW_CHECK_OK(client1.Get(oid_array1, 1, -1, &obj_buffer1));
ASSERT(data->data()[0] == obj_buffer1.data->data()[0]);
ObjectBuffer obj_buffer2;
ARROW_CHECK_OK(
client2.Create(oid2, data_size, metadata, metadata_size, &data));
init_data_123(data->mutable_data(), data_size, 2);
ARROW_CHECK_OK(client2.Seal(oid2));
ARROW_CHECK_OK(client1.Fetch(1, oid_array2));
ARROW_CHECK_OK(client1.Get(oid_array2, 1, -1, &obj_buffer2));
ASSERT(data->data()[0] == obj_buffer2.data->data()[0]);
sleep(1);
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
TEST plasma_get_multiple_tests(void) {
PlasmaClient client1, client2;
ARROW_CHECK_OK(client1.Connect("/tmp/store1", "/tmp/manager1",
plasma::kPlasmaDefaultReleaseDelay));
ARROW_CHECK_OK(client2.Connect("/tmp/store2", "/tmp/manager2",
plasma::kPlasmaDefaultReleaseDelay));
ObjectID oid1 = random_object_id();
ObjectID oid2 = random_object_id();
ObjectID obj_ids[NUM_OBJ_REQUEST];
ObjectBuffer obj_buffer[NUM_OBJ_REQUEST];
int obj1_first = 1, obj2_first = 2;
obj_ids[0] = oid1;
obj_ids[1] = oid2;
int64_t data_size = 4;
uint8_t metadata[] = {5};
int64_t metadata_size = sizeof(metadata);
std::shared_ptr<Buffer> data;
ARROW_CHECK_OK(
client1.Create(oid1, data_size, metadata, metadata_size, &data));
init_data_123(data->mutable_data(), data_size, obj1_first);
ARROW_CHECK_OK(client1.Seal(oid1));
/* This only waits for oid1. */
ARROW_CHECK_OK(client1.Get(obj_ids, 1, -1, obj_buffer));
ASSERT(data->data()[0] == obj_buffer[0].data->data()[0]);
ARROW_CHECK_OK(
client2.Create(oid2, data_size, metadata, metadata_size, &data));
init_data_123(data->mutable_data(), data_size, obj2_first);
ARROW_CHECK_OK(client2.Seal(oid2));
ARROW_CHECK_OK(client1.Fetch(2, obj_ids));
ARROW_CHECK_OK(client1.Get(obj_ids, 2, -1, obj_buffer));
ASSERT(obj1_first == obj_buffer[0].data->data()[0]);
ASSERT(obj2_first == obj_buffer[1].data->data()[0]);
sleep(1);
ARROW_CHECK_OK(client1.Disconnect());
ARROW_CHECK_OK(client2.Disconnect());
PASS();
}
SUITE(plasma_client_tests) {
RUN_TEST(plasma_status_tests);
RUN_TEST(plasma_fetch_tests);
RUN_TEST(plasma_nonblocking_get_tests);
RUN_TEST(plasma_wait_for_objects_tests);
RUN_TEST(plasma_get_tests);
RUN_TEST(plasma_get_multiple_tests);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(plasma_client_tests);
GREATEST_MAIN_END();
}
src/plasma/test/manager_tests.cc
-313
View File
@@ -1,313 +0,0 @@
#include "greatest.h"
#include <assert.h>
#include <unistd.h>
#include <poll.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <string>
#include "plasma/test-util.h"
#include "common.h"
#include "test/test_common.h"
#include "event_loop.h"
#include "io.h"
#include "../plasma_manager.h"
#include "plasma/client.h"
#include "../protocol.h"
namespace fb = plasma::flatbuf;
SUITE(plasma_manager_tests);
const char *plasma_store_socket_name = "/tmp/plasma_store_socket_1";
const char *plasma_manager_socket_name_format = "/tmp/plasma_manager_socket_%d";
const char *manager_addr = "127.0.0.1";
ObjectID object_id;
void wait_for_pollin(int fd) {
struct pollfd poll_list[1];
poll_list[0].fd = fd;
poll_list[0].events = POLLIN;
int retval = poll(poll_list, (unsigned long) 1, -1);
RAY_CHECK(retval > 0);
}
int test_done_handler(event_loop *loop, timer_id id, void *context) {
event_loop_stop(loop);
return AE_NOMORE;
}
typedef struct {
int port;
/** Connection to the manager's TCP socket. */
int manager_remote_fd;
/** Connection to the manager's Unix socket. */
int manager_local_fd;
int local_store;
int manager;
PlasmaManagerState *state;
event_loop *loop;
/* Accept a connection from the local manager on the remote manager. */
ClientConnection *write_conn;
ClientConnection *read_conn;
/* Connect a new client to the local plasma manager and mock a request to an
* object. */
plasma::PlasmaClient *plasma_client;
ClientConnection *client_conn;
} plasma_mock;
plasma_mock *init_plasma_mock(plasma_mock *remote_mock) {
plasma_mock *mock = (plasma_mock *) malloc(sizeof(plasma_mock));
/* Start listening on all the ports and initiate the local plasma manager. */
mock->port = bind_inet_sock_retry(&mock->manager_remote_fd);
mock->local_store = connect_ipc_sock_retry(plasma_store_socket_name, 5, 100);
std::string manager_socket_name = bind_ipc_sock_retry(
plasma_manager_socket_name_format, &mock->manager_local_fd);
RAY_CHECK(mock->manager_local_fd >= 0 && mock->local_store >= 0);
mock->state = PlasmaManagerState_init(plasma_store_socket_name,
manager_socket_name.c_str(),
manager_addr, mock->port, NULL, 0);
mock->loop = get_event_loop(mock->state);
/* Accept a connection from the local manager on the remote manager. */
if (remote_mock != NULL) {
mock->write_conn =
get_manager_connection(remote_mock->state, manager_addr, mock->port);
wait_for_pollin(mock->manager_remote_fd);
mock->read_conn = ClientConnection_listen(
mock->loop, mock->manager_remote_fd, mock->state,
plasma::kPlasmaDefaultReleaseDelay);
} else {
mock->write_conn = NULL;
mock->read_conn = NULL;
}
/* Connect a new client to the local plasma manager and mock a request to an
* object. */
mock->plasma_client = new plasma::PlasmaClient();
ARROW_CHECK_OK(mock->plasma_client->Connect(plasma_store_socket_name,
manager_socket_name.c_str(), 0));
wait_for_pollin(mock->manager_local_fd);
mock->client_conn = ClientConnection_listen(
mock->loop, mock->manager_local_fd, mock->state, 0);
return mock;
}
void destroy_plasma_mock(plasma_mock *mock) {
PlasmaManagerState_free(mock->state);
ARROW_CHECK_OK(mock->plasma_client->Disconnect());
delete mock->plasma_client;
close(mock->local_store);
close(mock->manager_local_fd);
close(mock->manager_remote_fd);
free(mock);
}
/**
* This test checks correct behavior of request_transfer in a non-failure
* scenario. Specifically, when one plasma manager calls request_transfer, the
* correct remote manager should receive the correct message. The test:
* - Buffer a transfer request for the remote manager.
* - Start and stop the event loop to make sure that we send the buffered
* request.
* - Expect to see a fb::MessageType::PlasmaDataRequest message on the remote
* manager with the correct object ID.
*/
TEST request_transfer_test(void) {
plasma_mock *local_mock = init_plasma_mock(NULL);
plasma_mock *remote_mock = init_plasma_mock(local_mock);
std::vector<std::string> manager_vector;
manager_vector.push_back(std::string("127.0.0.1:") +
std::to_string(remote_mock->port));
call_request_transfer(object_id, manager_vector, local_mock->state);
event_loop_add_timer(local_mock->loop,
RayConfig::instance().manager_timeout_milliseconds(),
test_done_handler, local_mock->state);
event_loop_run(local_mock->loop);
int read_fd = get_client_sock(remote_mock->read_conn);
std::vector<uint8_t> request_data;
ARROW_CHECK_OK(plasma::PlasmaReceive(
read_fd, fb::MessageType::PlasmaDataRequest, &request_data));
plasma::ObjectID object_id2;
char *address;
int port;
ARROW_CHECK_OK(plasma::ReadDataRequest(
request_data.data(), request_data.size(), &object_id2, &address, &port));
ASSERT(object_id == object_id2);
free(address);
/* Clean up. */
destroy_plasma_mock(remote_mock);
destroy_plasma_mock(local_mock);
PASS();
}
/**
* This test checks correct behavior of request_transfer in a scenario when the
* first manager we try times out. Specifically, when one plasma manager calls
* request_transfer on a list of remote managers and the first manager isn't
* reachable, the second remote manager should receive the correct message
* after the timeout. The test:
* - Buffer a transfer request for the remote managers.
* - Start and stop the event loop after a timeout to make sure that we
* trigger the timeout on the first manager.
* - Expect to see a fb::MessageType::PlasmaDataRequest message on the second
* remote manager with the correct object ID.
*/
TEST request_transfer_retry_test(void) {
plasma_mock *local_mock = init_plasma_mock(NULL);
plasma_mock *remote_mock1 = init_plasma_mock(local_mock);
plasma_mock *remote_mock2 = init_plasma_mock(local_mock);
std::vector<std::string> manager_vector;
manager_vector.push_back(std::string("127.0.0.1:") +
std::to_string(remote_mock1->port));
manager_vector.push_back(std::string("127.0.0.1:") +
std::to_string(remote_mock2->port));
call_request_transfer(object_id, manager_vector, local_mock->state);
event_loop_add_timer(local_mock->loop,
RayConfig::instance().manager_timeout_milliseconds() * 2,
test_done_handler, local_mock->state);
/* Register the fetch timeout handler. This is normally done when the plasma
* manager is started. It is needed here so that retries will happen when
* fetch requests time out. */
event_loop_add_timer(local_mock->loop,
RayConfig::instance().manager_timeout_milliseconds(),
fetch_timeout_handler, local_mock->state);
event_loop_run(local_mock->loop);
int read_fd = get_client_sock(remote_mock2->read_conn);
std::vector<uint8_t> request_data;
ARROW_CHECK_OK(plasma::PlasmaReceive(
read_fd, fb::MessageType::PlasmaDataRequest, &request_data));
plasma::ObjectID object_id2;
char *address;
int port;
ARROW_CHECK_OK(plasma::ReadDataRequest(
request_data.data(), request_data.size(), &object_id2, &address, &port));
free(address);
ASSERT(object_id == object_id2);
/* Clean up. */
destroy_plasma_mock(remote_mock2);
destroy_plasma_mock(remote_mock1);
destroy_plasma_mock(local_mock);
PASS();
}
/**
* This test checks correct behavior of reading and writing an object chunk
* from one manager to another.
* - Write a one-chunk object from the local to the remote manager.
* - Read the object chunk on the remote manager.
* - Expect to see the same data.
*/
TEST read_write_object_chunk_test(void) {
plasma_mock *local_mock = init_plasma_mock(NULL);
plasma_mock *remote_mock = init_plasma_mock(local_mock);
/* Create a mock object buffer to transfer. */
const char *data = "Hello world!";
const int data_size = strlen(data) + 1;
const int metadata_size = 0;
PlasmaRequestBuffer remote_buf;
remote_buf.type = fb::MessageType::PlasmaDataReply;
remote_buf.object_id = object_id;
remote_buf.data = (uint8_t *) data;
remote_buf.data_size = data_size;
remote_buf.metadata = (uint8_t *) data + data_size;
remote_buf.metadata_size = metadata_size;
PlasmaRequestBuffer local_buf;
local_buf.object_id = object_id;
local_buf.data_size = data_size;
local_buf.metadata_size = metadata_size;
local_buf.data = (uint8_t *) malloc(data_size);
/* The test:
* - Write the object data from the remote manager to the local.
* - Read the object data on the local manager.
* - Check that the data matches.
*/
ClientConnection_start_request(remote_mock->write_conn);
write_object_chunk(remote_mock->write_conn, &remote_buf);
ASSERT(ClientConnection_request_finished(remote_mock->write_conn));
/* Wait until the data is ready to be read. */
wait_for_pollin(get_client_sock(remote_mock->read_conn));
/* Read the data. */
ClientConnection_start_request(remote_mock->read_conn);
int err = read_object_chunk(remote_mock->read_conn, &local_buf);
ASSERT_EQ(err, 0);
ASSERT(ClientConnection_request_finished(remote_mock->read_conn));
ASSERT_EQ(memcmp(remote_buf.data, local_buf.data, data_size), 0);
/* Clean up. */
free(local_buf.data);
destroy_plasma_mock(remote_mock);
destroy_plasma_mock(local_mock);
PASS();
}
TEST object_notifications_test(void) {
plasma_mock *local_mock = init_plasma_mock(NULL);
/* Open a non-blocking socket pair to mock the object notifications from the
* plasma store. */
int fd[2];
socketpair(AF_UNIX, SOCK_STREAM, 0, fd);
int flags = fcntl(fd[1], F_GETFL, 0);
RAY_CHECK(fcntl(fd[1], F_SETFL, flags | O_NONBLOCK) == 0);
ObjectID object_id = plasma::random_object_id();
fb::ObjectInfoT info;
info.object_id = object_id.binary();
info.data_size = 10;
info.metadata_size = 1;
info.create_time = 0;
info.construct_duration = 0;
info.digest = std::string("0");
info.is_deletion = false;
/* Check that the object is not local at first. */
bool is_local = is_object_local(local_mock->state, object_id);
ASSERT(!is_local);
/* Check that the object is local after receiving an object notification. */
auto notification = plasma::CreateObjectInfoBuffer(&info);
int64_t size = *((int64_t *) notification.get());
send(fd[1], notification.get(), sizeof(int64_t) + size, 0);
process_object_notification(local_mock->loop, fd[0], local_mock->state, 0);
is_local = is_object_local(local_mock->state, object_id);
ASSERT(is_local);
/* Check that the object is not local after receiving a notification about
* the object deletion. */
info.is_deletion = true;
notification = plasma::CreateObjectInfoBuffer(&info);
size = *((int64_t *) notification.get());
send(fd[1], notification.get(), sizeof(int64_t) + size, 0);
process_object_notification(local_mock->loop, fd[0], local_mock->state, 0);
is_local = is_object_local(local_mock->state, object_id);
ASSERT(!is_local);
/* Clean up. */
close(fd[0]);
close(fd[1]);
destroy_plasma_mock(local_mock);
PASS();
}
SUITE(plasma_manager_tests) {
memset(&object_id, 1, sizeof(object_id));
RUN_TEST(request_transfer_test);
RUN_TEST(request_transfer_retry_test);
RUN_TEST(read_write_object_chunk_test);
RUN_TEST(object_notifications_test);
}
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN();
RUN_SUITE(plasma_manager_tests);
GREATEST_MAIN_END();
}
src/plasma/test/run_tests.sh
-59
View File
@@ -1,59 +0,0 @@
#!/usr/bin/env bash
# Cause the script to exit if a single command fails.
set -e
./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 0 &
sleep 1
./src/plasma/manager_tests
killall plasma_store_server
LaunchRedis() {
port=$1
if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
./src/credis/redis/src/redis-server \
--loglevel warning \
--loadmodule ./src/credis/build/src/libmember.so \
--loadmodule ./src/common/redis_module/libray_redis_module.so \
--port $port &
else
./src/common/thirdparty/redis/src/redis-server \
--loglevel warning \
--loadmodule ./src/common/redis_module/libray_redis_module.so \
--port $port &
fi
}
# Start the Redis shards.
LaunchRedis 6379
redis_pid1=$!
LaunchRedis 6380
redis_pid2=$!
sleep 1s
# Flush the redis server
./src/common/thirdparty/redis/src/redis-cli flushall
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
sleep 1
./src/plasma/plasma_store_server -s /tmp/store1 -m 1000000000 &
plasma1_pid=$!
./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 &
plasma2_pid=$!
./src/plasma/plasma_store_server -s /tmp/store2 -m 1000000000 &
plasma3_pid=$!
./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 &
plasma4_pid=$!
sleep 1
./src/plasma/client_tests
kill $plasma4_pid
kill $plasma3_pid
kill $plasma2_pid
kill $plasma1_pid
kill $redis_pid1
wait $redis_pid1
kill $redis_pid2
wait $redis_pid2
src/plasma/test/run_valgrind.sh
-11
View File
@@ -1,11 +0,0 @@
#!/usr/bin/env bash
set -x
# Cause the script to exit if a single command fails.
set -e
./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 0 &
sleep 1
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/plasma/manager_tests
killall plasma_store_server
src/plasma/thirdparty/ae/ae.c
Vendored
-465
View File
@@ -1,465 +0,0 @@
/* A simple event-driven programming library. Originally I wrote this code
* for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
* it in form of a library for easy reuse.
*
* Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <stdlib.h>
#include <poll.h>
#include <string.h>
#include <time.h>
#include <errno.h>
#include "ae.h"
#include "zmalloc.h"
#include "config.h"
/* Include the best multiplexing layer supported by this system.
* The following should be ordered by performances, descending. */
#ifdef HAVE_EVPORT
#include "ae_evport.c"
#else
#ifdef HAVE_EPOLL
#include "ae_epoll.c"
#else
#ifdef HAVE_KQUEUE
#include "ae_kqueue.c"
#else
#include "ae_select.c"
#endif
#endif
#endif
aeEventLoop *aeCreateEventLoop(int setsize) {
aeEventLoop *eventLoop;
int i;
if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err;
eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
eventLoop->setsize = setsize;
eventLoop->lastTime = time(NULL);
eventLoop->timeEventHead = NULL;
eventLoop->timeEventNextId = 0;
eventLoop->stop = 0;
eventLoop->maxfd = -1;
eventLoop->beforesleep = NULL;
if (aeApiCreate(eventLoop) == -1) goto err;
/* Events with mask == AE_NONE are not set. So let's initialize the
* vector with it. */
for (i = 0; i < setsize; i++)
eventLoop->events[i].mask = AE_NONE;
return eventLoop;
err:
if (eventLoop) {
zfree(eventLoop->events);
zfree(eventLoop->fired);
zfree(eventLoop);
}
return NULL;
}
/* Return the current set size. */
int aeGetSetSize(aeEventLoop *eventLoop) {
return eventLoop->setsize;
}
/* Resize the maximum set size of the event loop.
* If the requested set size is smaller than the current set size, but
* there is already a file descriptor in use that is >= the requested
* set size minus one, AE_ERR is returned and the operation is not
* performed at all.
*
* Otherwise AE_OK is returned and the operation is successful. */
int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) {
int i;
if (setsize == eventLoop->setsize) return AE_OK;
if (eventLoop->maxfd >= setsize) return AE_ERR;
if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR;
eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
eventLoop->setsize = setsize;
/* Make sure that if we created new slots, they are initialized with
* an AE_NONE mask. */
for (i = eventLoop->maxfd+1; i < setsize; i++)
eventLoop->events[i].mask = AE_NONE;
return AE_OK;
}
void aeDeleteEventLoop(aeEventLoop *eventLoop) {
aeApiFree(eventLoop);
zfree(eventLoop->events);
zfree(eventLoop->fired);
zfree(eventLoop);
}
void aeStop(aeEventLoop *eventLoop) {
eventLoop->stop = 1;
}
int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
aeFileProc *proc, void *clientData)
{
if (fd >= eventLoop->setsize) {
errno = ERANGE;
return AE_ERR;
}
aeFileEvent *fe = &eventLoop->events[fd];
if (aeApiAddEvent(eventLoop, fd, mask) == -1)
return AE_ERR;
fe->mask |= mask;
if (mask & AE_READABLE) fe->rfileProc = proc;
if (mask & AE_WRITABLE) fe->wfileProc = proc;
fe->clientData = clientData;
if (fd > eventLoop->maxfd)
eventLoop->maxfd = fd;
return AE_OK;
}
void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
{
if (fd >= eventLoop->setsize) return;
aeFileEvent *fe = &eventLoop->events[fd];
if (fe->mask == AE_NONE) return;
aeApiDelEvent(eventLoop, fd, mask);
fe->mask = fe->mask & (~mask);
if (fd == eventLoop->maxfd && fe->mask == AE_NONE) {
/* Update the max fd */
int j;
for (j = eventLoop->maxfd-1; j >= 0; j--)
if (eventLoop->events[j].mask != AE_NONE) break;
eventLoop->maxfd = j;
}
}
int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
if (fd >= eventLoop->setsize) return 0;
aeFileEvent *fe = &eventLoop->events[fd];
return fe->mask;
}
static void aeGetTime(long *seconds, long *milliseconds)
{
struct timeval tv;
gettimeofday(&tv, NULL);
*seconds = tv.tv_sec;
*milliseconds = tv.tv_usec/1000;
}
static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) {
long cur_sec, cur_ms, when_sec, when_ms;
aeGetTime(&cur_sec, &cur_ms);
when_sec = cur_sec + milliseconds/1000;
when_ms = cur_ms + milliseconds%1000;
if (when_ms >= 1000) {
when_sec ++;
when_ms -= 1000;
}
*sec = when_sec;
*ms = when_ms;
}
long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
aeTimeProc *proc, void *clientData,
aeEventFinalizerProc *finalizerProc)
{
long long id = eventLoop->timeEventNextId++;
aeTimeEvent *te;
te = zmalloc(sizeof(*te));
if (te == NULL) return AE_ERR;
te->id = id;
aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms);
te->timeProc = proc;
te->finalizerProc = finalizerProc;
te->clientData = clientData;
te->next = eventLoop->timeEventHead;
eventLoop->timeEventHead = te;
return id;
}
int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
{
aeTimeEvent *te = eventLoop->timeEventHead;
while(te) {
if (te->id == id) {
te->id = AE_DELETED_EVENT_ID;
return AE_OK;
}
te = te->next;
}
return AE_ERR; /* NO event with the specified ID found */
}
/* Search the first timer to fire.
* This operation is useful to know how many time the select can be
* put in sleep without to delay any event.
* If there are no timers NULL is returned.
*
* Note that's O(N) since time events are unsorted.
* Possible optimizations (not needed by Redis so far, but...):
* 1) Insert the event in order, so that the nearest is just the head.
* Much better but still insertion or deletion of timers is O(N).
* 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)).
*/
static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop)
{
aeTimeEvent *te = eventLoop->timeEventHead;
aeTimeEvent *nearest = NULL;
while(te) {
if (!nearest || te->when_sec < nearest->when_sec ||
(te->when_sec == nearest->when_sec &&
te->when_ms < nearest->when_ms))
nearest = te;
te = te->next;
}
return nearest;
}
/* Process time events */
static int processTimeEvents(aeEventLoop *eventLoop) {
int processed = 0;
aeTimeEvent *te, *prev;
long long maxId;
time_t now = time(NULL);
/* If the system clock is moved to the future, and then set back to the
* right value, time events may be delayed in a random way. Often this
* means that scheduled operations will not be performed soon enough.
*
* Here we try to detect system clock skews, and force all the time
* events to be processed ASAP when this happens: the idea is that
* processing events earlier is less dangerous than delaying them
* indefinitely, and practice suggests it is. */
if (now < eventLoop->lastTime) {
te = eventLoop->timeEventHead;
while(te) {
te->when_sec = 0;
te = te->next;
}
}
eventLoop->lastTime = now;
prev = NULL;
te = eventLoop->timeEventHead;
maxId = eventLoop->timeEventNextId-1;
while(te) {
long now_sec, now_ms;
long long id;
/* Remove events scheduled for deletion. */
if (te->id == AE_DELETED_EVENT_ID) {
aeTimeEvent *next = te->next;
if (prev == NULL)
eventLoop->timeEventHead = te->next;
else
prev->next = te->next;
if (te->finalizerProc)
te->finalizerProc(eventLoop, te->clientData);
zfree(te);
te = next;
continue;
}
/* Make sure we don't process time events created by time events in
* this iteration. Note that this check is currently useless: we always
* add new timers on the head, however if we change the implementation
* detail, this check may be useful again: we keep it here for future
* defense. */
if (te->id > maxId) {
te = te->next;
continue;
}
aeGetTime(&now_sec, &now_ms);
if (now_sec > te->when_sec ||
(now_sec == te->when_sec && now_ms >= te->when_ms))
{
int retval;
id = te->id;
retval = te->timeProc(eventLoop, id, te->clientData);
processed++;
if (retval != AE_NOMORE) {
aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
} else {
te->id = AE_DELETED_EVENT_ID;
}
}
prev = te;
te = te->next;
}
return processed;
}
/* Process every pending time event, then every pending file event
* (that may be registered by time event callbacks just processed).
* Without special flags the function sleeps until some file event
* fires, or when the next time event occurs (if any).
*
* If flags is 0, the function does nothing and returns.
* if flags has AE_ALL_EVENTS set, all the kind of events are processed.
* if flags has AE_FILE_EVENTS set, file events are processed.
* if flags has AE_TIME_EVENTS set, time events are processed.
* if flags has AE_DONT_WAIT set the function returns ASAP until all
* the events that's possible to process without to wait are processed.
*
* The function returns the number of events processed. */
int aeProcessEvents(aeEventLoop *eventLoop, int flags)
{
int processed = 0, numevents;
/* Nothing to do? return ASAP */
if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0;
/* Note that we want call select() even if there are no
* file events to process as long as we want to process time
* events, in order to sleep until the next time event is ready
* to fire. */
if (eventLoop->maxfd != -1 ||
((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) {
int j;
aeTimeEvent *shortest = NULL;
struct timeval tv, *tvp;
if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT))
shortest = aeSearchNearestTimer(eventLoop);
if (shortest) {
long now_sec, now_ms;
aeGetTime(&now_sec, &now_ms);
tvp = &tv;
/* How many milliseconds we need to wait for the next
* time event to fire? */
long long ms =
(shortest->when_sec - now_sec)*1000 +
shortest->when_ms - now_ms;
if (ms > 0) {
tvp->tv_sec = ms/1000;
tvp->tv_usec = (ms % 1000)*1000;
} else {
tvp->tv_sec = 0;
tvp->tv_usec = 0;
}
} else {
/* If we have to check for events but need to return
* ASAP because of AE_DONT_WAIT we need to set the timeout
* to zero */
if (flags & AE_DONT_WAIT) {
tv.tv_sec = tv.tv_usec = 0;
tvp = &tv;
} else {
/* Otherwise we can block */
tvp = NULL; /* wait forever */
}
}
numevents = aeApiPoll(eventLoop, tvp);
for (j = 0; j < numevents; j++) {
aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd];
int mask = eventLoop->fired[j].mask;
int fd = eventLoop->fired[j].fd;
int rfired = 0;
/* note the fe->mask & mask & ... code: maybe an already processed
* event removed an element that fired and we still didn't
* processed, so we check if the event is still valid. */
if (fe->mask & mask & AE_READABLE) {
rfired = 1;
fe->rfileProc(eventLoop,fd,fe->clientData,mask);
}
if (fe->mask & mask & AE_WRITABLE) {
if (!rfired || fe->wfileProc != fe->rfileProc)
fe->wfileProc(eventLoop,fd,fe->clientData,mask);
}
processed++;
}
}
/* Check time events */
if (flags & AE_TIME_EVENTS)
processed += processTimeEvents(eventLoop);
return processed; /* return the number of processed file/time events */
}
/* Wait for milliseconds until the given file descriptor becomes
* writable/readable/exception */
int aeWait(int fd, int mask, long long milliseconds) {
struct pollfd pfd;
int retmask = 0, retval;
memset(&pfd, 0, sizeof(pfd));
pfd.fd = fd;
if (mask & AE_READABLE) pfd.events |= POLLIN;
if (mask & AE_WRITABLE) pfd.events |= POLLOUT;
if ((retval = poll(&pfd, 1, milliseconds))== 1) {
if (pfd.revents & POLLIN) retmask |= AE_READABLE;
if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE;
if (pfd.revents & POLLERR) retmask |= AE_WRITABLE;
if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE;
return retmask;
} else {
return retval;
}
}
void aeMain(aeEventLoop *eventLoop) {
eventLoop->stop = 0;
while (!eventLoop->stop) {
if (eventLoop->beforesleep != NULL)
eventLoop->beforesleep(eventLoop);
aeProcessEvents(eventLoop, AE_ALL_EVENTS);
}
}
char *aeGetApiName(void) {
return aeApiName();
}
void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) {
eventLoop->beforesleep = beforesleep;
}
src/plasma/thirdparty/ae/ae.h
Vendored
-123
View File
@@ -1,123 +0,0 @@
/* A simple event-driven programming library. Originally I wrote this code
* for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
* it in form of a library for easy reuse.
*
* Copyright (c) 2006-2012, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __AE_H__
#define __AE_H__
#include <time.h>
#define AE_OK 0
#define AE_ERR -1
#define AE_NONE 0
#define AE_READABLE 1
#define AE_WRITABLE 2
#define AE_FILE_EVENTS 1
#define AE_TIME_EVENTS 2
#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
#define AE_DONT_WAIT 4
#define AE_NOMORE -1
#define AE_DELETED_EVENT_ID -1
/* Macros */
#define AE_NOTUSED(V) ((void) V)
struct aeEventLoop;
/* Types and data structures */
typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData);
typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData);
typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop);
/* File event structure */
typedef struct aeFileEvent {
int mask; /* one of AE_(READABLE|WRITABLE) */
aeFileProc *rfileProc;
aeFileProc *wfileProc;
void *clientData;
} aeFileEvent;
/* Time event structure */
typedef struct aeTimeEvent {
long long id; /* time event identifier. */
long when_sec; /* seconds */
long when_ms; /* milliseconds */
aeTimeProc *timeProc;
aeEventFinalizerProc *finalizerProc;
void *clientData;
struct aeTimeEvent *next;
} aeTimeEvent;
/* A fired event */
typedef struct aeFiredEvent {
int fd;
int mask;
} aeFiredEvent;
/* State of an event based program */
typedef struct aeEventLoop {
int maxfd; /* highest file descriptor currently registered */
int setsize; /* max number of file descriptors tracked */
long long timeEventNextId;
time_t lastTime; /* Used to detect system clock skew */
aeFileEvent *events; /* Registered events */
aeFiredEvent *fired; /* Fired events */
aeTimeEvent *timeEventHead;
int stop;
void *apidata; /* This is used for polling API specific data */
aeBeforeSleepProc *beforesleep;
} aeEventLoop;
/* Prototypes */
aeEventLoop *aeCreateEventLoop(int setsize);
void aeDeleteEventLoop(aeEventLoop *eventLoop);
void aeStop(aeEventLoop *eventLoop);
int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
aeFileProc *proc, void *clientData);
void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask);
int aeGetFileEvents(aeEventLoop *eventLoop, int fd);
long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
aeTimeProc *proc, void *clientData,
aeEventFinalizerProc *finalizerProc);
int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id);
int aeProcessEvents(aeEventLoop *eventLoop, int flags);
int aeWait(int fd, int mask, long long milliseconds);
void aeMain(aeEventLoop *eventLoop);
char *aeGetApiName(void);
void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep);
int aeGetSetSize(aeEventLoop *eventLoop);
int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
#endif
src/plasma/thirdparty/ae/ae_epoll.c
Vendored
-135
View File
@@ -1,135 +0,0 @@
/* Linux epoll(2) based ae.c module
*
* Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/epoll.h>
typedef struct aeApiState {
int epfd;
struct epoll_event *events;
} aeApiState;
static int aeApiCreate(aeEventLoop *eventLoop) {
aeApiState *state = zmalloc(sizeof(aeApiState));
if (!state) return -1;
state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize);
if (!state->events) {
zfree(state);
return -1;
}
state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */
if (state->epfd == -1) {
zfree(state->events);
zfree(state);
return -1;
}
eventLoop->apidata = state;
return 0;
}
static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
aeApiState *state = eventLoop->apidata;
state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize);
return 0;
}
static void aeApiFree(aeEventLoop *eventLoop) {
aeApiState *state = eventLoop->apidata;
close(state->epfd);
zfree(state->events);
zfree(state);
}
static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
aeApiState *state = eventLoop->apidata;
struct epoll_event ee = {0}; /* avoid valgrind warning */
/* If the fd was already monitored for some event, we need a MOD
* operation. Otherwise we need an ADD operation. */
int op = eventLoop->events[fd].mask == AE_NONE ?
EPOLL_CTL_ADD : EPOLL_CTL_MOD;
ee.events = 0;
mask |= eventLoop->events[fd].mask; /* Merge old events */
if (mask & AE_READABLE) ee.events |= EPOLLIN;
if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
ee.data.fd = fd;
if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1;
return 0;
}
static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) {
aeApiState *state = eventLoop->apidata;
struct epoll_event ee = {0}; /* avoid valgrind warning */
int mask = eventLoop->events[fd].mask & (~delmask);
ee.events = 0;
if (mask & AE_READABLE) ee.events |= EPOLLIN;
if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
ee.data.fd = fd;
if (mask != AE_NONE) {
epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee);
} else {
/* Note, Kernel < 2.6.9 requires a non null event pointer even for
* EPOLL_CTL_DEL. */
epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee);
}
}
static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
aeApiState *state = eventLoop->apidata;
int retval, numevents = 0;
retval = epoll_wait(state->epfd,state->events,eventLoop->setsize,
tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1);
if (retval > 0) {
int j;
numevents = retval;
for (j = 0; j < numevents; j++) {
int mask = 0;
struct epoll_event *e = state->events+j;
if (e->events & EPOLLIN) mask |= AE_READABLE;
if (e->events & EPOLLOUT) mask |= AE_WRITABLE;
if (e->events & EPOLLERR) mask |= AE_WRITABLE;
if (e->events & EPOLLHUP) mask |= AE_WRITABLE;
eventLoop->fired[j].fd = e->data.fd;
eventLoop->fired[j].mask = mask;
}
}
return numevents;
}
static char *aeApiName(void) {
return "epoll";
}
src/plasma/thirdparty/ae/ae_evport.c
Vendored
-320
View File
@@ -1,320 +0,0 @@
/* ae.c module for illumos event ports.
*
* Copyright (c) 2012, Joyent, Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <assert.h>
#include <errno.h>
#include <port.h>
#include <poll.h>
#include <sys/types.h>
#include <sys/time.h>
#include <stdio.h>
static int evport_debug = 0;
/*
* This file implements the ae API using event ports, present on Solaris-based
* systems since Solaris 10. Using the event port interface, we associate file
* descriptors with the port. Each association also includes the set of poll(2)
* events that the consumer is interested in (e.g., POLLIN and POLLOUT).
*
* There's one tricky piece to this implementation: when we return events via
* aeApiPoll, the corresponding file descriptors become dissociated from the
* port. This is necessary because poll events are level-triggered, so if the
* fd didn't become dissociated, it would immediately fire another event since
* the underlying state hasn't changed yet. We must re-associate the file
* descriptor, but only after we know that our caller has actually read from it.
* The ae API does not tell us exactly when that happens, but we do know that
* it must happen by the time aeApiPoll is called again. Our solution is to
* keep track of the last fds returned by aeApiPoll and re-associate them next
* time aeApiPoll is invoked.
*
* To summarize, in this module, each fd association is EITHER (a) represented
* only via the in-kernel association OR (b) represented by pending_fds and
* pending_masks. (b) is only true for the last fds we returned from aeApiPoll,
* and only until we enter aeApiPoll again (at which point we restore the
* in-kernel association).
*/
#define MAX_EVENT_BATCHSZ 512
typedef struct aeApiState {
int portfd; /* event port */
int npending; /* # of pending fds */
int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */
int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */
} aeApiState;
static int aeApiCreate(aeEventLoop *eventLoop) {
int i;
aeApiState *state = zmalloc(sizeof(aeApiState));
if (!state) return -1;
state->portfd = port_create();
if (state->portfd == -1) {
zfree(state);
return -1;
}
state->npending = 0;
for (i = 0; i < MAX_EVENT_BATCHSZ; i++) {
state->pending_fds[i] = -1;
state->pending_masks[i] = AE_NONE;
}
eventLoop->apidata = state;
return 0;
}
static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
/* Nothing to resize here. */
return 0;
}
static void aeApiFree(aeEventLoop *eventLoop) {
aeApiState *state = eventLoop->apidata;
close(state->portfd);
zfree(state);
}
static int aeApiLookupPending(aeApiState *state, int fd) {
int i;
for (i = 0; i < state->npending; i++) {
if (state->pending_fds[i] == fd)
return (i);
}
return (-1);
}
/*
* Helper function to invoke port_associate for the given fd and mask.
*/
static int aeApiAssociate(const char *where, int portfd, int fd, int mask) {
int events = 0;
int rv, err;
if (mask & AE_READABLE)
events |= POLLIN;
if (mask & AE_WRITABLE)
events |= POLLOUT;
if (evport_debug)
fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events);
rv = port_associate(portfd, PORT_SOURCE_FD, fd, events,
(void *)(uintptr_t)mask);
err = errno;
if (evport_debug)
fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err));
if (rv == -1) {
fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err));
if (err == EAGAIN)
fprintf(stderr, "aeApiAssociate: event port limit exceeded.");
}
return rv;
}
static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
aeApiState *state = eventLoop->apidata;
int fullmask, pfd;
if (evport_debug)
fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask);
/*
* Since port_associate's "events" argument replaces any existing events, we
* must be sure to include whatever events are already associated when
* we call port_associate() again.
*/
fullmask = mask | eventLoop->events[fd].mask;
pfd = aeApiLookupPending(state, fd);
if (pfd != -1) {
/*
* This fd was recently returned from aeApiPoll. It should be safe to
* assume that the consumer has processed that poll event, but we play
* it safer by simply updating pending_mask. The fd will be
* re-associated as usual when aeApiPoll is called again.
*/
if (evport_debug)
fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd);
state->pending_masks[pfd] |= fullmask;
return 0;
}
return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask));
}
static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
aeApiState *state = eventLoop->apidata;
int fullmask, pfd;
if (evport_debug)
fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask);
pfd = aeApiLookupPending(state, fd);
if (pfd != -1) {
if (evport_debug)
fprintf(stderr, "deleting event from pending fd %d\n", fd);
/*
* This fd was just returned from aeApiPoll, so it's not currently
* associated with the port. All we need to do is update
* pending_mask appropriately.
*/
state->pending_masks[pfd] &= ~mask;
if (state->pending_masks[pfd] == AE_NONE)
state->pending_fds[pfd] = -1;
return;
}
/*
* The fd is currently associated with the port. Like with the add case
* above, we must look at the full mask for the file descriptor before
* updating that association. We don't have a good way of knowing what the
* events are without looking into the eventLoop state directly. We rely on
* the fact that our caller has already updated the mask in the eventLoop.
*/
fullmask = eventLoop->events[fd].mask;
if (fullmask == AE_NONE) {
/*
* We're removing *all* events, so use port_dissociate to remove the
* association completely. Failure here indicates a bug.
*/
if (evport_debug)
fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd);
if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) {
perror("aeApiDelEvent: port_dissociate");
abort(); /* will not return */
}
} else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd,
fullmask) != 0) {
/*
* ENOMEM is a potentially transient condition, but the kernel won't
* generally return it unless things are really bad. EAGAIN indicates
* we've reached an resource limit, for which it doesn't make sense to
* retry (counter-intuitively). All other errors indicate a bug. In any
* of these cases, the best we can do is to abort.
*/
abort(); /* will not return */
}
}
static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
aeApiState *state = eventLoop->apidata;
struct timespec timeout, *tsp;
int mask, i;
uint_t nevents;
port_event_t event[MAX_EVENT_BATCHSZ];
/*
* If we've returned fd events before, we must re-associate them with the
* port now, before calling port_get(). See the block comment at the top of
* this file for an explanation of why.
*/
for (i = 0; i < state->npending; i++) {
if (state->pending_fds[i] == -1)
/* This fd has since been deleted. */
continue;
if (aeApiAssociate("aeApiPoll", state->portfd,
state->pending_fds[i], state->pending_masks[i]) != 0) {
/* See aeApiDelEvent for why this case is fatal. */
abort();
}
state->pending_masks[i] = AE_NONE;
state->pending_fds[i] = -1;
}
state->npending = 0;
if (tvp != NULL) {
timeout.tv_sec = tvp->tv_sec;
timeout.tv_nsec = tvp->tv_usec * 1000;
tsp = &timeout;
} else {
tsp = NULL;
}
/*
* port_getn can return with errno == ETIME having returned some events (!).
* So if we get ETIME, we check nevents, too.
*/
nevents = 1;
if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents,
tsp) == -1 && (errno != ETIME || nevents == 0)) {
if (errno == ETIME || errno == EINTR)
return 0;
/* Any other error indicates a bug. */
perror("aeApiPoll: port_get");
abort();
}
state->npending = nevents;
for (i = 0; i < nevents; i++) {
mask = 0;
if (event[i].portev_events & POLLIN)
mask |= AE_READABLE;
if (event[i].portev_events & POLLOUT)
mask |= AE_WRITABLE;
eventLoop->fired[i].fd = event[i].portev_object;
eventLoop->fired[i].mask = mask;
if (evport_debug)
fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n",
(int)event[i].portev_object, mask);
state->pending_fds[i] = event[i].portev_object;
state->pending_masks[i] = (uintptr_t)event[i].portev_user;
}
return nevents;
}
static char *aeApiName(void) {
return "evport";
}

Some files were not shown because too many files have changed in this diff Show More