diff --git a/.gitmodules b/.gitmodules index 9eaa0156b..89d94b72e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,17 +1,3 @@ -[submodule "thirdparty/grpc"] - path = thirdparty/grpc - url = https://github.com/grpc/grpc - ignore = dirty -[submodule "thirdparty/numbuf"] - path = thirdparty/numbuf - url = https://github.com/ray-project/numbuf.git [submodule "thirdparty/arrow"] path = thirdparty/arrow url = https://github.com/ray-project/arrow.git -[submodule "thirdparty/python"] - path = thirdparty/python - url = https://github.com/austinsc/python.git - ignore = dirty -[submodule "thirdparty/hiredis"] - path = thirdparty/hiredis - url = https://github.com/redis/hiredis.git diff --git a/src/common/Makefile b/src/common/Makefile new file mode 100644 index 000000000..6982b7945 --- /dev/null +++ b/src/common/Makefile @@ -0,0 +1,46 @@ +CC = gcc +CFLAGS = -g -Wall --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -fPIC -I. -Ithirdparty -Ithirdparty/ae -Wno-typedef-redefinition -Werror +BUILD = build + +all: hiredis $(BUILD)/libcommon.a + +$(BUILD)/libcommon.a: event_loop.o common.o task.o io.o state/redis.o thirdparty/ae/ae.o + ar rcs $@ $^ + +$(BUILD)/common_tests: test/common_tests.c $(BUILD)/libcommon.a + $(CC) -o $@ test/common_tests.c $(BUILD)/libcommon.a $(CFLAGS) + +$(BUILD)/db_tests: hiredis test/db_tests.c $(BUILD)/libcommon.a + $(CC) -o $@ test/db_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS) + +$(BUILD)/io_tests: test/io_tests.c $(BUILD)/libcommon.a + $(CC) -o $@ $^ $(CFLAGS) + +$(BUILD)/task_tests: test/task_tests.c $(BUILD)/libcommon.a + $(CC) -o $@ $^ $(CFLAGS) + +$(BUILD)/redis_tests: hiredis test/redis_tests.c $(BUILD)/libcommon.a logging.h + $(CC) -o $@ test/redis_tests.c logging.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS) + +clean: + rm -f *.o state/*.o test/*.o thirdparty/ae/*.o + rm -rf $(BUILD)/* + +redis: + cd thirdparty ; bash ./build-redis.sh + +hiredis: + git submodule update --init --recursive -- "thirdparty/hiredis" ; cd thirdparty/hiredis ; make + +test: hiredis redis $(BUILD)/common_tests $(BUILD)/db_tests $(BUILD)/io_tests $(BUILD)/task_tests $(BUILD)/redis_tests FORCE + ./thirdparty/redis-3.2.3/src/redis-server & + sleep 1s ; ./build/common_tests ; ./build/db_tests ; ./build/io_tests ; ./build/task_tests ; ./build/redis_tests + +valgrind: test + valgrind --leak-check=full --error-exitcode=1 ./build/common_tests + valgrind --leak-check=full --error-exitcode=1 ./build/db_tests + valgrind --leak-check=full --error-exitcode=1 ./build/io_tests + valgrind --leak-check=full --error-exitcode=1 ./build/task_tests + valgrind --leak-check=full --error-exitcode=1 ./build/redis_tests + +FORCE: diff --git a/src/common/build/.gitkeep b/src/common/build/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/src/common/common.c b/src/common/common.c new file mode 100644 index 000000000..d9fe1e951 --- /dev/null +++ b/src/common/common.c @@ -0,0 +1,36 @@ +#include "common.h" + +#include +#include +#include +#include +#include + +const unique_id NIL_ID = {{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}}; + +unique_id globally_unique_id(void) { + /* Use /dev/urandom for "real" randomness. */ + int fd; + if ((fd = open("/dev/urandom", O_RDONLY)) == -1) { + LOG_ERR("Could not generate random number"); + } + unique_id result; + read(fd, &result.id[0], UNIQUE_ID_SIZE); + close(fd); + return result; +} + +char *sha1_to_hex(const unsigned char *sha1, char *buffer) { + static const char hex[] = "0123456789abcdef"; + char *buf = buffer; + + for (int i = 0; i < UNIQUE_ID_SIZE; i++) { + unsigned int val = *sha1++; + *buf++ = hex[val >> 4]; + *buf++ = hex[val & 0xf]; + } + *buf = '\0'; + + return buffer; +} diff --git a/src/common/common.h b/src/common/common.h new file mode 100644 index 000000000..f09e91cd5 --- /dev/null +++ b/src/common/common.h @@ -0,0 +1,61 @@ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include + +#ifndef RAY_COMMON_DEBUG +#define LOG_DEBUG(M, ...) +#else +#define LOG_DEBUG(M, ...) \ + fprintf(stderr, "[DEBUG] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) +#endif + +#define LOG_ERR(M, ...) \ + fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, \ + errno == 0 ? "None" : strerror(errno), ##__VA_ARGS__) + +#define LOG_INFO(M, ...) \ + fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) + +#define CHECK(COND) \ + do { \ + if (!(COND)) { \ + LOG_ERR("Check failure: %s", #COND); \ + exit(-1); \ + } \ + } while (0); + +#define CHECKM(COND, M, ...) \ + do { \ + if (!(COND)) { \ + LOG_ERR("Check failure: %s \n" M, #COND, ##__VA_ARGS__); \ + exit(-1); \ + } \ + } while (0); + +#define UNIQUE_ID_SIZE 20 + +/* Cleanup method for running tests with the greatest library. + * Runs the test, then clears the Redis database. */ +#define RUN_REDIS_TEST(context, test) \ + RUN_TEST(test); \ + freeReplyObject(redisCommand(context, "FLUSHALL")); + +typedef struct { unsigned char id[UNIQUE_ID_SIZE]; } unique_id; + +extern const unique_id NIL_ID; + +/* Generate a globally unique ID. */ +unique_id globally_unique_id(void); + +/* Convert a 20 byte sha1 hash to a hexdecimal string. This function assumes + * that buffer points to an already allocated char array of size 2 * + * UNIQUE_ID_SIZE + 1 */ +char *sha1_to_hex(const unsigned char *sha1, char *buffer); + +typedef unique_id object_id; + +#endif diff --git a/src/common/doc/tasks.md b/src/common/doc/tasks.md new file mode 100644 index 000000000..b56f89783 --- /dev/null +++ b/src/common/doc/tasks.md @@ -0,0 +1,33 @@ +# Task specifications, task instances and task logs + +A *task specification* contains all information that is needed for computing +the results of a task: + +- The function ID of the function that executes the task +- The arguments (either object IDs for pass by reference +or values for pass by value) +- The IDs of the result objects + +From these, a task ID can be computed which is also stored in the task +specification. + +A *task instance* represents one execution of a task specification. +It consists of: + +- A scheduling state (WAITING, SCHEDULED, RUNNING, DONE) +- The target node where the task is scheduled or executed +- A unique task instance ID that identifies the particular execution + of the task. + +The task data structures are defined in `common/task.h`. + +The *task log* is a mapping from the task instance ID to a sequence of +updates to the status of the task instance. It is updated by various parts +of the system: + +1. The local scheduler writes it with status WAITING when submits a task to the global scheduler +2. The global scheduler appends an update WAITING -> SCHEDULED together with the node ID when assigning the task to a local scheduler +3. The local scheduler appends an update SCHEDULED -> RUNNING when it assigns a task to a worker +4. The local scheduler appends an update RUNNING -> DONE when the task finishes execution + +The task log is defined in `common/state/task_log.h`. diff --git a/src/common/event_loop.c b/src/common/event_loop.c new file mode 100644 index 000000000..7e7835445 --- /dev/null +++ b/src/common/event_loop.c @@ -0,0 +1,62 @@ +#include "event_loop.h" + +#include "common.h" +#include + +#define INITIAL_EVENT_LOOP_SIZE 1024 + +event_loop *event_loop_create() { + return aeCreateEventLoop(INITIAL_EVENT_LOOP_SIZE); +} + +void event_loop_destroy(event_loop *loop) { + /* Clean up timer events. This is to make valgrind happy. */ + aeTimeEvent *te = loop->timeEventHead; + while (te) { + aeTimeEvent *next = te->next; + free(te); + te = next; + } + aeDeleteEventLoop(loop); +} + +void event_loop_add_file(event_loop *loop, + int fd, + int events, + event_loop_file_handler handler, + void *context) { + /* Try to add the file descriptor. */ + int err = aeCreateFileEvent(loop, fd, events, handler, context); + /* If it cannot be added, increase the size of the event loop. */ + if (err == AE_ERR && errno == ERANGE) { + err = aeResizeSetSize(loop, 3 * aeGetSetSize(loop) / 2); + CHECK(err == AE_OK); + err = aeCreateFileEvent(loop, fd, events, handler, context); + } + /* In any case, test if there were errors. */ + CHECK(err == AE_OK); +} + +void event_loop_remove_file(event_loop *loop, int fd) { + aeDeleteFileEvent(loop, fd, EVENT_LOOP_READ | EVENT_LOOP_WRITE); +} + +int64_t event_loop_add_timer(event_loop *loop, + int64_t milliseconds, + event_loop_timer_handler handler, + void *context) { + return aeCreateTimeEvent(loop, milliseconds, handler, context, NULL); +} + +void event_loop_remove_timer(event_loop *loop, timer_id timer_id) { + int err = aeDeleteTimeEvent(loop, timer_id); + CHECK(err == AE_OK); /* timer id found? */ +} + +void event_loop_run(event_loop *loop) { + aeMain(loop); +} + +void event_loop_stop(event_loop *loop) { + aeStop(loop); +} diff --git a/src/common/event_loop.h b/src/common/event_loop.h new file mode 100644 index 000000000..7f0659ba7 --- /dev/null +++ b/src/common/event_loop.h @@ -0,0 +1,77 @@ +#ifndef EVENT_LOOP_H +#define EVENT_LOOP_H + +#include +#include "ae/ae.h" + +typedef long long timer_id; + +typedef aeEventLoop event_loop; + +/* File descriptor is readable. */ +#define EVENT_LOOP_READ AE_READABLE + +/* File descriptor is writable. */ +#define EVENT_LOOP_WRITE AE_WRITABLE + +/* Constant specifying that the timer is done and it will be removed. */ +#define EVENT_LOOP_TIMER_DONE AE_NOMORE + +/* Signature of the handler that will be called when there is a new event + * on the file descriptor that this handler has been registered for. The + * context is the one that was passed into add_file by the user. The + * events parameter indicates which event is available on the file, + * it can be EVENT_LOOP_READ or EVENT_LOOP_WRITE. */ +typedef void (*event_loop_file_handler)(event_loop *loop, + int fd, + void *context, + int events); + +/* This handler will be called when a timer times out. The id of the timer + * as well as the context that was specified when registering this handler + * are passed as arguments. The return is the number of milliseconds the + * timer shall be reset to or EVENT_LOOP_TIMER_DONE if the timer shall + * not be triggered again. */ +typedef int (*event_loop_timer_handler)(event_loop *loop, + timer_id timer_id, + void *context); + +/* Create and return a new event loop. */ +event_loop *event_loop_create(); + +/* Deallocate space associated with the event loop that was created + * with the "create" function. */ +void event_loop_destroy(event_loop *loop); + +/* Register a handler that will be called any time a new event happens on + * a file descriptor. Can specify a context that will be passed as an + * argument to the handler. Currently there can only be one handler per file. + * The events parameter specifies which events we listen to: EVENT_LOOP_READ + * or EVENT_LOOP_WRITE. */ +void event_loop_add_file(event_loop *loop, + int fd, + int events, + event_loop_file_handler handler, + void *context); + +/* Remove a registered file event handler from the event loop. */ +void event_loop_remove_file(event_loop *loop, int fd); + +/* Register a handler that will be called after a time slice of + * "milliseconds" milliseconds. Can specify a context that will be passed + * as an argument to the handler. Return the id of the time event. */ +int64_t event_loop_add_timer(event_loop *loop, + int64_t milliseconds, + event_loop_timer_handler handler, + void *context); + +/* Remove a registered time event handler from the event loop. */ +void event_loop_remove_timer(event_loop *loop, timer_id timer_id); + +/* Run the event loop. */ +void event_loop_run(event_loop *loop); + +/* Stop the event loop. */ +void event_loop_stop(event_loop *loop); + +#endif diff --git a/src/common/io.c b/src/common/io.c new file mode 100644 index 000000000..1f1125bc3 --- /dev/null +++ b/src/common/io.c @@ -0,0 +1,333 @@ +#include "io.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" + +/** + * Binds to an Internet socket at the given port. Removes any existing file at + * the pathname. Returns a non-blocking file descriptor for the socket, or -1 + * if an error occurred. + * + * @note Since the returned file descriptor is non-blocking, it is not + * recommended to use the Linux read and write calls directly, since these + * might read or write a partial message. Instead, use the provided + * write_message and read_message methods. + * + * @param port The port to bind to. + * @return A non-blocking file descriptor for the socket, or -1 if an error + * occurs. + */ +int bind_inet_sock(const int port) { + struct sockaddr_in name; + int socket_fd = socket(PF_INET, SOCK_STREAM, 0); + if (socket_fd < 0) { + LOG_ERR("socket() failed for port %d.", port); + return -1; + } + name.sin_family = AF_INET; + name.sin_port = htons(port); + name.sin_addr.s_addr = htonl(INADDR_ANY); + int on = 1; + /* TODO(pcm): http://stackoverflow.com/q/1150635 */ + if (ioctl(socket_fd, FIONBIO, (char *) &on) < 0) { + LOG_ERR("ioctl failed"); + close(socket_fd); + return -1; + } + if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0) { + LOG_ERR("setsockopt failed for port %d", port); + close(socket_fd); + return -1; + } + if (bind(socket_fd, (struct sockaddr *) &name, sizeof(name)) < 0) { + LOG_ERR("Bind failed for port %d", port); + close(socket_fd); + return -1; + } + if (listen(socket_fd, 5) == -1) { + LOG_ERR("Could not listen to socket %d", port); + close(socket_fd); + return -1; + } + return socket_fd; +} + +/** + * Binds to a Unix domain streaming socket at the given + * pathname. Removes any existing file at the pathname. + * + * @param socket_pathname The pathname for the socket. + * @return A blocking file descriptor for the socket, or -1 if an error + * occurs. + */ +int bind_ipc_sock(const char *socket_pathname) { + struct sockaddr_un socket_address; + int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (socket_fd < 0) { + LOG_ERR("socket() failed for pathname %s.", socket_pathname); + return -1; + } + /* Tell the system to allow the port to be reused. */ + int on = 1; + if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, + sizeof(on)) < 0) { + LOG_ERR("setsockopt failed for pathname %s", socket_pathname); + close(socket_fd); + return -1; + } + + unlink(socket_pathname); + memset(&socket_address, 0, sizeof(struct sockaddr_un)); + socket_address.sun_family = AF_UNIX; + if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) { + LOG_ERR("Socket pathname is too long."); + close(socket_fd); + return -1; + } + strncpy(socket_address.sun_path, socket_pathname, + strlen(socket_pathname) + 1); + + if (bind(socket_fd, (struct sockaddr *) &socket_address, + sizeof(struct sockaddr_un)) != 0) { + LOG_ERR("Bind failed for pathname %s.", socket_pathname); + close(socket_fd); + return -1; + } + if (listen(socket_fd, 5) == -1) { + LOG_ERR("Could not listen to socket %s", socket_pathname); + close(socket_fd); + return -1; + } + return socket_fd; +} + +/** + * Connects to a Unix domain streaming socket at the given + * pathname. Returns a file descriptor for the socket, or -1 if + * an error occurred. + */ +int connect_ipc_sock(const char *socket_pathname) { + struct sockaddr_un socket_address; + int socket_fd; + + socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (socket_fd < 0) { + LOG_ERR("socket() failed for pathname %s.", socket_pathname); + return -1; + } + + memset(&socket_address, 0, sizeof(struct sockaddr_un)); + socket_address.sun_family = AF_UNIX; + if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) { + LOG_ERR("Socket pathname is too long."); + return -1; + } + strncpy(socket_address.sun_path, socket_pathname, + strlen(socket_pathname) + 1); + + if (connect(socket_fd, (struct sockaddr *) &socket_address, + sizeof(struct sockaddr_un)) != 0) { + LOG_ERR("Connection to socket failed for pathname %s.", socket_pathname); + return -1; + } + + return socket_fd; +} + +/** + * Accept a new client connection on the given socket + * descriptor. Returns a descriptor for the new socket. + */ +int accept_client(int socket_fd) { + int client_fd = accept(socket_fd, NULL, NULL); + if (client_fd < 0) { + LOG_ERR("Error reading from socket."); + return -1; + } + return client_fd; +} + +/** + * Write a sequence of bytes into a file descriptor. This will block until one + * of the following happens: (1) there is an error (2) end of file, or (3) all + * length bytes have been written. + * + * @param fd The file descriptor to write to. It can be non-blocking. + * @param cursor The cursor pointing to the beginning of the bytes to send. + * @param length The size of the bytes sequence to write. + * @return int Whether there was an error while writing. 0 corresponds to + * success and -1 corresponds to an error (errno will be set). + */ +int write_bytes(int fd, uint8_t *cursor, size_t length) { + ssize_t nbytes = 0; + while (length > 0) { + /* While we haven't written the whole message, write to the file + * descriptor, advance the cursor, and decrease the amount left to write. */ + nbytes = write(fd, cursor, length); + if (nbytes < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + continue; + } + /* TODO(swang): Return the error instead of exiting. */ + /* Force an exit if there was any other type of error. */ + CHECK(nbytes < 0); + } + if (nbytes == 0) { + return -1; + } + cursor += nbytes; + length -= nbytes; + } + return 0; +} + +/** + * Write a sequence of bytes on a file descriptor. The bytes should then be read + * by read_message. + * + * @param fd The file descriptor to write to. It can be non-blocking. + * @param type The type of the message to send. + * @param length The size in bytes of the bytes parameter. + * @param bytes The address of the message to send. + * @return int Whether there was an error while writing. 0 corresponds to + * success and -1 corresponds to an error (errno will be set). + */ +int write_message(int fd, int64_t type, int64_t length, uint8_t *bytes) { + int closed; + closed = write_bytes(fd, (uint8_t *) &type, sizeof(type)); + if (closed) { + return closed; + } + closed = write_bytes(fd, (uint8_t *) &length, sizeof(length)); + if (closed) { + return closed; + } + closed = write_bytes(fd, bytes, length * sizeof(char)); + if (closed) { + return closed; + } + return 0; +} + +/** + * Read a sequence of bytes from a file descriptor into a buffer. This will + * block until one of the following happens: (1) there is an error (2) end of + * file, or (3) all length bytes have been written. + * + * @note The buffer pointed to by cursor must already have length number of + * bytes allocated before calling this method. + * + * @param fd The file descriptor to read from. It can be non-blocking. + * @param cursor The cursor pointing to the beginning of the buffer. + * @param length The size of the byte sequence to read. + * @return int Whether there was an error while writing. 0 corresponds to + * success and -1 corresponds to an error (errno will be set). + */ +int read_bytes(int fd, uint8_t *cursor, size_t length) { + ssize_t nbytes = 0; + while (length > 0) { + /* While we haven't read the whole message, read from the file descriptor, + * advance the cursor, and decrease the amount left to read. */ + nbytes = read(fd, cursor, length); + if (nbytes < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + continue; + } + /* Force an exit if there was any other type of error. */ + CHECK(nbytes < 0); + } + if (nbytes == 0) { + return -1; + } + cursor += nbytes; + length -= nbytes; + } + return 0; +} + +/** + * Read a sequence of bytes written by write_message from a file descriptor. + * This allocates space for the message. + * + * @note The caller must free the memory. + * + * @param fd The file descriptor to read from. It can be non-blocking. + * @param type The type of the message that is read will be written at this + address. If there was an error while reading, this will be + DISCONNECT_CLIENT. + * @param length The size in bytes of the message that is read will be written + at this address. This size does not include the bytes used to encode + the type and length. If there was an error while reading, this will + be 0. + * @param bytes The address at which to write the pointer to the bytes that are + read and allocated by this function. If there was an error while + reading, this will be NULL. + + * @return Void. + */ +void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes) { + int closed = read_bytes(fd, (uint8_t *) type, sizeof(int64_t)); + if (closed) { + goto disconnected; + } + closed = read_bytes(fd, (uint8_t *) length, sizeof(int64_t)); + if (closed) { + goto disconnected; + } + *bytes = malloc(*length * sizeof(uint8_t)); + closed = read_bytes(fd, *bytes, *length); + if (closed) { + free(*bytes); + goto disconnected; + } + return; + +disconnected: + /* Handle the case in which the socket is closed. */ + *type = DISCONNECT_CLIENT; + *length = 0; + *bytes = NULL; + return; +} + +/* Write a null-terminated string to a file descriptor. */ +void write_log_message(int fd, char *message) { + /* Account for the \0 at the end of the string. */ + write_message(fd, LOG_MESSAGE, strlen(message) + 1, (uint8_t *) message); +} + +/* Reads a null-terminated string from the file descriptor that has been + * written by write_log_message. Allocates and returns a pointer to the string. + * NOTE: Caller must free the memory! */ +char *read_log_message(int fd) { + uint8_t *bytes; + int64_t type; + int64_t length; + read_message(fd, &type, &length, &bytes); + CHECK(type == LOG_MESSAGE); + return (char *) bytes; +} + +void write_formatted_log_message(int socket_fd, const char *format, ...) { + UT_string *cmd; + va_list ap; + + utstring_new(cmd); + va_start(ap, format); + utstring_printf_va(cmd, format, ap); + va_end(ap); + + write_log_message(socket_fd, utstring_body(cmd)); + utstring_free(cmd); +} diff --git a/src/common/io.h b/src/common/io.h new file mode 100644 index 000000000..362b85fc1 --- /dev/null +++ b/src/common/io.h @@ -0,0 +1,32 @@ +#ifndef IO_H +#define IO_H + +#include + +enum common_message_type { + /** Disconnect a client. */ + DISCONNECT_CLIENT, + /** Log a message from a client. */ + LOG_MESSAGE, + /** Submit a task to the local scheduler. */ + SUBMIT_TASK, +}; + +/* Helper functions for socket communication. */ + +int bind_inet_sock(const int port); +int bind_ipc_sock(const char *socket_pathname); +int connect_ipc_sock(const char *socket_pathname); + +int accept_client(int socket_fd); + +/* Reading and writing data */ + +int write_message(int fd, int64_t type, int64_t length, uint8_t *bytes); +void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes); + +void write_log_message(int fd, char *message); +void write_formatted_log_message(int fd, const char *format, ...); +char *read_log_message(int fd); + +#endif diff --git a/src/common/lib/python/common_extension.c b/src/common/lib/python/common_extension.c new file mode 100644 index 000000000..14f3292d9 --- /dev/null +++ b/src/common/lib/python/common_extension.c @@ -0,0 +1,344 @@ +#include +#include "node.h" + +#include "common_extension.h" +#include "task.h" +#include "utarray.h" +#include "utstring.h" + +PyObject *CommonError; + +#define MARSHAL_VERSION 2 + +/* Define the PyObjectID class. */ + +int PyObjectToUniqueID(PyObject *object, object_id *objectid) { + if (PyObject_IsInstance(object, (PyObject *) &PyObjectIDType)) { + *objectid = ((PyObjectID *) object)->object_id; + return 1; + } else { + PyErr_SetString(PyExc_TypeError, "must be an ObjectID"); + return 0; + } +} + +static int PyObjectID_init(PyObjectID *self, PyObject *args, PyObject *kwds) { + const char *data; + int size; + if (!PyArg_ParseTuple(args, "s#", &data, &size)) { + return -1; + } + if (size != UNIQUE_ID_SIZE) { + PyErr_SetString(CommonError, + "ObjectID: object id string needs to have length 20"); + return -1; + } + memcpy(&self->object_id.id[0], data, UNIQUE_ID_SIZE); + return 0; +} + +/* Create a PyObjectID from C. */ +PyObject *PyObjectID_make(object_id object_id) { + PyObjectID *result = PyObject_New(PyObjectID, &PyObjectIDType); + result = (PyObjectID *) PyObject_Init((PyObject *) result, &PyObjectIDType); + result->object_id = object_id; + return (PyObject *) result; +} + +static PyObject *PyObjectID_id(PyObject *self) { + PyObjectID *s = (PyObjectID *) self; + return PyString_FromStringAndSize((char *) &s->object_id.id[0], + UNIQUE_ID_SIZE); +} + +static PyObject *PyObjectID___reduce__(PyObjectID *self) { + PyErr_SetString(CommonError, "ObjectID objects cannot be serialized."); + return NULL; +} + +static PyMethodDef PyObjectID_methods[] = { + {"id", (PyCFunction) PyObjectID_id, METH_NOARGS, + "Return the hash associated with this ObjectID"}, + {"__reduce__", (PyCFunction) PyObjectID___reduce__, METH_NOARGS, + "Say how to pickle this ObjectID. This raises an exception to prevent" + "object IDs from being serialized."}, + {NULL} /* Sentinel */ +}; + +static PyMemberDef PyObjectID_members[] = { + {NULL} /* Sentinel */ +}; + +PyTypeObject PyObjectIDType = { + PyObject_HEAD_INIT(NULL) 0, /* ob_size */ + "common.ObjectID", /* tp_name */ + sizeof(PyObjectID), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "ObjectID object", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyObjectID_methods, /* tp_methods */ + PyObjectID_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc) PyObjectID_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +/* Define the PyTask class. */ + +static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) { + function_id function_id; + /* Arguments of the task (can be PyObjectIDs or Python values). */ + PyObject *arguments; + /* Array of pointers to string representations of pass-by-value args. */ + UT_array *val_repr_ptrs; + utarray_new(val_repr_ptrs, &ut_ptr_icd); + int num_returns; + if (!PyArg_ParseTuple(args, "O&Oi", &PyObjectToUniqueID, &function_id, + &arguments, &num_returns)) { + return -1; + } + size_t size = PyList_Size(arguments); + /* Determine the size of pass by value data in bytes. */ + size_t value_data_bytes = 0; + for (size_t i = 0; i < size; ++i) { + PyObject *arg = PyList_GetItem(arguments, i); + if (!PyObject_IsInstance(arg, (PyObject *) &PyObjectIDType)) { + PyObject *data = PyMarshal_WriteObjectToString(arg, MARSHAL_VERSION); + value_data_bytes += PyString_Size(data); + utarray_push_back(val_repr_ptrs, &data); + } + } + /* Construct the task specification. */ + int val_repr_index = 0; + self->spec = + alloc_task_spec(function_id, size, num_returns, value_data_bytes); + /* Add the task arguments. */ + for (size_t i = 0; i < size; ++i) { + PyObject *arg = PyList_GetItem(arguments, i); + if (PyObject_IsInstance(arg, (PyObject *) &PyObjectIDType)) { + task_args_add_ref(self->spec, ((PyObjectID *) arg)->object_id); + } else { + PyObject *data = + *((PyObject **) utarray_eltptr(val_repr_ptrs, val_repr_index)); + task_args_add_val(self->spec, (uint8_t *) PyString_AS_STRING(data), + PyString_GET_SIZE(data)); + Py_DECREF(data); + val_repr_index += 1; + } + } + utarray_free(val_repr_ptrs); + /* Generate and add the object IDs for the return values. */ + for (size_t i = 0; i < num_returns; ++i) { + /* TODO(rkn): Later, this should be computed as a deterministic hash of (1) + * the contents of the task, (2) the index i, and (3) a counter of the + * number of tasks launched so far by the parent task. For now, we generate + * it randomly. */ + *task_return(self->spec, i) = globally_unique_id(); + } + return 0; +} + +static void PyTask_dealloc(PyTask *self) { + free_task_spec(self->spec); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject *PyTask_function_id(PyObject *self) { + function_id function_id = *task_function(((PyTask *) self)->spec); + return PyObjectID_make(function_id); +} + +static PyObject *PyTask_arguments(PyObject *self) { + int64_t num_args = task_num_args(((PyTask *) self)->spec); + PyObject *arg_list = PyList_New((Py_ssize_t) num_args); + task_spec *task = ((PyTask *) self)->spec; + for (int i = 0; i < num_args; ++i) { + if (task_arg_type(task, i) == ARG_BY_REF) { + object_id object_id = *task_arg_id(task, i); + PyList_SetItem(arg_list, i, PyObjectID_make(object_id)); + } else { + PyObject *s = + PyMarshal_ReadObjectFromString((char *) task_arg_val(task, i), + (Py_ssize_t) task_arg_length(task, i)); + PyList_SetItem(arg_list, i, s); + } + } + return arg_list; +} + +static PyObject *PyTask_returns(PyObject *self) { + int64_t num_returns = task_num_returns(((PyTask *) self)->spec); + PyObject *return_id_list = PyList_New((Py_ssize_t) num_returns); + task_spec *task = ((PyTask *) self)->spec; + for (int i = 0; i < num_returns; ++i) { + object_id object_id = *task_return(task, i); + PyList_SetItem(return_id_list, i, PyObjectID_make(object_id)); + } + return return_id_list; +} + +static PyMethodDef PyTask_methods[] = { + {"function_id", (PyCFunction) PyTask_function_id, METH_NOARGS, + "Return the function ID for this task."}, + {"arguments", (PyCFunction) PyTask_arguments, METH_NOARGS, + "Return the arguments for the task."}, + {"returns", (PyCFunction) PyTask_returns, METH_NOARGS, + "Return the object IDs for the return values of the task."}, + {NULL} /* Sentinel */ +}; + +PyTypeObject PyTaskType = { + PyObject_HEAD_INIT(NULL) 0, /* ob_size */ + "task.Task", /* tp_name */ + sizeof(PyTask), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor) PyTask_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Task object", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyTask_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc) PyTask_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +/* Create a PyTask from a C struct. The resulting PyTask takes ownership of the + * task_spec and will deallocate the task_spec in the PyTask destructor. */ +PyObject *PyTask_make(task_spec *task_spec) { + PyTask *result = PyObject_New(PyTask, &PyTaskType); + result = (PyTask *) PyObject_Init((PyObject *) result, &PyTaskType); + result->spec = task_spec; + return (PyObject *) result; +} + +/* Define the methods for the module. */ + +#define SIZE_LIMIT 100 +#define NUM_ELEMENTS_LIMIT 1000 + +/** + * This method checks if a Python object is sufficiently simple that it can be + * serialized and passed by value as an argument to a task (without being put in + * the object store). The details of which objects are sufficiently simple are + * defined by this method and are not particularly important. But for + * performance reasons, it is better to place "small" objects in the task itself + * and "large" objects in the object store. + * + * @param value The Python object in question. + * @param num_elements_contained If this method returns 1, then the number of + * objects recursively contained within this object will be added to the + * value at this address. This is used to make sure that we do not + * serialize objects that are too large. + * @return 0 if the object cannot be serialized in the task and 1 if it can. + */ +int is_simple_value(PyObject *value, int *num_elements_contained) { + *num_elements_contained += 1; + if (*num_elements_contained >= NUM_ELEMENTS_LIMIT) { + return 0; + } + if (PyInt_Check(value) || PyLong_Check(value) || value == Py_False || + value == Py_True || PyFloat_Check(value) || value == Py_None) { + return 1; + } + if (PyString_CheckExact(value)) { + *num_elements_contained += PyString_Size(value); + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + if (PyUnicode_CheckExact(value)) { + *num_elements_contained += PyUnicode_GET_SIZE(value); + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + if (PyList_CheckExact(value) && PyList_Size(value) < SIZE_LIMIT) { + for (size_t i = 0; i < PyList_Size(value); ++i) { + if (!is_simple_value(PyList_GetItem(value, i), num_elements_contained)) { + return 0; + } + } + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + if (PyDict_CheckExact(value) && PyDict_Size(value) < SIZE_LIMIT) { + PyObject *key, *val; + Py_ssize_t pos = 0; + while (PyDict_Next(value, &pos, &key, &val)) { + if (!is_simple_value(key, num_elements_contained) || + !is_simple_value(val, num_elements_contained)) { + return 0; + } + } + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + if (PyTuple_CheckExact(value) && PyTuple_Size(value) < SIZE_LIMIT) { + for (size_t i = 0; i < PyTuple_Size(value); ++i) { + if (!is_simple_value(PyTuple_GetItem(value, i), num_elements_contained)) { + return 0; + } + } + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + return 0; +} + +PyObject *check_simple_value(PyObject *self, PyObject *args) { + PyObject *value; + if (!PyArg_ParseTuple(args, "O", &value)) { + return NULL; + } + int num_elements_contained = 0; + if (is_simple_value(value, &num_elements_contained)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} diff --git a/src/common/lib/python/common_extension.h b/src/common/lib/python/common_extension.h new file mode 100644 index 000000000..1fce38e42 --- /dev/null +++ b/src/common/lib/python/common_extension.h @@ -0,0 +1,37 @@ +#ifndef COMMON_EXTENSION_H +#define COMMON_EXTENSION_H + +#include +#include "marshal.h" +#include "structmember.h" + +#include "common.h" +#include "task.h" + +extern PyObject *CommonError; + +// clang-format off +typedef struct { + PyObject_HEAD + object_id object_id; +} PyObjectID; + +typedef struct { + PyObject_HEAD + task_spec *spec; +} PyTask; +// clang-format on + +extern PyTypeObject PyObjectIDType; + +extern PyTypeObject PyTaskType; + +int PyObjectToUniqueID(PyObject *object, object_id *objectid); + +PyObject *PyObjectID_make(object_id object_id); + +PyObject *check_simple_value(PyObject *self, PyObject *args); + +PyObject *PyTask_make(task_spec *task_spec); + +#endif /* COMMON_EXTENSION_H */ diff --git a/src/common/lib/python/common_module.c b/src/common/lib/python/common_module.c new file mode 100644 index 000000000..d5222cd87 --- /dev/null +++ b/src/common/lib/python/common_module.c @@ -0,0 +1,38 @@ +#include +#include "node.h" + +#include "common_extension.h" + +static PyMethodDef common_methods[] = { + {"check_simple_value", check_simple_value, METH_VARARGS, + "Should the object be passed by value?"}, + {NULL} /* Sentinel */ +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif + +PyMODINIT_FUNC initcommon(void) { + PyObject *m; + + if (PyType_Ready(&PyTaskType) < 0) + return; + + if (PyType_Ready(&PyObjectIDType) < 0) + return; + + m = Py_InitModule3("common", common_methods, + "A module for common types. This is used for testing."); + + Py_INCREF(&PyTaskType); + PyModule_AddObject(m, "Task", (PyObject *) &PyTaskType); + + Py_INCREF(&PyObjectIDType); + PyModule_AddObject(m, "ObjectID", (PyObject *) &PyObjectIDType); + + char common_error[] = "common.error"; + CommonError = PyErr_NewException(common_error, NULL, NULL); + Py_INCREF(CommonError); + PyModule_AddObject(m, "common_error", CommonError); +} diff --git a/src/common/lib/python/setup.py b/src/common/lib/python/setup.py new file mode 100644 index 000000000..8ef8a1abd --- /dev/null +++ b/src/common/lib/python/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup, find_packages, Extension + +common_module = Extension("common", + sources=["common_module.c", "common_extension.c"], + include_dirs=["../../", "../../thirdparty"], + extra_objects=["../../build/libcommon.a"], + extra_compile_args=["--std=c99", "-Werror"]) + +setup(name="Common", + version="0.1", + description="Common library for Ray", + ext_modules=[common_module]) diff --git a/src/common/logging.c b/src/common/logging.c new file mode 100644 index 000000000..09cad7856 --- /dev/null +++ b/src/common/logging.c @@ -0,0 +1,80 @@ +#include "logging.h" + +#include +#include +#include +#include + +#include "state/redis.h" +#include "io.h" + +static const char *log_levels[5] = {"DEBUG", "INFO", "WARN", "ERROR", "FATAL"}; +static const char *log_fmt = + "HMSET log:%s:%s:%s log_level %s event_type %s message %s timestamp %s"; + +struct ray_logger_impl { + /* String that identifies this client type. */ + const char *client_type; + /* Suppress all log messages below this level. */ + int log_level; + /* Whether or not we have a direct connection to Redis. */ + int is_direct; + /* Either a db_handle or a socket to a process with a db_handle, + * depending on the is_direct flag. */ + void *conn; +}; + +ray_logger *init_ray_logger(const char *client_type, + int log_level, + int is_direct, + void *conn) { + ray_logger *logger = malloc(sizeof(ray_logger)); + logger->client_type = client_type; + logger->log_level = log_level; + logger->is_direct = is_direct; + logger->conn = conn; + return logger; +} + +void free_ray_logger(ray_logger *logger) { + free(logger); +} + +void ray_log(ray_logger *logger, + int log_level, + const char *event_type, + const char *message) { + if (log_level < logger->log_level) { + return; + } + if (log_level < RAY_DEBUG || log_level > RAY_FATAL) { + return; + } + struct timeval tv; + UT_string *timestamp; + utstring_new(timestamp); + gettimeofday(&tv, NULL); + utstring_printf(timestamp, "%ld.%ld", tv.tv_sec, (long) tv.tv_usec); + + UT_string *origin_id; + utstring_new(origin_id); + if (logger->is_direct) { + db_handle *db = (db_handle *) logger->conn; + utstring_printf(origin_id, "%" PRId64 ":%s", db->client_id, ""); + redisAsyncCommand(db->context, NULL, NULL, log_fmt, + utstring_body(timestamp), logger->client_type, + utstring_body(origin_id), log_levels[log_level], + event_type, message, utstring_body(timestamp)); + } else { + /* If we don't own a Redis connection, we leave our client + * ID to be filled in by someone else. */ + utstring_printf(origin_id, "%s:%s", "%ld", "%ld"); + int *socket_fd = (int *) logger->conn; + write_formatted_log_message(*socket_fd, log_fmt, utstring_body(timestamp), + logger->client_type, utstring_body(origin_id), + log_levels[log_level], event_type, message, + utstring_body(timestamp)); + } + utstring_free(origin_id); + utstring_free(timestamp); +} diff --git a/src/common/logging.h b/src/common/logging.h new file mode 100644 index 000000000..4ef7c8fcb --- /dev/null +++ b/src/common/logging.h @@ -0,0 +1,39 @@ +#ifndef LOGGING_H +#define LOGGING_H + +#define RAY_VERBOSE -1 +#define RAY_DEBUG 0 +#define RAY_INFO 1 +#define RAY_WARNING 2 +#define RAY_ERROR 3 +#define RAY_FATAL 4 + +/* Entity types. */ +#define RAY_FUNCTION "FUNCTION" +#define RAY_OBJECT "OBJECT" +#define RAY_TASK "TASK" + +typedef struct ray_logger_impl ray_logger; + +/* Initialize a Ray logger for the given client type and logging level. If the + * is_direct flag is set, the logger will treat the given connection as a + * direct connection to the log. Otherwise, it will treat it as a socket to + * another process with a connection to the log. + * NOTE: User is responsible for freeing the returned logger. */ +ray_logger *init_ray_logger(const char *client_type, + int log_level, + int is_direct, + void *conn); + +/* Free the logger. This does not free the connection to the log. */ +void free_ray_logger(ray_logger *logger); + +/* Log an event at the given log level with the given event_type. + * NOTE: message cannot contain spaces! JSON format is recommended. + * TODO: Support spaces in messages. */ +void ray_log(ray_logger *logger, + int log_level, + const char *event_type, + const char *message); + +#endif diff --git a/src/common/state/db.h b/src/common/state/db.h new file mode 100644 index 000000000..50536e271 --- /dev/null +++ b/src/common/state/db.h @@ -0,0 +1,32 @@ +#ifndef DB_H +#define DB_H + +#include "event_loop.h" + +typedef struct db_handle_impl db_handle; + +/* Connect to the global system store at address and port. Returns + * a handle to the database, which must be freed with db_disconnect + * after use. */ +db_handle *db_connect(const char *db_address, + int db_port, + const char *client_type, + const char *client_addr, + int client_port); + +/* Attach global system store connection to event loop. */ +void db_attach(db_handle *db, event_loop *loop); + +/* Disconnect from the global system store. */ +void db_disconnect(db_handle *db); + +/** + * Returns the client ID, according to the database. + * + * @param db The handle to the database. + * @returns int The client ID for this connection to the database. If + * this client has no connection to the database, returns -1. + */ +int get_client_id(db_handle *db); + +#endif diff --git a/src/common/state/object_table.h b/src/common/state/object_table.h new file mode 100644 index 000000000..bab54bc7b --- /dev/null +++ b/src/common/state/object_table.h @@ -0,0 +1,25 @@ +#include "common.h" +#include "db.h" + +/* The callback that is called when the result of a lookup + * in the object table comes back. The callback should free + * the manager_vector array, but NOT the strings they are pointing to. */ +typedef void (*lookup_callback)(object_id object_id, + int manager_count, + const char *manager_vector[], + void *context); + +/* Register a new object with the directory. */ +/* TODO(pcm): Retry, print for each attempt. */ +void object_table_add(db_handle *db, object_id object_id); + +/* Remove object from the directory. */ +void object_table_remove(db_handle *db, + object_id object_id, + const char *manager); + +/* Look up entry from the directory */ +void object_table_lookup(db_handle *db, + object_id object_id, + lookup_callback callback, + void *context); diff --git a/src/common/state/redis.c b/src/common/state/redis.c new file mode 100644 index 000000000..d53b0bf15 --- /dev/null +++ b/src/common/state/redis.c @@ -0,0 +1,244 @@ +/* Redis implementation of the global state store */ + +#include + +#include +#include "hiredis/adapters/ae.h" +#include "utstring.h" + +#include "common.h" +#include "db.h" +#include "object_table.h" +#include "task_log.h" +#include "event_loop.h" +#include "redis.h" +#include "io.h" + +#define LOG_REDIS_ERR(context, M, ...) \ + fprintf(stderr, "[ERROR] (%s:%d: message: %s) " M "\n", __FILE__, __LINE__, \ + context->errstr, ##__VA_ARGS__) + +#define CHECK_REDIS_CONNECT(CONTEXT_TYPE, context, M, ...) \ + do { \ + CONTEXT_TYPE *_context = (context); \ + if (!_context) { \ + LOG_ERR("could not allocate redis context"); \ + exit(-1); \ + } \ + if (_context->err) { \ + LOG_REDIS_ERR(_context, M, ##__VA_ARGS__); \ + exit(-1); \ + } \ + } while (0); + +db_handle *db_connect(const char *address, + int port, + const char *client_type, + const char *client_addr, + int client_port) { + db_handle *db = malloc(sizeof(db_handle)); + /* Sync connection for initial handshake */ + redisReply *reply; + long long num_clients; + redisContext *context = redisConnect(address, port); + CHECK_REDIS_CONNECT(redisContext, context, "could not connect to redis %s:%d", + address, port); + /* Add new client using optimistic locking. */ + while (1) { + reply = redisCommand(context, "WATCH %s", client_type); + freeReplyObject(reply); + reply = redisCommand(context, "HLEN %s", client_type); + num_clients = reply->integer; + freeReplyObject(reply); + reply = redisCommand(context, "MULTI"); + freeReplyObject(reply); + reply = redisCommand(context, "HSET %s %lld %s:%d", client_type, + num_clients, client_addr, client_port); + freeReplyObject(reply); + reply = redisCommand(context, "EXEC"); + CHECK(reply); + if (reply->type != REDIS_REPLY_NIL) { + freeReplyObject(reply); + break; + } + freeReplyObject(reply); + } + + db->client_type = strdup(client_type); + db->client_id = num_clients; + db->service_cache = NULL; + db->sync_context = context; + utarray_new(db->callback_freelist, &ut_ptr_icd); + + /* Establish async connection */ + db->context = redisAsyncConnect(address, port); + CHECK_REDIS_CONNECT(redisAsyncContext, db->context, + "could not connect to redis %s:%d", address, port); + db->context->data = (void *) db; + /* Establish async connection for subscription */ + db->sub_context = redisAsyncConnect(address, port); + CHECK_REDIS_CONNECT(redisAsyncContext, db->sub_context, + "could not connect to redis %s:%d", address, port); + db->sub_context->data = (void *) db; + + return db; +} + +void db_disconnect(db_handle *db) { + redisFree(db->sync_context); + redisAsyncFree(db->context); + redisAsyncFree(db->sub_context); + service_cache_entry *e, *tmp; + HASH_ITER(hh, db->service_cache, e, tmp) { + free(e->addr); + HASH_DEL(db->service_cache, e); + free(e); + } + free(db->client_type); + void **p = NULL; + while ((p = (void **) utarray_next(db->callback_freelist, p))) { + free(*p); + } + utarray_free(db->callback_freelist); + free(db); +} + +void db_attach(db_handle *db, event_loop *loop) { + redisAeAttach(loop, db->context); + redisAeAttach(loop, db->sub_context); +} + +void object_table_add(db_handle *db, unique_id object_id) { + redisAsyncCommand(db->context, NULL, NULL, "SADD obj:%b %d", &object_id.id[0], + UNIQUE_ID_SIZE, db->client_id); + if (db->context->err) { + LOG_REDIS_ERR(db->context, "could not add object_table entry"); + } +} + +void object_table_get_entry(redisAsyncContext *c, void *r, void *privdata) { + db_handle *db = c->data; + lookup_callback_data *cb_data = privdata; + redisReply *reply = r; + if (reply == NULL) + return; + int *result = malloc(reply->elements * sizeof(int)); + int64_t manager_count = reply->elements; + if (reply->type == REDIS_REPLY_ARRAY) { + for (int j = 0; j < reply->elements; j++) { + CHECK(reply->element[j]->type == REDIS_REPLY_STRING); + result[j] = atoi(reply->element[j]->str); + service_cache_entry *entry; + HASH_FIND_INT(db->service_cache, &result[j], entry); + if (!entry) { + redisReply *reply = redisCommand(db->sync_context, "HGET %s %lld", + db->client_type, result[j]); + CHECK(reply->type == REDIS_REPLY_STRING); + entry = malloc(sizeof(service_cache_entry)); + entry->service_id = result[j]; + entry->addr = strdup(reply->str); + HASH_ADD_INT(db->service_cache, service_id, entry); + freeReplyObject(reply); + } + } + } else { + LOG_ERR("expected integer or string, received type %d", reply->type); + exit(-1); + } + const char **manager_vector = malloc(manager_count * sizeof(char *)); + for (int j = 0; j < manager_count; ++j) { + service_cache_entry *entry; + HASH_FIND_INT(db->service_cache, &result[j], entry); + manager_vector[j] = entry->addr; + } + cb_data->callback(cb_data->object_id, manager_count, manager_vector, + cb_data->context); + free(privdata); + free(result); +} + +void object_table_lookup(db_handle *db, + object_id object_id, + lookup_callback callback, + void *context) { + lookup_callback_data *cb_data = malloc(sizeof(lookup_callback_data)); + cb_data->callback = callback; + cb_data->object_id = object_id; + cb_data->context = context; + redisAsyncCommand(db->context, object_table_get_entry, cb_data, + "SMEMBERS obj:%b", &object_id.id[0], UNIQUE_ID_SIZE); + if (db->context->err) { + LOG_REDIS_ERR(db->context, "error in object_table lookup"); + } +} + +void task_log_add_task(db_handle *db, task_instance *task_instance) { + task_iid task_iid = *task_instance_id(task_instance); + redisAsyncCommand(db->context, NULL, NULL, "HMSET tasklog:%b 0 %b", + (char *) &task_iid.id[0], UNIQUE_ID_SIZE, + (char *) task_instance, task_instance_size(task_instance)); + if (db->context->err) { + LOG_REDIS_ERR(db->context, "error setting task in task_log_add_task"); + } + node_id node = *task_instance_node(task_instance); + int32_t state = *task_instance_state(task_instance); + redisAsyncCommand(db->context, NULL, NULL, "PUBLISH task_log:%b:%d %b", + (char *) &node.id[0], UNIQUE_ID_SIZE, state, + (char *) task_instance, task_instance_size(task_instance)); + if (db->context->err) { + LOG_REDIS_ERR(db->context, "error publishing task in task_log_add_task"); + } +} + +void task_log_redis_callback(redisAsyncContext *c, + void *reply, + void *privdata) { + redisReply *r = reply; + if (reply == NULL) + return; + CHECK(r->type == REDIS_REPLY_ARRAY); + /* First entry is message type, second is topic, third is payload. */ + CHECK(r->elements > 2); + /* If this condition is true, we got the initial message that acknowledged the + * subscription. */ + if (r->element[2]->str == NULL) { + return; + } + /* Otherwise, parse the task and call the callback. */ + CHECK(privdata); + task_log_callback_data *callback_data = privdata; + task_instance *instance = malloc(r->element[2]->len); + memcpy(instance, r->element[2]->str, r->element[2]->len); + callback_data->callback(instance, callback_data->userdata); + task_instance_free(instance); +} +void task_log_register_callback(db_handle *db, + task_log_callback callback, + node_id node, + int32_t state, + void *userdata) { + task_log_callback_data *callback_data = + malloc(sizeof(task_log_callback_data)); + utarray_push_back(db->callback_freelist, &callback_data); + callback_data->callback = callback; + callback_data->userdata = userdata; + if (memcmp(&node.id[0], &NIL_ID.id[0], UNIQUE_ID_SIZE) == 0) { + redisAsyncCommand(db->sub_context, task_log_redis_callback, callback_data, + "PSUBSCRIBE task_log:*:%d", state); + } else { + redisAsyncCommand(db->sub_context, task_log_redis_callback, callback_data, + "SUBSCRIBE task_log:%b:%d", (char *) &node.id[0], + UNIQUE_ID_SIZE, state); + } + if (db->sub_context->err) { + LOG_REDIS_ERR(db->sub_context, "error in task_log_register_callback"); + } +} + +int get_client_id(db_handle *db) { + if (db) { + return db->client_id; + } else { + return -1; + } +} diff --git a/src/common/state/redis.h b/src/common/state/redis.h new file mode 100644 index 000000000..cf368b992 --- /dev/null +++ b/src/common/state/redis.h @@ -0,0 +1,68 @@ +#ifndef REDIS_H +#define REDIS_H + +#include "db.h" +#include "object_table.h" +#include "task_log.h" + +#include "hiredis/hiredis.h" +#include "hiredis/async.h" +#include "uthash.h" +#include "utarray.h" + +typedef struct { + /* Unique ID for this service. */ + int service_id; + /* IP address and port of this service. */ + char *addr; + /* Handle for the uthash table. */ + UT_hash_handle hh; +} service_cache_entry; + +typedef struct { + /* The callback that will be called. */ + task_log_callback callback; + /* Userdata associated with the callback. */ + void *userdata; +} task_log_callback_data; + +struct db_handle_impl { + /* String that identifies this client type. */ + char *client_type; + /* Unique ID for this client within the type. */ + int64_t client_id; + /* Redis context for this global state store connection. */ + redisAsyncContext *context; + /* Redis context for "subscribe" communication. + * Yes, we need a separate one for that, see + * https://github.com/redis/hiredis/issues/55 */ + redisAsyncContext *sub_context; + /* The event loop this global state store connection is part of. */ + event_loop *loop; + /* Index of the database connection in the event loop */ + int64_t db_index; + /* Cache for the IP addresses of services. */ + service_cache_entry *service_cache; + /* Redis context for synchronous connections. + * Should only be used very rarely, it is not asynchronous. */ + redisContext *sync_context; + /* Data structure for callbacks that needs to be freed. */ + UT_array *callback_freelist; +}; + +typedef struct { + /* The callback that will be called. */ + lookup_callback callback; + /* Object ID that is looked up. */ + object_id object_id; + /* Data context for the callback. */ + void *context; +} lookup_callback_data; + +void object_table_get_entry(redisAsyncContext *c, void *r, void *privdata); + +void object_table_lookup_callback(redisAsyncContext *c, + void *r, + void *privdata); + +#endif diff --git a/src/common/state/task_log.h b/src/common/state/task_log.h new file mode 100644 index 000000000..acf5dbcd0 --- /dev/null +++ b/src/common/state/task_log.h @@ -0,0 +1,41 @@ +#ifndef TASK_LOG_H +#define TASK_LOG_H + +#include "db.h" +#include "task.h" + +/* The task log is a message bus that is used for all communication between + * local and global schedulers (and also persisted to the state database). + * Here are examples of events that are recorded by the task log: + * + * 1) local scheduler writes it when submits a task to the global scheduler; + * 2) global scheduler reads it to get the task submitted by local schedulers; + * 3) global scheduler writes it when assigning the task to a local scheduler; + * 4) local scheduler reads it to get its tasks assigned by global scheduler; + * 5) local scheduler writes it when a task finishes execution; + * 6) global scheduler reads it to get the tasks that have finished; */ + +/* Callback for subscribing to the task log. */ +typedef void (*task_log_callback)(task_instance *task_instance, void *userdata); + +/* Initially add a task instance to the task log. */ +void task_log_add_task(db_handle *db, task_instance *task_instance); + +/* Update task instance in the task log. */ +void task_log_update_task(db_handle *db, + task_iid task_iid, + int32_t state, + node_id node); + +/* Register callback for a certain event. The node specifies the node whose + * events we want to listen to. If you want to listen to all events for this + * node, use state_filter = + * TASK_WAITING | TASK_SCHEDULED | TASK_RUNNING | TASK_DONE. + * If you want to register to updates from all nodes, set node = NIL_ID. */ +void task_log_register_callback(db_handle *db, + task_log_callback callback, + node_id node, + int32_t state_filter, + void *userdata); + +#endif /* TASK_LOG_H */ diff --git a/src/common/state/task_table.h b/src/common/state/task_table.h new file mode 100644 index 000000000..3a1852522 --- /dev/null +++ b/src/common/state/task_table.h @@ -0,0 +1,20 @@ +#ifndef TASK_TABLE_H +#define TASK_TABLE_H + +#include "db.h" +#include "task.h" + +/* Add task to the task table, handle errors here. */ +status task_table_add_task(db_handle *db, task_spec *task); + +/* Callback for getting an entry from the task table. Task spec will be freed + * by the system after the callback */ +typedef void (*task_table_callback)(task_spec *task, void *context); + +/* Get specific task from the task table. */ +status task_table_get_task(db_handle *db, + task_id task_id, + task_table_callback callback, + void *context); + +#endif /* TASK_TABLE_H */ diff --git a/src/common/task.c b/src/common/task.c new file mode 100644 index 000000000..0086eb337 --- /dev/null +++ b/src/common/task.c @@ -0,0 +1,218 @@ +#include +#include +#include + +#include "utarray.h" + +#include "task.h" +#include "common.h" +#include "io.h" + +/* TASK SPECIFICATIONS */ + +/* Tasks are stored in a consecutive chunk of memory, the first + * sizeof(task_spec) bytes are arranged according to the struct + * task_spec. Then there is an array of task_args of length + * (num_args + num_returns), and then follows the data of + * pass-by-value arguments of size args_value_size. The offsets in the + * task_arg.val are with respect to the end of the augmented structure, + * i.e. with respect to the address &task_spec.args_and_returns[0] + + * (task_spec->num_args + task_spec->num_returns) * sizeof(task_arg). */ + +typedef struct { + /* Either ARG_BY_REF or ARG_BY_VAL. */ + int8_t type; + union { + object_id obj_id; + struct { + /* Offset where the data associated to this arg is located relative + * to &task_spec.args_and_returns[0]. */ + ptrdiff_t offset; + int64_t length; + } value; + }; +} task_arg; + +struct task_spec_impl { + /* Function ID of the task. */ + function_id function_id; + /* Total number of arguments. */ + int64_t num_args; + /* Index of the last argument that has been constructed. */ + int64_t arg_index; + /* Number of return values. */ + int64_t num_returns; + /* Number of bytes the pass-by-value arguments are occupying. */ + int64_t args_value_size; + /* The offset of the number of bytes of pass-by-value data that + * has been written so far, relative to &task_spec->args_and_returns[0] + + * (task_spec->num_args + task_spec->num_returns) * sizeof(task_arg) */ + int64_t args_value_offset; + /* Argument and return IDs as well as offsets for pass-by-value args. */ + task_arg args_and_returns[0]; +}; + +/* The size of a task specification is given by the following expression. */ +#define TASK_SPEC_SIZE(NUM_ARGS, NUM_RETURNS, ARGS_VALUE_SIZE) \ + (sizeof(task_spec) + ((NUM_ARGS) + (NUM_RETURNS)) * sizeof(task_arg) + \ + (ARGS_VALUE_SIZE)) + +task_spec *alloc_task_spec(function_id function_id, + int64_t num_args, + int64_t num_returns, + int64_t args_value_size) { + int64_t size = TASK_SPEC_SIZE(num_args, num_returns, args_value_size); + task_spec *task = malloc(size); + memset(task, 0, size); + task->function_id = function_id; + task->num_args = num_args; + task->arg_index = 0; + task->num_returns = num_returns; + task->args_value_size = args_value_size; + return task; +} + +int64_t task_size(task_spec *spec) { + return TASK_SPEC_SIZE(spec->num_args, spec->num_returns, + spec->args_value_size); +} + +unique_id *task_function(task_spec *spec) { + return &spec->function_id; +} + +int64_t task_num_args(task_spec *spec) { + return spec->num_args; +} + +int64_t task_num_returns(task_spec *spec) { + return spec->num_returns; +} + +int8_t task_arg_type(task_spec *spec, int64_t arg_index) { + CHECK(0 <= arg_index && arg_index < spec->num_args); + return spec->args_and_returns[arg_index].type; +} + +object_id *task_arg_id(task_spec *spec, int64_t arg_index) { + CHECK(0 <= arg_index && arg_index < spec->num_args); + task_arg *arg = &spec->args_and_returns[arg_index]; + CHECK(arg->type == ARG_BY_REF) + return &arg->obj_id; +} + +uint8_t *task_arg_val(task_spec *spec, int64_t arg_index) { + CHECK(0 <= arg_index && arg_index < spec->num_args); + task_arg *arg = &spec->args_and_returns[arg_index]; + CHECK(arg->type == ARG_BY_VAL); + uint8_t *data = (uint8_t *) &spec->args_and_returns[0]; + data += (spec->num_args + spec->num_returns) * sizeof(task_arg); + return data + arg->value.offset; +} + +int64_t task_arg_length(task_spec *spec, int64_t arg_index) { + CHECK(0 <= arg_index && arg_index < spec->num_args); + task_arg *arg = &spec->args_and_returns[arg_index]; + CHECK(arg->type == ARG_BY_VAL); + return arg->value.length; +} + +int64_t task_args_add_ref(task_spec *spec, object_id obj_id) { + task_arg *arg = &spec->args_and_returns[spec->arg_index]; + arg->type = ARG_BY_REF; + arg->obj_id = obj_id; + return spec->arg_index++; +} + +int64_t task_args_add_val(task_spec *spec, uint8_t *data, int64_t length) { + task_arg *arg = &spec->args_and_returns[spec->arg_index]; + arg->type = ARG_BY_VAL; + arg->value.offset = spec->args_value_offset; + arg->value.length = length; + uint8_t *addr = task_arg_val(spec, spec->arg_index); + CHECK(spec->args_value_offset + length <= spec->args_value_size); + CHECK(spec->arg_index != spec->num_args - 1 || + spec->args_value_offset + length == spec->args_value_size); + memcpy(addr, data, length); + spec->args_value_offset += length; + return spec->arg_index++; +} + +object_id *task_return(task_spec *spec, int64_t ret_index) { + CHECK(0 <= ret_index && ret_index < spec->num_returns); + task_arg *ret = &spec->args_and_returns[spec->num_args + ret_index]; + CHECK(ret->type == ARG_BY_REF); /* No memory corruption. */ + return &ret->obj_id; +} + +void free_task_spec(task_spec *spec) { + CHECK(spec->arg_index == spec->num_args); /* Task was fully constructed */ + free(spec); +} + +void print_task(task_spec *spec, UT_string *output) { + /* For converting an id to hex, which has double the number + * of bytes compared to the id (+ 1 byte for '\0'). */ + static char hex[2 * UNIQUE_ID_SIZE + 1]; + /* Print function id. */ + sha1_to_hex(&task_function(spec)->id[0], &hex[0]); + utstring_printf(output, "fun %s ", &hex[0]); + /* Print arguments. */ + for (int i = 0; i < task_num_args(spec); ++i) { + sha1_to_hex(&task_arg_id(spec, i)->id[0], &hex[0]); + utstring_printf(output, " id:%d %s", i, &hex[0]); + } + /* Print return ids. */ + for (int i = 0; i < task_num_returns(spec); ++i) { + object_id *object_id = task_return(spec, i); + sha1_to_hex(&object_id->id[0], &hex[0]); + utstring_printf(output, " ret:%d %s", i, &hex[0]); + } +} + +/* TASK INSTANCES */ + +struct task_instance_impl { + task_iid iid; + int32_t state; + node_id node; + task_spec spec; +}; + +task_instance *make_task_instance(task_iid task_iid, + task_spec *spec, + int32_t state, + node_id node) { + int64_t size = sizeof(task_instance) - sizeof(task_spec) + task_size(spec); + task_instance *result = malloc(size); + memset(result, 0, size); + result->iid = task_iid; + result->state = state; + result->node = node; + memcpy(&result->spec, spec, task_size(spec)); + return result; +} + +int64_t task_instance_size(task_instance *instance) { + return sizeof(task_instance) - sizeof(task_spec) + task_size(&instance->spec); +} + +task_iid *task_instance_id(task_instance *instance) { + return &instance->iid; +} + +int32_t *task_instance_state(task_instance *instance) { + return &instance->state; +} + +node_id *task_instance_node(task_instance *instance) { + return &instance->node; +} + +task_spec *task_instance_task_spec(task_instance *instance) { + return &instance->spec; +} + +void task_instance_free(task_instance *instance) { + free(instance); +} diff --git a/src/common/task.h b/src/common/task.h new file mode 100644 index 000000000..782bf685b --- /dev/null +++ b/src/common/task.h @@ -0,0 +1,132 @@ +#ifndef TASK_H +#define TASK_H + +/* This API specifies the task data structures. It is in C so we can + * easily construct tasks from other languages like Python. The datastructures + * are also defined in such a way that memory is contiguous and all pointers + * are relative, so that we can memcpy the datastructure and ship it over the + * network without serialization and deserialization. */ + +#include +#include +#include "common.h" +#include "utstring.h" + +typedef unique_id function_id; + +/* The task ID is a deterministic hash of the function ID that + * the task executes and the argument IDs or argument values */ +typedef unique_id task_id; + +/* The task instance ID is a globally unique ID generated which + * identifies this particular execution of the task */ +typedef unique_id task_iid; + +/* The node id is an identifier for the node the task is + * scheduled on */ +typedef unique_id node_id; + +/* + * TASK SPECIFICATIONS: Contain all the information neccessary + * to execute the task (function id, arguments, return object ids). + * + */ + +typedef struct task_spec_impl task_spec; + +/* If argument is passed by value or reference. */ +enum arg_type { ARG_BY_REF, ARG_BY_VAL }; + +/* Construct and modify task specifications. */ + +/* Allocating and initializing a task. */ +task_spec *alloc_task_spec(function_id function_id, + int64_t num_args, + int64_t num_returns, + int64_t args_value_size); + +/* Size of the task in bytes. */ +int64_t task_size(task_spec *spec); + +/* Return the function ID of the task. */ +unique_id *task_function(task_spec *spec); + +/* Getting the number of arguments and returns. */ +int64_t task_num_args(task_spec *spec); +int64_t task_num_returns(task_spec *spec); + +/* Getting task arguments. */ +int8_t task_arg_type(task_spec *spec, int64_t arg_index); +unique_id *task_arg_id(task_spec *spec, int64_t arg_index); +uint8_t *task_arg_val(task_spec *spec, int64_t arg_index); +int64_t task_arg_length(task_spec *spec, int64_t arg_index); + +/* Setting task arguments. Note that this API only allows you to set the + * arguments in their order of appearance. */ +int64_t task_args_add_ref(task_spec *spec, object_id obj_id); +int64_t task_args_add_val(task_spec *spec, uint8_t *data, int64_t length); + +/* Getting and setting return arguments. Tasks return by reference for now. */ +unique_id *task_return(task_spec *spec, int64_t ret_index); + +/* Freeing the task datastructure. */ +void free_task_spec(task_spec *spec); + +/* Write the task specification to a file or socket. */ +void write_task(int fd, task_spec *spec); + +/* Read the task specification from a file or socket. It is the user's + * responsibility to free the task after it has been used. */ +task_spec *read_task(int fd); + +/* Print task as a humanly readable string. */ +void print_task(task_spec *spec, UT_string *output); + +/* + * SCHEDULED TASK: Contains information about a scheduled task: + * the task iid, the task specification and the task status + * (WAITING, SCHEDULED, RUNNING, DONE) and which node the + * task is scheduled on. + * + */ + +/* The scheduling_state can be used as a flag when we are listening + * for an event, for example TASK_WAITING | TASK_SCHEDULED. */ +enum scheduling_state { + TASK_STATUS_WAITING = 1, + TASK_STATUS_SCHEDULED = 2, + TASK_STATUS_RUNNING = 4, + TASK_STATUS_DONE = 8 +}; + +/* A task instance is one execution of a task specification. + * It has a unique instance id, a state of execution (see scheduling_state) + * and a node it is scheduled on or running on. */ +typedef struct task_instance_impl task_instance; + +/* Allocate and initialize a new task instance. Must be freed with + * scheduled_task_free after use. */ +task_instance *make_task_instance(task_iid task_iid, + task_spec *task, + int32_t state, + node_id node); + +/* Size of task instance structure in bytes. */ +int64_t task_instance_size(task_instance *instance); + +/* Instance ID of the task instance. */ +task_iid *task_instance_id(task_instance *instance); + +/* The scheduling state of the task instance. */ +int32_t *task_instance_state(task_instance *instance); + +/* Node this task instance has been assigned to or is running on. */ +node_id *task_instance_node(task_instance *instance); + +/* Task specification of this task instance. */ +task_spec *task_instance_task_spec(task_instance *instance); + +/* Free this task instance datastructure. */ +void task_instance_free(task_instance *instance); + +#endif diff --git a/src/common/test/common_tests.c b/src/common/test/common_tests.c new file mode 100644 index 000000000..47b643039 --- /dev/null +++ b/src/common/test/common_tests.c @@ -0,0 +1,24 @@ +#include "greatest.h" + +#include "common.h" + +SUITE(common_tests); + +TEST sha1_test(void) { + static char hex[2 * UNIQUE_ID_SIZE + 1]; + unique_id uid = globally_unique_id(); + sha1_to_hex(&uid.id[0], &hex[0]); + PASS(); +} + +SUITE(common_tests) { + RUN_TEST(sha1_test); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(common_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/db_tests.c b/src/common/test/db_tests.c new file mode 100644 index 000000000..74dd9a850 --- /dev/null +++ b/src/common/test/db_tests.c @@ -0,0 +1,175 @@ +#include "greatest.h" + +#include +#include +#include + +#include "event_loop.h" +#include "test/example_task.h" +#include "state/db.h" +#include "state/object_table.h" +#include "state/task_log.h" +#include "state/redis.h" +#include "task.h" + +SUITE(db_tests); + +const char *manager_addr = "127.0.0.1"; +int manager_port1 = 12345; +int manager_port2 = 12346; +char received_addr1[16] = {0}; +char received_port1[6] = {0}; +char received_addr2[16] = {0}; +char received_port2[6] = {0}; + +/* Test if entries have been written to the database. */ +void test_callback(object_id object_id, + int manager_count, + const char *manager_vector[], + void *context) { + CHECK(manager_count == 2); + if (!manager_vector[0] || + sscanf(manager_vector[0], "%15[0-9.]:%5[0-9]", received_addr1, + received_port1) != 2) { + CHECK(0); + } + if (!manager_vector[1] || + sscanf(manager_vector[1], "%15[0-9.]:%5[0-9]", received_addr2, + received_port2) != 2) { + CHECK(0); + } + free(manager_vector); +} + +int timeout_handler(event_loop *loop, timer_id timer_id, void *context) { + event_loop_stop(loop); + return EVENT_LOOP_TIMER_DONE; +} + +TEST object_table_lookup_test(void) { + event_loop *loop = event_loop_create(); + db_handle *db1 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr, + manager_port1); + db_handle *db2 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr, + manager_port2); + db_attach(db1, loop); + db_attach(db2, loop); + unique_id id = globally_unique_id(); + object_table_add(db1, id); + object_table_add(db2, id); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + object_table_lookup(db1, id, test_callback, NULL); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + int port1 = atoi(received_port1); + int port2 = atoi(received_port2); + ASSERT_STR_EQ(&received_addr1[0], manager_addr); + ASSERT((port1 == manager_port1 && port2 == manager_port2) || + (port2 == manager_port1 && port1 == manager_port2)); + + db_disconnect(db1); + db_disconnect(db2); + + event_loop_destroy(loop); + PASS(); +} + +void task_log_test_callback(task_instance *instance, void *userdata) { + task_instance *other = userdata; + CHECK(*task_instance_state(instance) == TASK_STATUS_SCHEDULED); + CHECK(task_instance_size(instance) == task_instance_size(other)); + CHECK(memcmp(instance, other, task_instance_size(instance)) == 0); +} + +TEST task_log_test(void) { + event_loop *loop = event_loop_create(); + db_handle *db = db_connect("127.0.0.1", 6379, "local_scheduler", "", -1); + db_attach(db, loop); + node_id node = globally_unique_id(); + task_spec *task = example_task(); + task_instance *instance = make_task_instance(globally_unique_id(), task, + TASK_STATUS_SCHEDULED, node); + task_log_register_callback(db, task_log_test_callback, node, + TASK_STATUS_SCHEDULED, instance); + task_log_add_task(db, instance); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + task_instance_free(instance); + free_task_spec(task); + db_disconnect(db); + event_loop_destroy(loop); + PASS(); +} + +int num_test_callback_called = 0; + +void task_log_all_test_callback(task_instance *instance, void *userdata) { + num_test_callback_called += 1; +} + +TEST task_log_all_test(void) { + event_loop *loop = event_loop_create(); + db_handle *db = db_connect("127.0.0.1", 6379, "local_scheduler", "", -1); + db_attach(db, loop); + task_spec *task = example_task(); + /* Schedule two tasks on different nodes. */ + task_instance *instance1 = make_task_instance( + globally_unique_id(), task, TASK_STATUS_SCHEDULED, globally_unique_id()); + task_instance *instance2 = make_task_instance( + globally_unique_id(), task, TASK_STATUS_SCHEDULED, globally_unique_id()); + task_log_register_callback(db, task_log_all_test_callback, NIL_ID, + TASK_STATUS_SCHEDULED, NULL); + task_log_add_task(db, instance1); + task_log_add_task(db, instance2); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + task_instance_free(instance2); + task_instance_free(instance1); + free_task_spec(task); + db_disconnect(db); + event_loop_destroy(loop); + ASSERT(num_test_callback_called == 2); + PASS(); +} + +TEST unique_client_id_test(void) { + const int num_conns = 50; + + db_handle *db; + pid_t pid = fork(); + for (int i = 0; i < num_conns; ++i) { + db = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr, + manager_port1); + db_disconnect(db); + } + if (pid == 0) { + exit(0); + } else { + wait(NULL); + } + + db = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr, + manager_port1); + ASSERT_EQ(get_client_id(db), num_conns * 2); + db_disconnect(db); + PASS(); +} + +SUITE(db_tests) { + redisContext *context = redisConnect("127.0.0.1", 6379); + freeReplyObject(redisCommand(context, "FLUSHALL")); + RUN_REDIS_TEST(context, object_table_lookup_test); + RUN_REDIS_TEST(context, task_log_test); + RUN_REDIS_TEST(context, task_log_all_test); + RUN_REDIS_TEST(context, unique_client_id_test); + redisFree(context); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(db_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/example_task.h b/src/common/test/example_task.h new file mode 100644 index 000000000..0dddc4dc1 --- /dev/null +++ b/src/common/test/example_task.h @@ -0,0 +1,14 @@ +#ifndef EXAMPLE_TASK_H +#define EXAMPLE_TASK_H + +#include "task.h" + +task_spec *example_task(void) { + function_id func_id = globally_unique_id(); + task_spec *task = alloc_task_spec(func_id, 2, 1, 0); + task_args_add_ref(task, globally_unique_id()); + task_args_add_ref(task, globally_unique_id()); + return task; +} + +#endif diff --git a/src/common/test/io_tests.c b/src/common/test/io_tests.c new file mode 100644 index 000000000..56ebf0607 --- /dev/null +++ b/src/common/test/io_tests.c @@ -0,0 +1,106 @@ +#include "greatest.h" + +#include +#include +#include + +#include "io.h" +#include "utstring.h" + +SUITE(io_tests); + +TEST ipc_socket_test(void) { + const char *socket_pathname = "test-socket"; + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + + char *test_string = "hello world"; + char *test_bytes = "another string"; + pid_t pid = fork(); + if (pid == 0) { + close(socket_fd); + socket_fd = connect_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + write_log_message(socket_fd, test_string); + write_message(socket_fd, LOG_MESSAGE, strlen(test_bytes), + (uint8_t *) test_bytes); + close(socket_fd); + exit(0); + } else { + int client_fd = accept_client(socket_fd); + ASSERT(client_fd >= 0); + char *message = read_log_message(client_fd); + ASSERT(message != NULL); + ASSERT_STR_EQ(test_string, message); + free(message); + int64_t type; + int64_t len; + uint8_t *bytes; + read_message(client_fd, &type, &len, &bytes); + ASSERT(type == LOG_MESSAGE); + ASSERT(memcmp(test_bytes, bytes, len) == 0); + free(bytes); + close(client_fd); + close(socket_fd); + unlink(socket_pathname); + } + + PASS(); +} + +TEST long_ipc_socket_test(void) { + const char *socket_pathname = "long-test-socket"; + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + + UT_string *test_string; + utstring_new(test_string); + for (int i = 0; i < 10000; i++) { + utstring_printf(test_string, "hello world "); + } + char *test_bytes = "another string"; + pid_t pid = fork(); + if (pid == 0) { + close(socket_fd); + socket_fd = connect_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + write_log_message(socket_fd, utstring_body(test_string)); + write_message(socket_fd, LOG_MESSAGE, strlen(test_bytes), + (uint8_t *) test_bytes); + close(socket_fd); + exit(0); + } else { + int client_fd = accept_client(socket_fd); + ASSERT(client_fd >= 0); + char *message = read_log_message(client_fd); + ASSERT(message != NULL); + ASSERT_STR_EQ(utstring_body(test_string), message); + free(message); + int64_t type; + int64_t len; + uint8_t *bytes; + read_message(client_fd, &type, &len, &bytes); + ASSERT(type == LOG_MESSAGE); + ASSERT(memcmp(test_bytes, bytes, len) == 0); + free(bytes); + close(client_fd); + close(socket_fd); + unlink(socket_pathname); + } + + utstring_free(test_string); + PASS(); +} + +SUITE(io_tests) { + RUN_TEST(ipc_socket_test); + RUN_TEST(long_ipc_socket_test); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(io_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/redis_tests.c b/src/common/test/redis_tests.c new file mode 100644 index 000000000..0a50c7d1b --- /dev/null +++ b/src/common/test/redis_tests.c @@ -0,0 +1,225 @@ +#include "greatest.h" + +#include +#include + +#include "utarray.h" + +#include "event_loop.h" +#include "state/db.h" +#include "state/redis.h" +#include "io.h" +#include "logging.h" + +SUITE(redis_tests); + +const char *test_set_format = "SET %s %s"; +const char *test_get_format = "GET %s"; +const char *test_key = "foo"; +const char *test_value = "bar"; +UT_array *connections = NULL; + +int async_redis_socket_test_callback_called = 0; + +void async_redis_socket_test_callback(redisAsyncContext *ac, + void *r, + void *privdata) { + async_redis_socket_test_callback_called = 1; + redisContext *context = redisConnect("127.0.0.1", 6379); + redisReply *reply = redisCommand(context, test_get_format, test_key); + redisFree(context); + CHECK(reply != NULL); + if (strcmp(reply->str, test_value)) { + freeReplyObject(reply); + CHECK(0); + } + freeReplyObject(reply); +} + +TEST redis_socket_test(void) { + const char *socket_pathname = "redis-test-socket"; + redisContext *context = redisConnect("127.0.0.1", 6379); + ASSERT(context != NULL); + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + + int client_fd = connect_ipc_sock(socket_pathname); + ASSERT(client_fd >= 0); + write_formatted_log_message(client_fd, test_set_format, test_key, test_value); + + int server_fd = accept_client(socket_fd); + char *cmd = read_log_message(server_fd); + close(client_fd); + close(server_fd); + close(socket_fd); + unlink(socket_pathname); + + redisReply *reply; + reply = redisCommand(context, cmd, 0, 0); + freeReplyObject(reply); + reply = redisCommand(context, "GET %s", test_key); + ASSERT(reply != NULL); + ASSERT_STR_EQ(reply->str, test_value); + freeReplyObject(reply); + + free(cmd); + redisFree(context); + PASS(); +} + +void redis_read_callback(event_loop *loop, int fd, void *context, int events) { + db_handle *db = context; + char *cmd = read_log_message(fd); + redisAsyncCommand(db->context, async_redis_socket_test_callback, NULL, cmd, + db->client_id, 0); + free(cmd); +} + +void redis_accept_callback(event_loop *loop, + int socket_fd, + void *context, + int events) { + int accept_fd = accept_client(socket_fd); + CHECK(accept_fd >= 0); + utarray_push_back(connections, &accept_fd); + event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, redis_read_callback, + context); +} + +int timeout_handler(event_loop *loop, timer_id timer_id, void *context) { + event_loop_stop(loop); + return EVENT_LOOP_TIMER_DONE; +} + +TEST async_redis_socket_test(void) { + utarray_new(connections, &ut_int_icd); + event_loop *loop = event_loop_create(); + + /* Start IPC channel. */ + const char *socket_pathname = "async-redis-test-socket"; + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + utarray_push_back(connections, &socket_fd); + + /* Start connection to Redis. */ + db_handle *db = db_connect("127.0.0.1", 6379, "", "", 0); + db_attach(db, loop); + + /* Send a command to the Redis process. */ + int client_fd = connect_ipc_sock(socket_pathname); + ASSERT(client_fd >= 0); + utarray_push_back(connections, &client_fd); + write_formatted_log_message(client_fd, test_set_format, test_key, test_value); + + event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, redis_read_callback, + db); + event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, redis_accept_callback, + db); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + + CHECK(async_redis_socket_test_callback_called); + + db_disconnect(db); + event_loop_destroy(loop); + for (int *p = (int *) utarray_front(connections); p != NULL; + p = (int *) utarray_next(connections, p)) { + close(*p); + } + unlink(socket_pathname); + utarray_free(connections); + PASS(); +} + +int logging_test_callback_called = 0; + +void logging_test_callback(redisAsyncContext *ac, void *r, void *privdata) { + logging_test_callback_called = 1; + redisContext *context = redisConnect("127.0.0.1", 6379); + redisReply *reply = redisCommand(context, "KEYS %s", "log:*"); + redisFree(context); + CHECK(reply != NULL); + CHECK(reply->elements > 0); + freeReplyObject(reply); +} + +void logging_read_callback(event_loop *loop, + int fd, + void *context, + int events) { + db_handle *conn = context; + char *cmd = read_log_message(fd); + redisAsyncCommand(conn->context, logging_test_callback, NULL, cmd, + conn->client_id, 0); + free(cmd); +} + +void logging_accept_callback(event_loop *loop, + int socket_fd, + void *context, + int events) { + int accept_fd = accept_client(socket_fd); + CHECK(accept_fd >= 0); + utarray_push_back(connections, &accept_fd); + event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, logging_read_callback, + context); +} + +TEST logging_test(void) { + utarray_new(connections, &ut_int_icd); + event_loop *loop = event_loop_create(); + + /* Start IPC channel. */ + const char *socket_pathname = "logging-test-socket"; + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + utarray_push_back(connections, &socket_fd); + + /* Start connection to Redis. */ + db_handle *conn = db_connect("127.0.0.1", 6379, "", "", 0); + db_attach(conn, loop); + + /* Send a command to the Redis process. */ + int client_fd = connect_ipc_sock(socket_pathname); + ASSERT(client_fd >= 0); + utarray_push_back(connections, &client_fd); + ray_logger *logger = init_ray_logger("worker", RAY_INFO, 0, &client_fd); + ray_log(logger, RAY_INFO, "TEST", "Message"); + + event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, logging_accept_callback, + conn); + event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, logging_read_callback, + conn); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + + CHECK(logging_test_callback_called); + + free_ray_logger(logger); + db_disconnect(conn); + event_loop_destroy(loop); + for (int *p = (int *) utarray_front(connections); p != NULL; + p = (int *) utarray_next(connections, p)) { + close(*p); + } + unlink(socket_pathname); + utarray_free(connections); + PASS(); +} + +SUITE(redis_tests) { + redisContext *context = redisConnect("127.0.0.1", 6379); + freeReplyObject(redisCommand(context, "FLUSHALL")); + RUN_REDIS_TEST(context, redis_socket_test); + RUN_REDIS_TEST(context, async_redis_socket_test); + RUN_REDIS_TEST(context, logging_test); + redisFree(context); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(redis_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/task_tests.c b/src/common/test/task_tests.c new file mode 100644 index 000000000..77f43bbf0 --- /dev/null +++ b/src/common/test/task_tests.c @@ -0,0 +1,77 @@ +#include "greatest.h" + +#include +#include +#include + +#include "common.h" +#include "test/example_task.h" +#include "task.h" +#include "io.h" + +SUITE(task_tests); + +TEST task_test(void) { + function_id func_id = globally_unique_id(); + task_spec *task = alloc_task_spec(func_id, 4, 2, 10); + ASSERT(task_num_args(task) == 4); + ASSERT(task_num_returns(task) == 2); + + unique_id arg1 = globally_unique_id(); + ASSERT(task_args_add_ref(task, arg1) == 0); + ASSERT(task_args_add_val(task, (uint8_t *) "hello", 5) == 1); + unique_id arg2 = globally_unique_id(); + ASSERT(task_args_add_ref(task, arg2) == 2); + ASSERT(task_args_add_val(task, (uint8_t *) "world", 5) == 3); + + unique_id ret0 = globally_unique_id(); + unique_id ret1 = globally_unique_id(); + memcpy(task_return(task, 0), &ret0, sizeof(ret0)); + memcpy(task_return(task, 1), &ret1, sizeof(ret1)); + + ASSERT(memcmp(task_arg_id(task, 0), &arg1, sizeof(arg1)) == 0); + ASSERT(memcmp(task_arg_val(task, 1), (uint8_t *) "hello", + task_arg_length(task, 1)) == 0); + ASSERT(memcmp(task_arg_id(task, 2), &arg2, sizeof(arg2)) == 0); + ASSERT(memcmp(task_arg_val(task, 3), (uint8_t *) "world", + task_arg_length(task, 3)) == 0); + + ASSERT(memcmp(task_return(task, 0), &ret0, sizeof(unique_id)) == 0); + ASSERT(memcmp(task_return(task, 1), &ret1, sizeof(unique_id)) == 0); + + free_task_spec(task); + PASS(); +} + +TEST send_task(void) { + function_id func_id = globally_unique_id(); + task_spec *task = alloc_task_spec(func_id, 4, 2, 10); + *task_return(task, 1) = globally_unique_id(); + int fd[2]; + socketpair(AF_UNIX, SOCK_STREAM, 0, fd); + write_message(fd[0], SUBMIT_TASK, task_size(task), (uint8_t *) task); + int64_t type; + int64_t length; + uint8_t *message; + read_message(fd[1], &type, &length, &message); + task_spec *result = (task_spec *) message; + ASSERT(type == SUBMIT_TASK); + ASSERT(memcmp(task, result, task_size(task)) == 0); + ASSERT(memcmp(task, result, task_size(result)) == 0); + free(task); + free(result); + PASS(); +} + +SUITE(task_tests) { + RUN_TEST(task_test); + RUN_TEST(send_task); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(task_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/test.py b/src/common/test/test.py new file mode 100644 index 000000000..359e8c030 --- /dev/null +++ b/src/common/test/test.py @@ -0,0 +1,119 @@ +from __future__ import print_function + +import pickle +import unittest + +import common + +BASE_SIMPLE_OBJECTS = [ + 0, 1, 100000, 0L, 1L, 100000L, 1L << 100, 0.0, 0.5, 0.9, 100000.1, (), [], {}, + "", 990 * "h", u"", 990 * u"h" +] + +LIST_SIMPLE_OBJECTS = [[obj] for obj in BASE_SIMPLE_OBJECTS] +TUPLE_SIMPLE_OBJECTS = [(obj,) for obj in BASE_SIMPLE_OBJECTS] +DICT_SIMPLE_OBJECTS = [{(): obj} for obj in BASE_SIMPLE_OBJECTS] + +SIMPLE_OBJECTS = (BASE_SIMPLE_OBJECTS + + LIST_SIMPLE_OBJECTS + + TUPLE_SIMPLE_OBJECTS + + DICT_SIMPLE_OBJECTS) + +# Create some complex objects that cannot be serialized by value in tasks. + +l = [] +l.append(l) + +class Foo(object): + def __init__(self): + pass + +BASE_COMPLEX_OBJECTS = [999 * "h", 999 * u"h", l, Foo(), 10 * [10 * [10 * [1]]]] + +LIST_COMPLEX_OBJECTS = [[obj] for obj in BASE_COMPLEX_OBJECTS] +TUPLE_COMPLEX_OBJECTS = [(obj,) for obj in BASE_COMPLEX_OBJECTS] +DICT_COMPLEX_OBJECTS = [{(): obj} for obj in BASE_COMPLEX_OBJECTS] + +COMPLEX_OBJECTS = (BASE_COMPLEX_OBJECTS + + LIST_COMPLEX_OBJECTS + + TUPLE_COMPLEX_OBJECTS + + DICT_COMPLEX_OBJECTS) + +class TestSerialization(unittest.TestCase): + + def test_serialize_by_value(self): + + for val in SIMPLE_OBJECTS: + self.assertTrue(common.check_simple_value(val)) + for val in COMPLEX_OBJECTS: + self.assertFalse(common.check_simple_value(val)) + +class TestObjectID(unittest.TestCase): + + def test_create_object_id(self): + object_id = common.ObjectID(20 * "a") + + def test_cannot_pickle_object_ids(self): + object_ids = [common.ObjectID(20 * chr(i)) for i in range(256)] + def f(): + return object_ids + def g(val=object_ids): + return 1 + def h(): + x = object_ids[0] + return 1 + # Make sure that object IDs cannot be pickled (including functions that + # close over object IDs). + self.assertRaises(Exception, lambda : pickling.dumps(object_ids[0])) + self.assertRaises(Exception, lambda : pickling.dumps(object_ids)) + self.assertRaises(Exception, lambda : pickling.dumps(f)) + self.assertRaises(Exception, lambda : pickling.dumps(g)) + self.assertRaises(Exception, lambda : pickling.dumps(h)) + +class TestTask(unittest.TestCase): + + def test_create_task(self): + # TODO(rkn): The function ID should be a FunctionID object, not an ObjectID. + function_id = common.ObjectID(20 * "a") + object_ids = [common.ObjectID(20 * chr(i)) for i in range(256)] + args_list = [ + [], + 1 * [1], + 10 * [1], + 100 * [1], + 1000 * [1], + 1 * ["a"], + 10 * ["a"], + 100 * ["a"], + 1000 * ["a"], + [1, 1.3, 2L, 1L << 100, "hi", u"hi", [1, 2]], + object_ids[:1], + object_ids[:2], + object_ids[:3], + object_ids[:4], + object_ids[:5], + object_ids[:10], + object_ids[:100], + object_ids[:256], + [1, object_ids[0]], + [object_ids[0], "a"], + [1, object_ids[0], "a"], + [object_ids[0], 1, object_ids[1], "a"], + object_ids[:3] + [1, "hi", 2.3] + object_ids[:5], + object_ids + 100 * ["a"] + object_ids + ] + for args in args_list: + for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: + task = common.Task(function_id, args, num_return_vals) + self.assertEqual(function_id.id(), task.function_id().id()) + retrieved_args = task.arguments() + self.assertEqual(num_return_vals, len(task.returns())) + self.assertEqual(len(args), len(retrieved_args)) + for i in range(len(retrieved_args)): + if isinstance(retrieved_args[i], common.ObjectID): + self.assertEqual(retrieved_args[i].id(), args[i].id()) + else: + self.assertEqual(retrieved_args[i], args[i]) + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/src/common/thirdparty/ae/ae.c b/src/common/thirdparty/ae/ae.c new file mode 100644 index 000000000..e66808a81 --- /dev/null +++ b/src/common/thirdparty/ae/ae.c @@ -0,0 +1,465 @@ +/* A simple event-driven programming library. Originally I wrote this code + * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated + * it in form of a library for easy reuse. + * + * Copyright (c) 2006-2010, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ae.h" +#include "zmalloc.h" +#include "config.h" + +/* Include the best multiplexing layer supported by this system. + * The following should be ordered by performances, descending. */ +#ifdef HAVE_EVPORT +#include "ae_evport.c" +#else + #ifdef HAVE_EPOLL + #include "ae_epoll.c" + #else + #ifdef HAVE_KQUEUE + #include "ae_kqueue.c" + #else + #include "ae_select.c" + #endif + #endif +#endif + +aeEventLoop *aeCreateEventLoop(int setsize) { + aeEventLoop *eventLoop; + int i; + + if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err; + eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize); + eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize); + if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err; + eventLoop->setsize = setsize; + eventLoop->lastTime = time(NULL); + eventLoop->timeEventHead = NULL; + eventLoop->timeEventNextId = 0; + eventLoop->stop = 0; + eventLoop->maxfd = -1; + eventLoop->beforesleep = NULL; + if (aeApiCreate(eventLoop) == -1) goto err; + /* Events with mask == AE_NONE are not set. So let's initialize the + * vector with it. */ + for (i = 0; i < setsize; i++) + eventLoop->events[i].mask = AE_NONE; + return eventLoop; + +err: + if (eventLoop) { + zfree(eventLoop->events); + zfree(eventLoop->fired); + zfree(eventLoop); + } + return NULL; +} + +/* Return the current set size. */ +int aeGetSetSize(aeEventLoop *eventLoop) { + return eventLoop->setsize; +} + +/* Resize the maximum set size of the event loop. + * If the requested set size is smaller than the current set size, but + * there is already a file descriptor in use that is >= the requested + * set size minus one, AE_ERR is returned and the operation is not + * performed at all. + * + * Otherwise AE_OK is returned and the operation is successful. */ +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { + int i; + + if (setsize == eventLoop->setsize) return AE_OK; + if (eventLoop->maxfd >= setsize) return AE_ERR; + if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR; + + eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize); + eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize); + eventLoop->setsize = setsize; + + /* Make sure that if we created new slots, they are initialized with + * an AE_NONE mask. */ + for (i = eventLoop->maxfd+1; i < setsize; i++) + eventLoop->events[i].mask = AE_NONE; + return AE_OK; +} + +void aeDeleteEventLoop(aeEventLoop *eventLoop) { + aeApiFree(eventLoop); + zfree(eventLoop->events); + zfree(eventLoop->fired); + zfree(eventLoop); +} + +void aeStop(aeEventLoop *eventLoop) { + eventLoop->stop = 1; +} + +int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, + aeFileProc *proc, void *clientData) +{ + if (fd >= eventLoop->setsize) { + errno = ERANGE; + return AE_ERR; + } + aeFileEvent *fe = &eventLoop->events[fd]; + + if (aeApiAddEvent(eventLoop, fd, mask) == -1) + return AE_ERR; + fe->mask |= mask; + if (mask & AE_READABLE) fe->rfileProc = proc; + if (mask & AE_WRITABLE) fe->wfileProc = proc; + fe->clientData = clientData; + if (fd > eventLoop->maxfd) + eventLoop->maxfd = fd; + return AE_OK; +} + +void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) +{ + if (fd >= eventLoop->setsize) return; + aeFileEvent *fe = &eventLoop->events[fd]; + if (fe->mask == AE_NONE) return; + + aeApiDelEvent(eventLoop, fd, mask); + fe->mask = fe->mask & (~mask); + if (fd == eventLoop->maxfd && fe->mask == AE_NONE) { + /* Update the max fd */ + int j; + + for (j = eventLoop->maxfd-1; j >= 0; j--) + if (eventLoop->events[j].mask != AE_NONE) break; + eventLoop->maxfd = j; + } +} + +int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { + if (fd >= eventLoop->setsize) return 0; + aeFileEvent *fe = &eventLoop->events[fd]; + + return fe->mask; +} + +static void aeGetTime(long *seconds, long *milliseconds) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + *seconds = tv.tv_sec; + *milliseconds = tv.tv_usec/1000; +} + +static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) { + long cur_sec, cur_ms, when_sec, when_ms; + + aeGetTime(&cur_sec, &cur_ms); + when_sec = cur_sec + milliseconds/1000; + when_ms = cur_ms + milliseconds%1000; + if (when_ms >= 1000) { + when_sec ++; + when_ms -= 1000; + } + *sec = when_sec; + *ms = when_ms; +} + +long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, + aeTimeProc *proc, void *clientData, + aeEventFinalizerProc *finalizerProc) +{ + long long id = eventLoop->timeEventNextId++; + aeTimeEvent *te; + + te = zmalloc(sizeof(*te)); + if (te == NULL) return AE_ERR; + te->id = id; + aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms); + te->timeProc = proc; + te->finalizerProc = finalizerProc; + te->clientData = clientData; + te->next = eventLoop->timeEventHead; + eventLoop->timeEventHead = te; + return id; +} + +int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) +{ + aeTimeEvent *te = eventLoop->timeEventHead; + while(te) { + if (te->id == id) { + te->id = AE_DELETED_EVENT_ID; + return AE_OK; + } + te = te->next; + } + return AE_ERR; /* NO event with the specified ID found */ +} + +/* Search the first timer to fire. + * This operation is useful to know how many time the select can be + * put in sleep without to delay any event. + * If there are no timers NULL is returned. + * + * Note that's O(N) since time events are unsorted. + * Possible optimizations (not needed by Redis so far, but...): + * 1) Insert the event in order, so that the nearest is just the head. + * Much better but still insertion or deletion of timers is O(N). + * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)). + */ +static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop) +{ + aeTimeEvent *te = eventLoop->timeEventHead; + aeTimeEvent *nearest = NULL; + + while(te) { + if (!nearest || te->when_sec < nearest->when_sec || + (te->when_sec == nearest->when_sec && + te->when_ms < nearest->when_ms)) + nearest = te; + te = te->next; + } + return nearest; +} + +/* Process time events */ +static int processTimeEvents(aeEventLoop *eventLoop) { + int processed = 0; + aeTimeEvent *te, *prev; + long long maxId; + time_t now = time(NULL); + + /* If the system clock is moved to the future, and then set back to the + * right value, time events may be delayed in a random way. Often this + * means that scheduled operations will not be performed soon enough. + * + * Here we try to detect system clock skews, and force all the time + * events to be processed ASAP when this happens: the idea is that + * processing events earlier is less dangerous than delaying them + * indefinitely, and practice suggests it is. */ + if (now < eventLoop->lastTime) { + te = eventLoop->timeEventHead; + while(te) { + te->when_sec = 0; + te = te->next; + } + } + eventLoop->lastTime = now; + + prev = NULL; + te = eventLoop->timeEventHead; + maxId = eventLoop->timeEventNextId-1; + while(te) { + long now_sec, now_ms; + long long id; + + /* Remove events scheduled for deletion. */ + if (te->id == AE_DELETED_EVENT_ID) { + aeTimeEvent *next = te->next; + if (prev == NULL) + eventLoop->timeEventHead = te->next; + else + prev->next = te->next; + if (te->finalizerProc) + te->finalizerProc(eventLoop, te->clientData); + zfree(te); + te = next; + continue; + } + + /* Make sure we don't process time events created by time events in + * this iteration. Note that this check is currently useless: we always + * add new timers on the head, however if we change the implementation + * detail, this check may be useful again: we keep it here for future + * defense. */ + if (te->id > maxId) { + te = te->next; + continue; + } + aeGetTime(&now_sec, &now_ms); + if (now_sec > te->when_sec || + (now_sec == te->when_sec && now_ms >= te->when_ms)) + { + int retval; + + id = te->id; + retval = te->timeProc(eventLoop, id, te->clientData); + processed++; + if (retval != AE_NOMORE) { + aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms); + } else { + te->id = AE_DELETED_EVENT_ID; + } + } + prev = te; + te = te->next; + } + return processed; +} + +/* Process every pending time event, then every pending file event + * (that may be registered by time event callbacks just processed). + * Without special flags the function sleeps until some file event + * fires, or when the next time event occurs (if any). + * + * If flags is 0, the function does nothing and returns. + * if flags has AE_ALL_EVENTS set, all the kind of events are processed. + * if flags has AE_FILE_EVENTS set, file events are processed. + * if flags has AE_TIME_EVENTS set, time events are processed. + * if flags has AE_DONT_WAIT set the function returns ASAP until all + * the events that's possible to process without to wait are processed. + * + * The function returns the number of events processed. */ +int aeProcessEvents(aeEventLoop *eventLoop, int flags) +{ + int processed = 0, numevents; + + /* Nothing to do? return ASAP */ + if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0; + + /* Note that we want call select() even if there are no + * file events to process as long as we want to process time + * events, in order to sleep until the next time event is ready + * to fire. */ + if (eventLoop->maxfd != -1 || + ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) { + int j; + aeTimeEvent *shortest = NULL; + struct timeval tv, *tvp; + + if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT)) + shortest = aeSearchNearestTimer(eventLoop); + if (shortest) { + long now_sec, now_ms; + + aeGetTime(&now_sec, &now_ms); + tvp = &tv; + + /* How many milliseconds we need to wait for the next + * time event to fire? */ + long long ms = + (shortest->when_sec - now_sec)*1000 + + shortest->when_ms - now_ms; + + if (ms > 0) { + tvp->tv_sec = ms/1000; + tvp->tv_usec = (ms % 1000)*1000; + } else { + tvp->tv_sec = 0; + tvp->tv_usec = 0; + } + } else { + /* If we have to check for events but need to return + * ASAP because of AE_DONT_WAIT we need to set the timeout + * to zero */ + if (flags & AE_DONT_WAIT) { + tv.tv_sec = tv.tv_usec = 0; + tvp = &tv; + } else { + /* Otherwise we can block */ + tvp = NULL; /* wait forever */ + } + } + + numevents = aeApiPoll(eventLoop, tvp); + for (j = 0; j < numevents; j++) { + aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd]; + int mask = eventLoop->fired[j].mask; + int fd = eventLoop->fired[j].fd; + int rfired = 0; + + /* note the fe->mask & mask & ... code: maybe an already processed + * event removed an element that fired and we still didn't + * processed, so we check if the event is still valid. */ + if (fe->mask & mask & AE_READABLE) { + rfired = 1; + fe->rfileProc(eventLoop,fd,fe->clientData,mask); + } + if (fe->mask & mask & AE_WRITABLE) { + if (!rfired || fe->wfileProc != fe->rfileProc) + fe->wfileProc(eventLoop,fd,fe->clientData,mask); + } + processed++; + } + } + /* Check time events */ + if (flags & AE_TIME_EVENTS) + processed += processTimeEvents(eventLoop); + + return processed; /* return the number of processed file/time events */ +} + +/* Wait for milliseconds until the given file descriptor becomes + * writable/readable/exception */ +int aeWait(int fd, int mask, long long milliseconds) { + struct pollfd pfd; + int retmask = 0, retval; + + memset(&pfd, 0, sizeof(pfd)); + pfd.fd = fd; + if (mask & AE_READABLE) pfd.events |= POLLIN; + if (mask & AE_WRITABLE) pfd.events |= POLLOUT; + + if ((retval = poll(&pfd, 1, milliseconds))== 1) { + if (pfd.revents & POLLIN) retmask |= AE_READABLE; + if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE; + if (pfd.revents & POLLERR) retmask |= AE_WRITABLE; + if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE; + return retmask; + } else { + return retval; + } +} + +void aeMain(aeEventLoop *eventLoop) { + eventLoop->stop = 0; + while (!eventLoop->stop) { + if (eventLoop->beforesleep != NULL) + eventLoop->beforesleep(eventLoop); + aeProcessEvents(eventLoop, AE_ALL_EVENTS); + } +} + +char *aeGetApiName(void) { + return aeApiName(); +} + +void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) { + eventLoop->beforesleep = beforesleep; +} diff --git a/src/common/thirdparty/ae/ae.h b/src/common/thirdparty/ae/ae.h new file mode 100644 index 000000000..827c4c9e4 --- /dev/null +++ b/src/common/thirdparty/ae/ae.h @@ -0,0 +1,123 @@ +/* A simple event-driven programming library. Originally I wrote this code + * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated + * it in form of a library for easy reuse. + * + * Copyright (c) 2006-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __AE_H__ +#define __AE_H__ + +#include + +#define AE_OK 0 +#define AE_ERR -1 + +#define AE_NONE 0 +#define AE_READABLE 1 +#define AE_WRITABLE 2 + +#define AE_FILE_EVENTS 1 +#define AE_TIME_EVENTS 2 +#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS) +#define AE_DONT_WAIT 4 + +#define AE_NOMORE -1 +#define AE_DELETED_EVENT_ID -1 + +/* Macros */ +#define AE_NOTUSED(V) ((void) V) + +struct aeEventLoop; + +/* Types and data structures */ +typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask); +typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData); +typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData); +typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop); + +/* File event structure */ +typedef struct aeFileEvent { + int mask; /* one of AE_(READABLE|WRITABLE) */ + aeFileProc *rfileProc; + aeFileProc *wfileProc; + void *clientData; +} aeFileEvent; + +/* Time event structure */ +typedef struct aeTimeEvent { + long long id; /* time event identifier. */ + long when_sec; /* seconds */ + long when_ms; /* milliseconds */ + aeTimeProc *timeProc; + aeEventFinalizerProc *finalizerProc; + void *clientData; + struct aeTimeEvent *next; +} aeTimeEvent; + +/* A fired event */ +typedef struct aeFiredEvent { + int fd; + int mask; +} aeFiredEvent; + +/* State of an event based program */ +typedef struct aeEventLoop { + int maxfd; /* highest file descriptor currently registered */ + int setsize; /* max number of file descriptors tracked */ + long long timeEventNextId; + time_t lastTime; /* Used to detect system clock skew */ + aeFileEvent *events; /* Registered events */ + aeFiredEvent *fired; /* Fired events */ + aeTimeEvent *timeEventHead; + int stop; + void *apidata; /* This is used for polling API specific data */ + aeBeforeSleepProc *beforesleep; +} aeEventLoop; + +/* Prototypes */ +aeEventLoop *aeCreateEventLoop(int setsize); +void aeDeleteEventLoop(aeEventLoop *eventLoop); +void aeStop(aeEventLoop *eventLoop); +int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, + aeFileProc *proc, void *clientData); +void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask); +int aeGetFileEvents(aeEventLoop *eventLoop, int fd); +long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, + aeTimeProc *proc, void *clientData, + aeEventFinalizerProc *finalizerProc); +int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id); +int aeProcessEvents(aeEventLoop *eventLoop, int flags); +int aeWait(int fd, int mask, long long milliseconds); +void aeMain(aeEventLoop *eventLoop); +char *aeGetApiName(void); +void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep); +int aeGetSetSize(aeEventLoop *eventLoop); +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); + +#endif diff --git a/src/common/thirdparty/ae/ae_epoll.c b/src/common/thirdparty/ae/ae_epoll.c new file mode 100644 index 000000000..410aac70d --- /dev/null +++ b/src/common/thirdparty/ae/ae_epoll.c @@ -0,0 +1,135 @@ +/* Linux epoll(2) based ae.c module + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include + +typedef struct aeApiState { + int epfd; + struct epoll_event *events; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize); + if (!state->events) { + zfree(state); + return -1; + } + state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */ + if (state->epfd == -1) { + zfree(state->events); + zfree(state); + return -1; + } + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize); + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->epfd); + zfree(state->events); + zfree(state); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct epoll_event ee = {0}; /* avoid valgrind warning */ + /* If the fd was already monitored for some event, we need a MOD + * operation. Otherwise we need an ADD operation. */ + int op = eventLoop->events[fd].mask == AE_NONE ? + EPOLL_CTL_ADD : EPOLL_CTL_MOD; + + ee.events = 0; + mask |= eventLoop->events[fd].mask; /* Merge old events */ + if (mask & AE_READABLE) ee.events |= EPOLLIN; + if (mask & AE_WRITABLE) ee.events |= EPOLLOUT; + ee.data.fd = fd; + if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1; + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) { + aeApiState *state = eventLoop->apidata; + struct epoll_event ee = {0}; /* avoid valgrind warning */ + int mask = eventLoop->events[fd].mask & (~delmask); + + ee.events = 0; + if (mask & AE_READABLE) ee.events |= EPOLLIN; + if (mask & AE_WRITABLE) ee.events |= EPOLLOUT; + ee.data.fd = fd; + if (mask != AE_NONE) { + epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee); + } else { + /* Note, Kernel < 2.6.9 requires a non null event pointer even for + * EPOLL_CTL_DEL. */ + epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee); + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, numevents = 0; + + retval = epoll_wait(state->epfd,state->events,eventLoop->setsize, + tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1); + if (retval > 0) { + int j; + + numevents = retval; + for (j = 0; j < numevents; j++) { + int mask = 0; + struct epoll_event *e = state->events+j; + + if (e->events & EPOLLIN) mask |= AE_READABLE; + if (e->events & EPOLLOUT) mask |= AE_WRITABLE; + if (e->events & EPOLLERR) mask |= AE_WRITABLE; + if (e->events & EPOLLHUP) mask |= AE_WRITABLE; + eventLoop->fired[j].fd = e->data.fd; + eventLoop->fired[j].mask = mask; + } + } + return numevents; +} + +static char *aeApiName(void) { + return "epoll"; +} diff --git a/src/common/thirdparty/ae/ae_evport.c b/src/common/thirdparty/ae/ae_evport.c new file mode 100644 index 000000000..5c317becb --- /dev/null +++ b/src/common/thirdparty/ae/ae_evport.c @@ -0,0 +1,320 @@ +/* ae.c module for illumos event ports. + * + * Copyright (c) 2012, Joyent, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include +#include +#include + +#include +#include + +#include + +static int evport_debug = 0; + +/* + * This file implements the ae API using event ports, present on Solaris-based + * systems since Solaris 10. Using the event port interface, we associate file + * descriptors with the port. Each association also includes the set of poll(2) + * events that the consumer is interested in (e.g., POLLIN and POLLOUT). + * + * There's one tricky piece to this implementation: when we return events via + * aeApiPoll, the corresponding file descriptors become dissociated from the + * port. This is necessary because poll events are level-triggered, so if the + * fd didn't become dissociated, it would immediately fire another event since + * the underlying state hasn't changed yet. We must re-associate the file + * descriptor, but only after we know that our caller has actually read from it. + * The ae API does not tell us exactly when that happens, but we do know that + * it must happen by the time aeApiPoll is called again. Our solution is to + * keep track of the last fds returned by aeApiPoll and re-associate them next + * time aeApiPoll is invoked. + * + * To summarize, in this module, each fd association is EITHER (a) represented + * only via the in-kernel association OR (b) represented by pending_fds and + * pending_masks. (b) is only true for the last fds we returned from aeApiPoll, + * and only until we enter aeApiPoll again (at which point we restore the + * in-kernel association). + */ +#define MAX_EVENT_BATCHSZ 512 + +typedef struct aeApiState { + int portfd; /* event port */ + int npending; /* # of pending fds */ + int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */ + int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */ +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + int i; + aeApiState *state = zmalloc(sizeof(aeApiState)); + if (!state) return -1; + + state->portfd = port_create(); + if (state->portfd == -1) { + zfree(state); + return -1; + } + + state->npending = 0; + + for (i = 0; i < MAX_EVENT_BATCHSZ; i++) { + state->pending_fds[i] = -1; + state->pending_masks[i] = AE_NONE; + } + + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Nothing to resize here. */ + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->portfd); + zfree(state); +} + +static int aeApiLookupPending(aeApiState *state, int fd) { + int i; + + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == fd) + return (i); + } + + return (-1); +} + +/* + * Helper function to invoke port_associate for the given fd and mask. + */ +static int aeApiAssociate(const char *where, int portfd, int fd, int mask) { + int events = 0; + int rv, err; + + if (mask & AE_READABLE) + events |= POLLIN; + if (mask & AE_WRITABLE) + events |= POLLOUT; + + if (evport_debug) + fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events); + + rv = port_associate(portfd, PORT_SOURCE_FD, fd, events, + (void *)(uintptr_t)mask); + err = errno; + + if (evport_debug) + fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err)); + + if (rv == -1) { + fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err)); + + if (err == EAGAIN) + fprintf(stderr, "aeApiAssociate: event port limit exceeded."); + } + + return rv; +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask, pfd; + + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask); + + /* + * Since port_associate's "events" argument replaces any existing events, we + * must be sure to include whatever events are already associated when + * we call port_associate() again. + */ + fullmask = mask | eventLoop->events[fd].mask; + pfd = aeApiLookupPending(state, fd); + + if (pfd != -1) { + /* + * This fd was recently returned from aeApiPoll. It should be safe to + * assume that the consumer has processed that poll event, but we play + * it safer by simply updating pending_mask. The fd will be + * re-associated as usual when aeApiPoll is called again. + */ + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd); + state->pending_masks[pfd] |= fullmask; + return 0; + } + + return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask)); +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask, pfd; + + if (evport_debug) + fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask); + + pfd = aeApiLookupPending(state, fd); + + if (pfd != -1) { + if (evport_debug) + fprintf(stderr, "deleting event from pending fd %d\n", fd); + + /* + * This fd was just returned from aeApiPoll, so it's not currently + * associated with the port. All we need to do is update + * pending_mask appropriately. + */ + state->pending_masks[pfd] &= ~mask; + + if (state->pending_masks[pfd] == AE_NONE) + state->pending_fds[pfd] = -1; + + return; + } + + /* + * The fd is currently associated with the port. Like with the add case + * above, we must look at the full mask for the file descriptor before + * updating that association. We don't have a good way of knowing what the + * events are without looking into the eventLoop state directly. We rely on + * the fact that our caller has already updated the mask in the eventLoop. + */ + + fullmask = eventLoop->events[fd].mask; + if (fullmask == AE_NONE) { + /* + * We're removing *all* events, so use port_dissociate to remove the + * association completely. Failure here indicates a bug. + */ + if (evport_debug) + fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd); + + if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) { + perror("aeApiDelEvent: port_dissociate"); + abort(); /* will not return */ + } + } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd, + fullmask) != 0) { + /* + * ENOMEM is a potentially transient condition, but the kernel won't + * generally return it unless things are really bad. EAGAIN indicates + * we've reached an resource limit, for which it doesn't make sense to + * retry (counter-intuitively). All other errors indicate a bug. In any + * of these cases, the best we can do is to abort. + */ + abort(); /* will not return */ + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + struct timespec timeout, *tsp; + int mask, i; + uint_t nevents; + port_event_t event[MAX_EVENT_BATCHSZ]; + + /* + * If we've returned fd events before, we must re-associate them with the + * port now, before calling port_get(). See the block comment at the top of + * this file for an explanation of why. + */ + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == -1) + /* This fd has since been deleted. */ + continue; + + if (aeApiAssociate("aeApiPoll", state->portfd, + state->pending_fds[i], state->pending_masks[i]) != 0) { + /* See aeApiDelEvent for why this case is fatal. */ + abort(); + } + + state->pending_masks[i] = AE_NONE; + state->pending_fds[i] = -1; + } + + state->npending = 0; + + if (tvp != NULL) { + timeout.tv_sec = tvp->tv_sec; + timeout.tv_nsec = tvp->tv_usec * 1000; + tsp = &timeout; + } else { + tsp = NULL; + } + + /* + * port_getn can return with errno == ETIME having returned some events (!). + * So if we get ETIME, we check nevents, too. + */ + nevents = 1; + if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents, + tsp) == -1 && (errno != ETIME || nevents == 0)) { + if (errno == ETIME || errno == EINTR) + return 0; + + /* Any other error indicates a bug. */ + perror("aeApiPoll: port_get"); + abort(); + } + + state->npending = nevents; + + for (i = 0; i < nevents; i++) { + mask = 0; + if (event[i].portev_events & POLLIN) + mask |= AE_READABLE; + if (event[i].portev_events & POLLOUT) + mask |= AE_WRITABLE; + + eventLoop->fired[i].fd = event[i].portev_object; + eventLoop->fired[i].mask = mask; + + if (evport_debug) + fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n", + (int)event[i].portev_object, mask); + + state->pending_fds[i] = event[i].portev_object; + state->pending_masks[i] = (uintptr_t)event[i].portev_user; + } + + return nevents; +} + +static char *aeApiName(void) { + return "evport"; +} diff --git a/src/common/thirdparty/ae/ae_kqueue.c b/src/common/thirdparty/ae/ae_kqueue.c new file mode 100644 index 000000000..6796f4ceb --- /dev/null +++ b/src/common/thirdparty/ae/ae_kqueue.c @@ -0,0 +1,138 @@ +/* Kqueue(2)-based ae.c module + * + * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include +#include + +typedef struct aeApiState { + int kqfd; + struct kevent *events; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize); + if (!state->events) { + zfree(state); + return -1; + } + state->kqfd = kqueue(); + if (state->kqfd == -1) { + zfree(state->events); + zfree(state); + return -1; + } + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct kevent)*setsize); + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->kqfd); + zfree(state->events); + zfree(state); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct kevent ke; + + if (mask & AE_READABLE) { + EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL); + if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; + } + if (mask & AE_WRITABLE) { + EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL); + if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; + } + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct kevent ke; + + if (mask & AE_READABLE) { + EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); + kevent(state->kqfd, &ke, 1, NULL, 0, NULL); + } + if (mask & AE_WRITABLE) { + EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL); + kevent(state->kqfd, &ke, 1, NULL, 0, NULL); + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, numevents = 0; + + if (tvp != NULL) { + struct timespec timeout; + timeout.tv_sec = tvp->tv_sec; + timeout.tv_nsec = tvp->tv_usec * 1000; + retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize, + &timeout); + } else { + retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize, + NULL); + } + + if (retval > 0) { + int j; + + numevents = retval; + for(j = 0; j < numevents; j++) { + int mask = 0; + struct kevent *e = state->events+j; + + if (e->filter == EVFILT_READ) mask |= AE_READABLE; + if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE; + eventLoop->fired[j].fd = e->ident; + eventLoop->fired[j].mask = mask; + } + } + return numevents; +} + +static char *aeApiName(void) { + return "kqueue"; +} diff --git a/src/common/thirdparty/ae/ae_select.c b/src/common/thirdparty/ae/ae_select.c new file mode 100644 index 000000000..c039a8ea3 --- /dev/null +++ b/src/common/thirdparty/ae/ae_select.c @@ -0,0 +1,106 @@ +/* Select()-based ae.c module. + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include + +typedef struct aeApiState { + fd_set rfds, wfds; + /* We need to have a copy of the fd sets as it's not safe to reuse + * FD sets after select(). */ + fd_set _rfds, _wfds; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + FD_ZERO(&state->rfds); + FD_ZERO(&state->wfds); + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Just ensure we have enough room in the fd_set type. */ + if (setsize >= FD_SETSIZE) return -1; + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + zfree(eventLoop->apidata); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + + if (mask & AE_READABLE) FD_SET(fd,&state->rfds); + if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds); + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + + if (mask & AE_READABLE) FD_CLR(fd,&state->rfds); + if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds); +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, j, numevents = 0; + + memcpy(&state->_rfds,&state->rfds,sizeof(fd_set)); + memcpy(&state->_wfds,&state->wfds,sizeof(fd_set)); + + retval = select(eventLoop->maxfd+1, + &state->_rfds,&state->_wfds,NULL,tvp); + if (retval > 0) { + for (j = 0; j <= eventLoop->maxfd; j++) { + int mask = 0; + aeFileEvent *fe = &eventLoop->events[j]; + + if (fe->mask == AE_NONE) continue; + if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds)) + mask |= AE_READABLE; + if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds)) + mask |= AE_WRITABLE; + eventLoop->fired[numevents].fd = j; + eventLoop->fired[numevents].mask = mask; + numevents++; + } + } + return numevents; +} + +static char *aeApiName(void) { + return "select"; +} diff --git a/src/common/thirdparty/ae/config.h b/src/common/thirdparty/ae/config.h new file mode 100644 index 000000000..4f8e1ea1b --- /dev/null +++ b/src/common/thirdparty/ae/config.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __CONFIG_H +#define __CONFIG_H + +#ifdef __APPLE__ +#include +#endif + +/* Test for polling API */ +#ifdef __linux__ +#define HAVE_EPOLL 1 +#endif + +#if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__) +#define HAVE_KQUEUE 1 +#endif + +#ifdef __sun +#include +#ifdef _DTRACE_VERSION +#define HAVE_EVPORT 1 +#endif +#endif + + +#endif diff --git a/src/common/thirdparty/ae/zmalloc.h b/src/common/thirdparty/ae/zmalloc.h new file mode 100644 index 000000000..54c8a69cb --- /dev/null +++ b/src/common/thirdparty/ae/zmalloc.h @@ -0,0 +1,16 @@ +#ifndef _ZMALLOC_H +#define _ZMALLOC_H + +#ifndef zmalloc +#define zmalloc malloc +#endif + +#ifndef zfree +#define zfree free +#endif + +#ifndef zrealloc +#define zrealloc realloc +#endif + +#endif /* _ZMALLOC_H */ diff --git a/src/common/thirdparty/build-redis.sh b/src/common/thirdparty/build-redis.sh new file mode 100644 index 000000000..230e9ae29 --- /dev/null +++ b/src/common/thirdparty/build-redis.sh @@ -0,0 +1,6 @@ +if [ ! -f redis-3.2.3/src/redis-server ]; then + wget http://download.redis.io/releases/redis-3.2.3.tar.gz + tar xvfz redis-3.2.3.tar.gz + cd redis-3.2.3 + make +fi diff --git a/src/common/thirdparty/greatest.h b/src/common/thirdparty/greatest.h new file mode 100644 index 000000000..eb34ff426 --- /dev/null +++ b/src/common/thirdparty/greatest.h @@ -0,0 +1,1023 @@ +/* + * Copyright (c) 2011-2016 Scott Vokes + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef GREATEST_H +#define GREATEST_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* 1.2.1 */ +#define GREATEST_VERSION_MAJOR 1 +#define GREATEST_VERSION_MINOR 2 +#define GREATEST_VERSION_PATCH 1 + +/* A unit testing system for C, contained in 1 file. + * It doesn't use dynamic allocation or depend on anything + * beyond ANSI C89. + * + * An up-to-date version can be found at: + * https://github.com/silentbicycle/greatest/ + */ + + +/********************************************************************* + * Minimal test runner template + *********************************************************************/ +#if 0 +#include "greatest.h" +TEST foo_should_foo(void) { + PASS(); +} +static void setup_cb(void *data) { + printf("setup callback for each test case\n"); +} +static void teardown_cb(void *data) { + printf("teardown callback for each test case\n"); +} +SUITE(suite) { + /* Optional setup/teardown callbacks which will be run before/after + * every test case. If using a test suite, they will be cleared when + * the suite finishes. */ + SET_SETUP(setup_cb, voidp_to_callback_data); + SET_TEARDOWN(teardown_cb, voidp_to_callback_data); + RUN_TEST(foo_should_foo); +} +/* Add definitions that need to be in the test runner's main file. */ +GREATEST_MAIN_DEFS(); +/* Set up, run suite(s) of tests, report pass/fail/skip stats. */ +int run_tests(void) { + GREATEST_INIT(); /* init. greatest internals */ + /* List of suites to run (if any). */ + RUN_SUITE(suite); + /* Tests can also be run directly, without using test suites. */ + RUN_TEST(foo_should_foo); + GREATEST_PRINT_REPORT(); /* display results */ + return greatest_all_passed(); +} +/* main(), for a standalone command-line test runner. + * This replaces run_tests above, and adds command line option + * handling and exiting with a pass/fail status. */ +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); /* init & parse command-line args */ + RUN_SUITE(suite); + GREATEST_MAIN_END(); /* display results */ +} +#endif +/*********************************************************************/ + + +#include +#include +#include +#include + +/*********** + * Options * + ***********/ + +/* Default column width for non-verbose output. */ +#ifndef GREATEST_DEFAULT_WIDTH +#define GREATEST_DEFAULT_WIDTH 72 +#endif + +/* FILE *, for test logging. */ +#ifndef GREATEST_STDOUT +#define GREATEST_STDOUT stdout +#endif + +/* Remove GREATEST_ prefix from most commonly used symbols? */ +#ifndef GREATEST_USE_ABBREVS +#define GREATEST_USE_ABBREVS 1 +#endif + +/* Set to 0 to disable all use of setjmp/longjmp. */ +#ifndef GREATEST_USE_LONGJMP +#define GREATEST_USE_LONGJMP 1 +#endif + +#if GREATEST_USE_LONGJMP +#include +#endif + +/* Set to 0 to disable all use of time.h / clock(). */ +#ifndef GREATEST_USE_TIME +#define GREATEST_USE_TIME 1 +#endif + +#if GREATEST_USE_TIME +#include +#endif + +/* Floating point type, for ASSERT_IN_RANGE. */ +#ifndef GREATEST_FLOAT +#define GREATEST_FLOAT double +#define GREATEST_FLOAT_FMT "%g" +#endif + +/********* + * Types * + *********/ + +/* Info for the current running suite. */ +typedef struct greatest_suite_info { + unsigned int tests_run; + unsigned int passed; + unsigned int failed; + unsigned int skipped; + +#if GREATEST_USE_TIME + /* timers, pre/post running suite and individual tests */ + clock_t pre_suite; + clock_t post_suite; + clock_t pre_test; + clock_t post_test; +#endif +} greatest_suite_info; + +/* Type for a suite function. */ +typedef void (greatest_suite_cb)(void); + +/* Types for setup/teardown callbacks. If non-NULL, these will be run + * and passed the pointer to their additional data. */ +typedef void (greatest_setup_cb)(void *udata); +typedef void (greatest_teardown_cb)(void *udata); + +/* Type for an equality comparison between two pointers of the same type. + * Should return non-0 if equal, otherwise 0. + * UDATA is a closure value, passed through from ASSERT_EQUAL_T[m]. */ +typedef int greatest_equal_cb(const void *exp, const void *got, void *udata); + +/* Type for a callback that prints a value pointed to by T. + * Return value has the same meaning as printf's. + * UDATA is a closure value, passed through from ASSERT_EQUAL_T[m]. */ +typedef int greatest_printf_cb(const void *t, void *udata); + +/* Callbacks for an arbitrary type; needed for type-specific + * comparisons via GREATEST_ASSERT_EQUAL_T[m].*/ +typedef struct greatest_type_info { + greatest_equal_cb *equal; + greatest_printf_cb *print; +} greatest_type_info; + +typedef struct greatest_memory_cmp_env { + const unsigned char *exp; + const unsigned char *got; + size_t size; +} greatest_memory_cmp_env; + +/* Callbacks for string and raw memory types. */ +extern greatest_type_info greatest_type_info_string; +extern greatest_type_info greatest_type_info_memory; + +typedef enum { + GREATEST_FLAG_FIRST_FAIL = 0x01, + GREATEST_FLAG_LIST_ONLY = 0x02 +} greatest_flag_t; + +/* Struct containing all test runner state. */ +typedef struct greatest_run_info { + unsigned char flags; + unsigned char verbosity; + unsigned int tests_run; /* total test count */ + + /* overall pass/fail/skip counts */ + unsigned int passed; + unsigned int failed; + unsigned int skipped; + unsigned int assertions; + + /* currently running test suite */ + greatest_suite_info suite; + + /* info to print about the most recent failure */ + const char *fail_file; + unsigned int fail_line; + const char *msg; + + /* current setup/teardown hooks and userdata */ + greatest_setup_cb *setup; + void *setup_udata; + greatest_teardown_cb *teardown; + void *teardown_udata; + + /* formatting info for ".....s...F"-style output */ + unsigned int col; + unsigned int width; + + /* only run a specific suite or test */ + const char *suite_filter; + const char *test_filter; + +#if GREATEST_USE_TIME + /* overall timers */ + clock_t begin; + clock_t end; +#endif + +#if GREATEST_USE_LONGJMP + jmp_buf jump_dest; +#endif +} greatest_run_info; + +struct greatest_report_t { + /* overall pass/fail/skip counts */ + unsigned int passed; + unsigned int failed; + unsigned int skipped; + unsigned int assertions; +}; + +/* Global var for the current testing context. + * Initialized by GREATEST_MAIN_DEFS(). */ +extern greatest_run_info greatest_info; + +/* Type for ASSERT_ENUM_EQ's ENUM_STR argument. */ +typedef const char *greatest_enum_str_fun(int value); + +/********************** + * Exported functions * + **********************/ + +/* These are used internally by greatest. */ +void greatest_do_pass(const char *name); +void greatest_do_fail(const char *name); +void greatest_do_skip(const char *name); +int greatest_pre_test(const char *name); +void greatest_post_test(const char *name, int res); +void greatest_usage(const char *name); +int greatest_do_assert_equal_t(const void *exp, const void *got, + greatest_type_info *type_info, void *udata); + +/* These are part of the public greatest API. */ +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata); +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, void *udata); +int greatest_all_passed(void); +void greatest_set_test_filter(const char *name); +void greatest_set_suite_filter(const char *name); +void greatest_get_report(struct greatest_report_t *report); +unsigned int greatest_get_verbosity(void); +void greatest_set_verbosity(unsigned int verbosity); +void greatest_set_flag(greatest_flag_t flag); + + +/******************** +* Language Support * +********************/ + +/* If __VA_ARGS__ (C99) is supported, allow parametric testing +* without needing to manually manage the argument struct. */ +#if __STDC_VERSION__ >= 19901L || _MSC_VER >= 1800 +#define GREATEST_VA_ARGS +#endif + + +/********** + * Macros * + **********/ + +/* Define a suite. */ +#define GREATEST_SUITE(NAME) void NAME(void); void NAME(void) + +/* Declare a suite, provided by another compilation unit. */ +#define GREATEST_SUITE_EXTERN(NAME) void NAME(void) + +/* Start defining a test function. + * The arguments are not included, to allow parametric testing. */ +#define GREATEST_TEST static enum greatest_test_res + +/* PASS/FAIL/SKIP result from a test. Used internally. */ +typedef enum greatest_test_res { + GREATEST_TEST_RES_PASS = 0, + GREATEST_TEST_RES_FAIL = -1, + GREATEST_TEST_RES_SKIP = 1 +} greatest_test_res; + +/* Run a suite. */ +#define GREATEST_RUN_SUITE(S_NAME) greatest_run_suite(S_NAME, #S_NAME) + +/* Run a test in the current suite. */ +#define GREATEST_RUN_TEST(TEST) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + enum greatest_test_res res = GREATEST_SAVE_CONTEXT(); \ + if (res == GREATEST_TEST_RES_PASS) { \ + res = TEST(); \ + } \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +/* Ignore a test, don't warn about it being unused. */ +#define GREATEST_IGNORE_TEST(TEST) (void)TEST + +/* Run a test in the current suite with one void * argument, + * which can be a pointer to a struct with multiple arguments. */ +#define GREATEST_RUN_TEST1(TEST, ENV) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(ENV); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +#ifdef GREATEST_VA_ARGS +#define GREATEST_RUN_TESTp(TEST, ...) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(__VA_ARGS__); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) +#endif + + +/* Check if the test runner is in verbose mode. */ +#define GREATEST_IS_VERBOSE() ((greatest_info.verbosity) > 0) +#define GREATEST_LIST_ONLY() \ + (greatest_info.flags & GREATEST_FLAG_LIST_ONLY) +#define GREATEST_FIRST_FAIL() \ + (greatest_info.flags & GREATEST_FLAG_FIRST_FAIL) +#define GREATEST_FAILURE_ABORT() \ + (greatest_info.suite.failed > 0 && GREATEST_FIRST_FAIL()) + +/* Message-less forms of tests defined below. */ +#define GREATEST_PASS() GREATEST_PASSm(NULL) +#define GREATEST_FAIL() GREATEST_FAILm(NULL) +#define GREATEST_SKIP() GREATEST_SKIPm(NULL) +#define GREATEST_ASSERT(COND) \ + GREATEST_ASSERTm(#COND, COND) +#define GREATEST_ASSERT_OR_LONGJMP(COND) \ + GREATEST_ASSERT_OR_LONGJMPm(#COND, COND) +#define GREATEST_ASSERT_FALSE(COND) \ + GREATEST_ASSERT_FALSEm(#COND, COND) +#define GREATEST_ASSERT_EQ(EXP, GOT) \ + GREATEST_ASSERT_EQm(#EXP " != " #GOT, EXP, GOT) +#define GREATEST_ASSERT_EQ_FMT(EXP, GOT, FMT) \ + GREATEST_ASSERT_EQ_FMTm(#EXP " != " #GOT, EXP, GOT, FMT) +#define GREATEST_ASSERT_IN_RANGE(EXP, GOT, TOL) \ + GREATEST_ASSERT_IN_RANGEm(#EXP " != " #GOT " +/- " #TOL, EXP, GOT, TOL) +#define GREATEST_ASSERT_EQUAL_T(EXP, GOT, TYPE_INFO, UDATA) \ + GREATEST_ASSERT_EQUAL_Tm(#EXP " != " #GOT, EXP, GOT, TYPE_INFO, UDATA) +#define GREATEST_ASSERT_STR_EQ(EXP, GOT) \ + GREATEST_ASSERT_STR_EQm(#EXP " != " #GOT, EXP, GOT) +#define GREATEST_ASSERT_STRN_EQ(EXP, GOT, SIZE) \ + GREATEST_ASSERT_STRN_EQm(#EXP " != " #GOT, EXP, GOT, SIZE) +#define GREATEST_ASSERT_MEM_EQ(EXP, GOT, SIZE) \ + GREATEST_ASSERT_MEM_EQm(#EXP " != " #GOT, EXP, GOT, SIZE) +#define GREATEST_ASSERT_ENUM_EQ(EXP, GOT, ENUM_STR) \ + GREATEST_ASSERT_ENUM_EQm(#EXP " != " #GOT, EXP, GOT, ENUM_STR) + +/* The following forms take an additional message argument first, + * to be displayed by the test runner. */ + +/* Fail if a condition is not true, with message. */ +#define GREATEST_ASSERTm(MSG, COND) \ + do { \ + greatest_info.assertions++; \ + if (!(COND)) { GREATEST_FAILm(MSG); } \ + } while (0) + +/* Fail if a condition is not true, longjmping out of test. */ +#define GREATEST_ASSERT_OR_LONGJMPm(MSG, COND) \ + do { \ + greatest_info.assertions++; \ + if (!(COND)) { GREATEST_FAIL_WITH_LONGJMPm(MSG); } \ + } while (0) + +/* Fail if a condition is not false, with message. */ +#define GREATEST_ASSERT_FALSEm(MSG, COND) \ + do { \ + greatest_info.assertions++; \ + if ((COND)) { GREATEST_FAILm(MSG); } \ + } while (0) + +/* Fail if EXP != GOT (equality comparison by ==). */ +#define GREATEST_ASSERT_EQm(MSG, EXP, GOT) \ + do { \ + greatest_info.assertions++; \ + if ((EXP) != (GOT)) { GREATEST_FAILm(MSG); } \ + } while (0) + +/* Fail if EXP != GOT (equality comparison by ==). + * Warning: EXP and GOT will be evaluated more than once on failure. */ +#define GREATEST_ASSERT_EQ_FMTm(MSG, EXP, GOT, FMT) \ + do { \ + const char *greatest_FMT = ( FMT ); \ + greatest_info.assertions++; \ + if ((EXP) != (GOT)) { \ + fprintf(GREATEST_STDOUT, "\nExpected: "); \ + fprintf(GREATEST_STDOUT, greatest_FMT, EXP); \ + fprintf(GREATEST_STDOUT, "\n Got: "); \ + fprintf(GREATEST_STDOUT, greatest_FMT, GOT); \ + fprintf(GREATEST_STDOUT, "\n"); \ + GREATEST_FAILm(MSG); \ + } \ + } while (0) + +/* Fail if EXP is not equal to GOT, printing enum IDs. */ +#define GREATEST_ASSERT_ENUM_EQm(MSG, EXP, GOT, ENUM_STR) \ + do { \ + int greatest_EXP = (int)(EXP); \ + int greatest_GOT = (int)(GOT); \ + greatest_enum_str_fun *greatest_ENUM_STR = ENUM_STR; \ + if (greatest_EXP != greatest_GOT) { \ + fprintf(GREATEST_STDOUT, "\nExpected: %s", \ + greatest_ENUM_STR(greatest_EXP)); \ + fprintf(GREATEST_STDOUT, "\n Got: %s\n", \ + greatest_ENUM_STR(greatest_GOT)); \ + GREATEST_FAILm(MSG); \ + } \ + } while (0) \ + +/* Fail if GOT not in range of EXP +|- TOL. */ +#define GREATEST_ASSERT_IN_RANGEm(MSG, EXP, GOT, TOL) \ + do { \ + GREATEST_FLOAT greatest_EXP = (EXP); \ + GREATEST_FLOAT greatest_GOT = (GOT); \ + GREATEST_FLOAT greatest_TOL = (TOL); \ + greatest_info.assertions++; \ + if ((greatest_EXP > greatest_GOT && \ + greatest_EXP - greatest_GOT > greatest_TOL) || \ + (greatest_EXP < greatest_GOT && \ + greatest_GOT - greatest_EXP > greatest_TOL)) { \ + fprintf(GREATEST_STDOUT, \ + "\nExpected: " GREATEST_FLOAT_FMT \ + " +/- " GREATEST_FLOAT_FMT \ + "\n Got: " GREATEST_FLOAT_FMT \ + "\n", \ + greatest_EXP, greatest_TOL, greatest_GOT); \ + GREATEST_FAILm(MSG); \ + } \ + } while (0) + +/* Fail if EXP is not equal to GOT, according to strcmp. */ +#define GREATEST_ASSERT_STR_EQm(MSG, EXP, GOT) \ + do { \ + GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, \ + &greatest_type_info_string, NULL); \ + } while (0) \ + +/* Fail if EXP is not equal to GOT, according to strcmp. */ +#define GREATEST_ASSERT_STRN_EQm(MSG, EXP, GOT, SIZE) \ + do { \ + size_t size = SIZE; \ + GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, \ + &greatest_type_info_string, &size); \ + } while (0) \ + +/* Fail if EXP is not equal to GOT, according to memcmp. */ +#define GREATEST_ASSERT_MEM_EQm(MSG, EXP, GOT, SIZE) \ + do { \ + greatest_memory_cmp_env env; \ + env.exp = (const unsigned char *)EXP; \ + env.got = (const unsigned char *)GOT; \ + env.size = SIZE; \ + GREATEST_ASSERT_EQUAL_Tm(MSG, env.exp, env.got, \ + &greatest_type_info_memory, &env); \ + } while (0) \ + +/* Fail if EXP is not equal to GOT, according to a comparison + * callback in TYPE_INFO. If they are not equal, optionally use a + * print callback in TYPE_INFO to print them. */ +#define GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, TYPE_INFO, UDATA) \ + do { \ + greatest_type_info *type_info = (TYPE_INFO); \ + greatest_info.assertions++; \ + if (!greatest_do_assert_equal_t(EXP, GOT, \ + type_info, UDATA)) { \ + if (type_info == NULL || type_info->equal == NULL) { \ + GREATEST_FAILm("type_info->equal callback missing!"); \ + } else { \ + GREATEST_FAILm(MSG); \ + } \ + } \ + } while (0) \ + +/* Pass. */ +#define GREATEST_PASSm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return GREATEST_TEST_RES_PASS; \ + } while (0) + +/* Fail. */ +#define GREATEST_FAILm(MSG) \ + do { \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + greatest_info.msg = MSG; \ + return GREATEST_TEST_RES_FAIL; \ + } while (0) + +/* Optional GREATEST_FAILm variant that longjmps. */ +#if GREATEST_USE_LONGJMP +#define GREATEST_FAIL_WITH_LONGJMP() GREATEST_FAIL_WITH_LONGJMPm(NULL) +#define GREATEST_FAIL_WITH_LONGJMPm(MSG) \ + do { \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + greatest_info.msg = MSG; \ + longjmp(greatest_info.jump_dest, GREATEST_TEST_RES_FAIL); \ + } while (0) +#endif + +/* Skip the current test. */ +#define GREATEST_SKIPm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return GREATEST_TEST_RES_SKIP; \ + } while (0) + +/* Check the result of a subfunction using ASSERT, etc. */ +#define GREATEST_CHECK_CALL(RES) \ + do { \ + enum greatest_test_res greatest_RES = RES; \ + if (greatest_RES != GREATEST_TEST_RES_PASS) { \ + return greatest_RES; \ + } \ + } while (0) \ + +#if GREATEST_USE_TIME +#define GREATEST_SET_TIME(NAME) \ + NAME = clock(); \ + if (NAME == (clock_t) -1) { \ + fprintf(GREATEST_STDOUT, \ + "clock error: %s\n", #NAME); \ + exit(EXIT_FAILURE); \ + } + +#define GREATEST_CLOCK_DIFF(C1, C2) \ + fprintf(GREATEST_STDOUT, " (%lu ticks, %.3f sec)", \ + (long unsigned int) (C2) - (long unsigned int)(C1), \ + (double)((C2) - (C1)) / (1.0 * (double)CLOCKS_PER_SEC)) +#else +#define GREATEST_SET_TIME(UNUSED) +#define GREATEST_CLOCK_DIFF(UNUSED1, UNUSED2) +#endif + +#if GREATEST_USE_LONGJMP +#define GREATEST_SAVE_CONTEXT() \ + /* setjmp returns 0 (GREATEST_TEST_RES_PASS) on first call */ \ + /* so the test runs, then RES_FAIL from FAIL_WITH_LONGJMP. */ \ + ((enum greatest_test_res)(setjmp(greatest_info.jump_dest))) +#else +#define GREATEST_SAVE_CONTEXT() \ + /*a no-op, since setjmp/longjmp aren't being used */ \ + GREATEST_TEST_RES_PASS +#endif + +/* Include several function definitions in the main test file. */ +#define GREATEST_MAIN_DEFS() \ + \ +/* Is FILTER a subset of NAME? */ \ +static int greatest_name_match(const char *name, \ + const char *filter) { \ + size_t offset = 0; \ + size_t filter_len = strlen(filter); \ + while (name[offset] != '\0') { \ + if (name[offset] == filter[0]) { \ + if (0 == strncmp(&name[offset], filter, filter_len)) { \ + return 1; \ + } \ + } \ + offset++; \ + } \ + \ + return 0; \ +} \ + \ +int greatest_pre_test(const char *name) { \ + if (!GREATEST_LIST_ONLY() \ + && (!GREATEST_FIRST_FAIL() || greatest_info.suite.failed == 0) \ + && (greatest_info.test_filter == NULL || \ + greatest_name_match(name, greatest_info.test_filter))) { \ + GREATEST_SET_TIME(greatest_info.suite.pre_test); \ + if (greatest_info.setup) { \ + greatest_info.setup(greatest_info.setup_udata); \ + } \ + return 1; /* test should be run */ \ + } else { \ + return 0; /* skipped */ \ + } \ +} \ + \ +void greatest_post_test(const char *name, int res) { \ + GREATEST_SET_TIME(greatest_info.suite.post_test); \ + if (greatest_info.teardown) { \ + void *udata = greatest_info.teardown_udata; \ + greatest_info.teardown(udata); \ + } \ + \ + if (res <= GREATEST_TEST_RES_FAIL) { \ + greatest_do_fail(name); \ + } else if (res >= GREATEST_TEST_RES_SKIP) { \ + greatest_do_skip(name); \ + } else if (res == GREATEST_TEST_RES_PASS) { \ + greatest_do_pass(name); \ + } \ + greatest_info.suite.tests_run++; \ + greatest_info.col++; \ + if (GREATEST_IS_VERBOSE()) { \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_test, \ + greatest_info.suite.post_test); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } else if (greatest_info.col % greatest_info.width == 0) { \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + } \ + if (GREATEST_STDOUT == stdout) fflush(stdout); \ +} \ + \ +static void report_suite(void) { \ + if (greatest_info.suite.tests_run > 0) { \ + fprintf(GREATEST_STDOUT, \ + "\n%u test%s - %u passed, %u failed, %u skipped", \ + greatest_info.suite.tests_run, \ + greatest_info.suite.tests_run == 1 ? "" : "s", \ + greatest_info.suite.passed, \ + greatest_info.suite.failed, \ + greatest_info.suite.skipped); \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_suite, \ + greatest_info.suite.post_suite); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } \ +} \ + \ +static void update_counts_and_reset_suite(void) { \ + greatest_info.setup = NULL; \ + greatest_info.setup_udata = NULL; \ + greatest_info.teardown = NULL; \ + greatest_info.teardown_udata = NULL; \ + greatest_info.passed += greatest_info.suite.passed; \ + greatest_info.failed += greatest_info.suite.failed; \ + greatest_info.skipped += greatest_info.suite.skipped; \ + greatest_info.tests_run += greatest_info.suite.tests_run; \ + memset(&greatest_info.suite, 0, sizeof(greatest_info.suite)); \ + greatest_info.col = 0; \ +} \ + \ +static void greatest_run_suite(greatest_suite_cb *suite_cb, \ + const char *suite_name) { \ + if (greatest_info.suite_filter && \ + !greatest_name_match(suite_name, greatest_info.suite_filter)) { \ + return; \ + } \ + update_counts_and_reset_suite(); \ + if (GREATEST_FIRST_FAIL() && greatest_info.failed > 0) { return; } \ + fprintf(GREATEST_STDOUT, "\n* Suite %s:\n", suite_name); \ + GREATEST_SET_TIME(greatest_info.suite.pre_suite); \ + suite_cb(); \ + GREATEST_SET_TIME(greatest_info.suite.post_suite); \ + report_suite(); \ +} \ + \ +void greatest_do_pass(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "PASS %s: %s", \ + name, greatest_info.msg ? greatest_info.msg : ""); \ + } else { \ + fprintf(GREATEST_STDOUT, "."); \ + } \ + greatest_info.suite.passed++; \ +} \ + \ +void greatest_do_fail(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, \ + "FAIL %s: %s (%s:%u)", \ + name, greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } else { \ + fprintf(GREATEST_STDOUT, "F"); \ + greatest_info.col++; \ + /* add linebreak if in line of '.'s */ \ + if (greatest_info.col != 0) { \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + } \ + fprintf(GREATEST_STDOUT, "FAIL %s: %s (%s:%u)\n", \ + name, \ + greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } \ + greatest_info.suite.failed++; \ +} \ + \ +void greatest_do_skip(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "SKIP %s: %s", \ + name, \ + greatest_info.msg ? \ + greatest_info.msg : "" ); \ + } else { \ + fprintf(GREATEST_STDOUT, "s"); \ + } \ + greatest_info.suite.skipped++; \ +} \ + \ +int greatest_do_assert_equal_t(const void *exp, const void *got, \ + greatest_type_info *type_info, void *udata) { \ + int eq = 0; \ + if (type_info == NULL || type_info->equal == NULL) { \ + return 0; \ + } \ + eq = type_info->equal(exp, got, udata); \ + if (!eq) { \ + if (type_info->print != NULL) { \ + fprintf(GREATEST_STDOUT, "\nExpected: "); \ + (void)type_info->print(exp, udata); \ + fprintf(GREATEST_STDOUT, "\n Got: "); \ + (void)type_info->print(got, udata); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } else { \ + fprintf(GREATEST_STDOUT, \ + "GREATEST_ASSERT_EQUAL_T failure at %s:%u\n", \ + greatest_info.fail_file, \ + greatest_info.fail_line); \ + } \ + } \ + return eq; \ +} \ + \ +void greatest_usage(const char *name) { \ + fprintf(GREATEST_STDOUT, \ + "Usage: %s [-hlfv] [-s SUITE] [-t TEST]\n" \ + " -h, --help print this Help\n" \ + " -l List suites and their tests, then exit\n" \ + " -f Stop runner after first failure\n" \ + " -v Verbose output\n" \ + " -s SUITE only run suites containing string SUITE\n" \ + " -t TEST only run tests containing string TEST\n", \ + name); \ +} \ + \ +static void greatest_parse_args(int argc, char **argv) { \ + int i = 0; \ + for (i = 1; i < argc; i++) { \ + if (0 == strncmp("-t", argv[i], 2)) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.test_filter = argv[i+1]; \ + i++; \ + } else if (0 == strncmp("-s", argv[i], 2)) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.suite_filter = argv[i+1]; \ + i++; \ + } else if (0 == strncmp("-f", argv[i], 2)) { \ + greatest_info.flags |= GREATEST_FLAG_FIRST_FAIL; \ + } else if (0 == strncmp("-v", argv[i], 2)) { \ + greatest_info.verbosity++; \ + } else if (0 == strncmp("-l", argv[i], 2)) { \ + greatest_info.flags |= GREATEST_FLAG_LIST_ONLY; \ + } else if (0 == strncmp("-h", argv[i], 2) || \ + 0 == strncmp("--help", argv[i], 6)) { \ + greatest_usage(argv[0]); \ + exit(EXIT_SUCCESS); \ + } else if (0 == strncmp("--", argv[i], 2)) { \ + break; \ + } else { \ + fprintf(GREATEST_STDOUT, \ + "Unknown argument '%s'\n", argv[i]); \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + } \ +} \ + \ +int greatest_all_passed(void) { return (greatest_info.failed == 0); } \ + \ +void greatest_set_test_filter(const char *name) { \ + greatest_info.test_filter = name; \ +} \ + \ +void greatest_set_suite_filter(const char *name) { \ + greatest_info.suite_filter = name; \ +} \ + \ +void greatest_get_report(struct greatest_report_t *report) { \ + if (report) { \ + report->passed = greatest_info.passed; \ + report->failed = greatest_info.failed; \ + report->skipped = greatest_info.skipped; \ + report->assertions = greatest_info.assertions; \ + } \ +} \ + \ +unsigned int greatest_get_verbosity(void) { \ + return greatest_info.verbosity; \ +} \ + \ +void greatest_set_verbosity(unsigned int verbosity) { \ + greatest_info.verbosity = (unsigned char)verbosity; \ +} \ + \ +void greatest_set_flag(greatest_flag_t flag) { \ + greatest_info.flags |= flag; \ +} \ + \ +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata) { \ + greatest_info.setup = cb; \ + greatest_info.setup_udata = udata; \ +} \ + \ +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, \ + void *udata) { \ + greatest_info.teardown = cb; \ + greatest_info.teardown_udata = udata; \ +} \ + \ +static int greatest_string_equal_cb(const void *exp, const void *got, \ + void *udata) { \ + size_t *size = (size_t *)udata; \ + return (size != NULL \ + ? (0 == strncmp((const char *)exp, (const char *)got, *size)) \ + : (0 == strcmp((const char *)exp, (const char *)got))); \ +} \ + \ +static int greatest_string_printf_cb(const void *t, void *udata) { \ + (void)udata; /* note: does not check \0 termination. */ \ + return fprintf(GREATEST_STDOUT, "%s", (const char *)t); \ +} \ + \ +greatest_type_info greatest_type_info_string = { \ + greatest_string_equal_cb, \ + greatest_string_printf_cb, \ +}; \ + \ +static int greatest_memory_equal_cb(const void *exp, const void *got, \ + void *udata) { \ + greatest_memory_cmp_env *env = (greatest_memory_cmp_env *)udata; \ + return (0 == memcmp(exp, got, env->size)); \ +} \ + \ +static int greatest_memory_printf_cb(const void *t, void *udata) { \ + greatest_memory_cmp_env *env = (greatest_memory_cmp_env *)udata; \ + unsigned char *buf = (unsigned char *)t, diff_mark = ' '; \ + FILE *out = GREATEST_STDOUT; \ + size_t i, line_i, line_len = 0; \ + int len = 0; /* format hexdump with differences highlighted */ \ + for (i = 0; i < env->size; i+= line_len) { \ + diff_mark = ' '; \ + line_len = env->size - i; \ + if (line_len > 16) { line_len = 16; } \ + for (line_i = i; line_i < i + line_len; line_i++) { \ + if (env->exp[line_i] != env->got[line_i]) diff_mark = 'X'; \ + } \ + len += fprintf(out, "\n%04x %c ", (unsigned int)i, diff_mark); \ + for (line_i = i; line_i < i + line_len; line_i++) { \ + int m = env->exp[line_i] == env->got[line_i]; /* match? */ \ + len += fprintf(out, "%02x%c", buf[line_i], m ? ' ' : '<'); \ + } \ + for (line_i = 0; line_i < 16 - line_len; line_i++) { \ + len += fprintf(out, " "); \ + } \ + fprintf(out, " "); \ + for (line_i = i; line_i < i + line_len; line_i++) { \ + unsigned char c = buf[line_i]; \ + len += fprintf(out, "%c", isprint(c) ? c : '.'); \ + } \ + } \ + len += fprintf(out, "\n"); \ + return len; \ +} \ + \ +greatest_type_info greatest_type_info_memory = { \ + greatest_memory_equal_cb, \ + greatest_memory_printf_cb, \ +}; \ + \ +greatest_run_info greatest_info + +/* Init internals. */ +#define GREATEST_INIT() \ + do { \ + /* Suppress unused function warning if features aren't used */ \ + (void)greatest_run_suite; \ + (void)greatest_parse_args; \ + \ + memset(&greatest_info, 0, sizeof(greatest_info)); \ + greatest_info.width = GREATEST_DEFAULT_WIDTH; \ + GREATEST_SET_TIME(greatest_info.begin); \ + } while (0) \ + +/* Handle command-line arguments, etc. */ +#define GREATEST_MAIN_BEGIN() \ + do { \ + GREATEST_INIT(); \ + greatest_parse_args(argc, argv); \ + } while (0) + +/* Report passes, failures, skipped tests, the number of + * assertions, and the overall run time. */ +#define GREATEST_PRINT_REPORT() \ + do { \ + if (!GREATEST_LIST_ONLY()) { \ + update_counts_and_reset_suite(); \ + GREATEST_SET_TIME(greatest_info.end); \ + fprintf(GREATEST_STDOUT, \ + "\nTotal: %u test%s", \ + greatest_info.tests_run, \ + greatest_info.tests_run == 1 ? "" : "s"); \ + GREATEST_CLOCK_DIFF(greatest_info.begin, \ + greatest_info.end); \ + fprintf(GREATEST_STDOUT, ", %u assertion%s\n", \ + greatest_info.assertions, \ + greatest_info.assertions == 1 ? "" : "s"); \ + fprintf(GREATEST_STDOUT, \ + "Pass: %u, fail: %u, skip: %u.\n", \ + greatest_info.passed, \ + greatest_info.failed, greatest_info.skipped); \ + } \ + } while (0) + +/* Report results, exit with exit status based on results. */ +#define GREATEST_MAIN_END() \ + do { \ + GREATEST_PRINT_REPORT(); \ + return (greatest_all_passed() ? EXIT_SUCCESS : EXIT_FAILURE); \ + } while (0) + +/* Make abbreviations without the GREATEST_ prefix for the + * most commonly used symbols. */ +#if GREATEST_USE_ABBREVS +#define TEST GREATEST_TEST +#define SUITE GREATEST_SUITE +#define SUITE_EXTERN GREATEST_SUITE_EXTERN +#define RUN_TEST GREATEST_RUN_TEST +#define RUN_TEST1 GREATEST_RUN_TEST1 +#define RUN_SUITE GREATEST_RUN_SUITE +#define IGNORE_TEST GREATEST_IGNORE_TEST +#define ASSERT GREATEST_ASSERT +#define ASSERTm GREATEST_ASSERTm +#define ASSERT_FALSE GREATEST_ASSERT_FALSE +#define ASSERT_EQ GREATEST_ASSERT_EQ +#define ASSERT_EQ_FMT GREATEST_ASSERT_EQ_FMT +#define ASSERT_IN_RANGE GREATEST_ASSERT_IN_RANGE +#define ASSERT_EQUAL_T GREATEST_ASSERT_EQUAL_T +#define ASSERT_STR_EQ GREATEST_ASSERT_STR_EQ +#define ASSERT_STRN_EQ GREATEST_ASSERT_STRN_EQ +#define ASSERT_MEM_EQ GREATEST_ASSERT_MEM_EQ +#define ASSERT_ENUM_EQ GREATEST_ASSERT_ENUM_EQ +#define ASSERT_FALSEm GREATEST_ASSERT_FALSEm +#define ASSERT_EQm GREATEST_ASSERT_EQm +#define ASSERT_EQ_FMTm GREATEST_ASSERT_EQ_FMTm +#define ASSERT_IN_RANGEm GREATEST_ASSERT_IN_RANGEm +#define ASSERT_EQUAL_Tm GREATEST_ASSERT_EQUAL_Tm +#define ASSERT_STR_EQm GREATEST_ASSERT_STR_EQm +#define ASSERT_STRN_EQm GREATEST_ASSERT_STRN_EQm +#define ASSERT_MEM_EQm GREATEST_ASSERT_MEM_EQm +#define ASSERT_ENUM_EQm GREATEST_ASSERT_ENUM_EQm +#define PASS GREATEST_PASS +#define FAIL GREATEST_FAIL +#define SKIP GREATEST_SKIP +#define PASSm GREATEST_PASSm +#define FAILm GREATEST_FAILm +#define SKIPm GREATEST_SKIPm +#define SET_SETUP GREATEST_SET_SETUP_CB +#define SET_TEARDOWN GREATEST_SET_TEARDOWN_CB +#define CHECK_CALL GREATEST_CHECK_CALL + +#ifdef GREATEST_VA_ARGS +#define RUN_TESTp GREATEST_RUN_TESTp +#endif + +#if GREATEST_USE_LONGJMP +#define ASSERT_OR_LONGJMP GREATEST_ASSERT_OR_LONGJMP +#define ASSERT_OR_LONGJMPm GREATEST_ASSERT_OR_LONGJMPm +#define FAIL_WITH_LONGJMP GREATEST_FAIL_WITH_LONGJMP +#define FAIL_WITH_LONGJMPm GREATEST_FAIL_WITH_LONGJMPm +#endif + +#endif /* USE_ABBREVS */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/thirdparty/hiredis b/src/common/thirdparty/hiredis similarity index 100% rename from thirdparty/hiredis rename to src/common/thirdparty/hiredis diff --git a/src/common/thirdparty/utarray.h b/src/common/thirdparty/utarray.h new file mode 100644 index 000000000..979e99e98 --- /dev/null +++ b/src/common/thirdparty/utarray.h @@ -0,0 +1,238 @@ +/* +Copyright (c) 2008-2016, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* a dynamic array implementation using macros + */ +#ifndef UTARRAY_H +#define UTARRAY_H + +#define UTARRAY_VERSION 2.0.1 + +#ifdef __GNUC__ +#define _UNUSED_ __attribute__ ((__unused__)) +#else +#define _UNUSED_ +#endif + +#include /* size_t */ +#include /* memset, etc */ +#include /* exit */ + +#ifndef oom +#define oom() exit(-1) +#endif + +typedef void (ctor_f)(void *dst, const void *src); +typedef void (dtor_f)(void *elt); +typedef void (init_f)(void *elt); +typedef struct { + size_t sz; + init_f *init; + ctor_f *copy; + dtor_f *dtor; +} UT_icd; + +typedef struct { + unsigned i,n;/* i: index of next available slot, n: num slots */ + UT_icd icd; /* initializer, copy and destructor functions */ + char *d; /* n slots of size icd->sz*/ +} UT_array; + +#define utarray_init(a,_icd) do { \ + memset(a,0,sizeof(UT_array)); \ + (a)->icd = *(_icd); \ +} while(0) + +#define utarray_done(a) do { \ + if ((a)->n) { \ + if ((a)->icd.dtor) { \ + unsigned _ut_i; \ + for(_ut_i=0; _ut_i < (a)->i; _ut_i++) { \ + (a)->icd.dtor(utarray_eltptr(a,_ut_i)); \ + } \ + } \ + free((a)->d); \ + } \ + (a)->n=0; \ +} while(0) + +#define utarray_new(a,_icd) do { \ + (a) = (UT_array*)malloc(sizeof(UT_array)); \ + if ((a) == NULL) oom(); \ + utarray_init(a,_icd); \ +} while(0) + +#define utarray_free(a) do { \ + utarray_done(a); \ + free(a); \ +} while(0) + +#define utarray_reserve(a,by) do { \ + if (((a)->i+(by)) > (a)->n) { \ + char *utarray_tmp; \ + while (((a)->i+(by)) > (a)->n) { (a)->n = ((a)->n ? (2*(a)->n) : 8); } \ + utarray_tmp=(char*)realloc((a)->d, (a)->n*(a)->icd.sz); \ + if (utarray_tmp == NULL) oom(); \ + (a)->d=utarray_tmp; \ + } \ +} while(0) + +#define utarray_push_back(a,p) do { \ + utarray_reserve(a,1); \ + if ((a)->icd.copy) { (a)->icd.copy( _utarray_eltptr(a,(a)->i++), p); } \ + else { memcpy(_utarray_eltptr(a,(a)->i++), p, (a)->icd.sz); }; \ +} while(0) + +#define utarray_pop_back(a) do { \ + if ((a)->icd.dtor) { (a)->icd.dtor( _utarray_eltptr(a,--((a)->i))); } \ + else { (a)->i--; } \ +} while(0) + +#define utarray_extend_back(a) do { \ + utarray_reserve(a,1); \ + if ((a)->icd.init) { (a)->icd.init(_utarray_eltptr(a,(a)->i)); } \ + else { memset(_utarray_eltptr(a,(a)->i),0,(a)->icd.sz); } \ + (a)->i++; \ +} while(0) + +#define utarray_len(a) ((a)->i) + +#define utarray_eltptr(a,j) (((j) < (a)->i) ? _utarray_eltptr(a,j) : NULL) +#define _utarray_eltptr(a,j) ((a)->d + ((a)->icd.sz * (j))) + +#define utarray_insert(a,p,j) do { \ + if ((j) > (a)->i) utarray_resize(a,j); \ + utarray_reserve(a,1); \ + if ((j) < (a)->i) { \ + memmove( _utarray_eltptr(a,(j)+1), _utarray_eltptr(a,j), \ + ((a)->i - (j))*((a)->icd.sz)); \ + } \ + if ((a)->icd.copy) { (a)->icd.copy( _utarray_eltptr(a,j), p); } \ + else { memcpy(_utarray_eltptr(a,j), p, (a)->icd.sz); }; \ + (a)->i++; \ +} while(0) + +#define utarray_inserta(a,w,j) do { \ + if (utarray_len(w) == 0) break; \ + if ((j) > (a)->i) utarray_resize(a,j); \ + utarray_reserve(a,utarray_len(w)); \ + if ((j) < (a)->i) { \ + memmove(_utarray_eltptr(a,(j)+utarray_len(w)), \ + _utarray_eltptr(a,j), \ + ((a)->i - (j))*((a)->icd.sz)); \ + } \ + if ((a)->icd.copy) { \ + unsigned _ut_i; \ + for(_ut_i=0;_ut_i<(w)->i;_ut_i++) { \ + (a)->icd.copy(_utarray_eltptr(a, (j) + _ut_i), _utarray_eltptr(w, _ut_i)); \ + } \ + } else { \ + memcpy(_utarray_eltptr(a,j), _utarray_eltptr(w,0), \ + utarray_len(w)*((a)->icd.sz)); \ + } \ + (a)->i += utarray_len(w); \ +} while(0) + +#define utarray_resize(dst,num) do { \ + unsigned _ut_i; \ + if ((dst)->i > (unsigned)(num)) { \ + if ((dst)->icd.dtor) { \ + for (_ut_i = (num); _ut_i < (dst)->i; ++_ut_i) { \ + (dst)->icd.dtor(_utarray_eltptr(dst, _ut_i)); \ + } \ + } \ + } else if ((dst)->i < (unsigned)(num)) { \ + utarray_reserve(dst, (num) - (dst)->i); \ + if ((dst)->icd.init) { \ + for (_ut_i = (dst)->i; _ut_i < (unsigned)(num); ++_ut_i) { \ + (dst)->icd.init(_utarray_eltptr(dst, _ut_i)); \ + } \ + } else { \ + memset(_utarray_eltptr(dst, (dst)->i), 0, (dst)->icd.sz*((num) - (dst)->i)); \ + } \ + } \ + (dst)->i = (num); \ +} while(0) + +#define utarray_concat(dst,src) do { \ + utarray_inserta(dst, src, utarray_len(dst)); \ +} while(0) + +#define utarray_erase(a,pos,len) do { \ + if ((a)->icd.dtor) { \ + unsigned _ut_i; \ + for (_ut_i = 0; _ut_i < (len); _ut_i++) { \ + (a)->icd.dtor(utarray_eltptr(a, (pos) + _ut_i)); \ + } \ + } \ + if ((a)->i > ((pos) + (len))) { \ + memmove(_utarray_eltptr(a, pos), _utarray_eltptr(a, (pos) + (len)), \ + ((a)->i - ((pos) + (len))) * (a)->icd.sz); \ + } \ + (a)->i -= (len); \ +} while(0) + +#define utarray_renew(a,u) do { \ + if (a) utarray_clear(a); \ + else utarray_new(a, u); \ +} while(0) + +#define utarray_clear(a) do { \ + if ((a)->i > 0) { \ + if ((a)->icd.dtor) { \ + unsigned _ut_i; \ + for(_ut_i=0; _ut_i < (a)->i; _ut_i++) { \ + (a)->icd.dtor(_utarray_eltptr(a, _ut_i)); \ + } \ + } \ + (a)->i = 0; \ + } \ +} while(0) + +#define utarray_sort(a,cmp) do { \ + qsort((a)->d, (a)->i, (a)->icd.sz, cmp); \ +} while(0) + +#define utarray_find(a,v,cmp) bsearch((v),(a)->d,(a)->i,(a)->icd.sz,cmp) + +#define utarray_front(a) (((a)->i) ? (_utarray_eltptr(a,0)) : NULL) +#define utarray_next(a,e) (((e)==NULL) ? utarray_front(a) : ((((a)->i) > (utarray_eltidx(a,e)+1)) ? _utarray_eltptr(a,utarray_eltidx(a,e)+1) : NULL)) +#define utarray_prev(a,e) (((e)==NULL) ? utarray_back(a) : ((utarray_eltidx(a,e) > 0) ? _utarray_eltptr(a,utarray_eltidx(a,e)-1) : NULL)) +#define utarray_back(a) (((a)->i) ? (_utarray_eltptr(a,(a)->i-1)) : NULL) +#define utarray_eltidx(a,e) (((char*)(e) >= (a)->d) ? (((char*)(e) - (a)->d)/(a)->icd.sz) : -1) + +/* last we pre-define a few icd for common utarrays of ints and strings */ +static void utarray_str_cpy(void *dst, const void *src) { + char **_src = (char**)src, **_dst = (char**)dst; + *_dst = (*_src == NULL) ? NULL : strdup(*_src); +} +static void utarray_str_dtor(void *elt) { + char **eltc = (char**)elt; + if (*eltc != NULL) free(*eltc); +} +static const UT_icd ut_str_icd _UNUSED_ = {sizeof(char*),NULL,utarray_str_cpy,utarray_str_dtor}; +static const UT_icd ut_int_icd _UNUSED_ = {sizeof(int),NULL,NULL,NULL}; +static const UT_icd ut_ptr_icd _UNUSED_ = {sizeof(void*),NULL,NULL,NULL}; + + +#endif /* UTARRAY_H */ diff --git a/src/common/thirdparty/uthash.h b/src/common/thirdparty/uthash.h new file mode 100644 index 000000000..45d1f9fc1 --- /dev/null +++ b/src/common/thirdparty/uthash.h @@ -0,0 +1,1074 @@ +/* +Copyright (c) 2003-2016, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef UTHASH_H +#define UTHASH_H + +#define UTHASH_VERSION 2.0.1 + +#include /* memcmp,strlen */ +#include /* ptrdiff_t */ +#include /* exit() */ + +/* These macros use decltype or the earlier __typeof GNU extension. + As decltype is only available in newer compilers (VS2010 or gcc 4.3+ + when compiling c++ source) this code uses whatever method is needed + or, for VS2008 where neither is available, uses casting workarounds. */ +#if defined(_MSC_VER) /* MS compiler */ +#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ +#define DECLTYPE(x) (decltype(x)) +#else /* VS2008 or older (or VS2010 in C mode) */ +#define NO_DECLTYPE +#define DECLTYPE(x) +#endif +#elif defined(__BORLANDC__) || defined(__LCC__) || defined(__WATCOMC__) +#define NO_DECLTYPE +#define DECLTYPE(x) +#else /* GNU, Sun and other compilers */ +#define DECLTYPE(x) (__typeof(x)) +#endif + +#ifdef NO_DECLTYPE +#define DECLTYPE_ASSIGN(dst,src) \ +do { \ + char **_da_dst = (char**)(&(dst)); \ + *_da_dst = (char*)(src); \ +} while (0) +#else +#define DECLTYPE_ASSIGN(dst,src) \ +do { \ + (dst) = DECLTYPE(dst)(src); \ +} while (0) +#endif + +/* a number of the hash function use uint32_t which isn't defined on Pre VS2010 */ +#if defined(_WIN32) +#if defined(_MSC_VER) && _MSC_VER >= 1600 +#include +#elif defined(__WATCOMC__) || defined(__MINGW32__) || defined(__CYGWIN__) +#include +#else +typedef unsigned int uint32_t; +typedef unsigned char uint8_t; +#endif +#elif defined(__GNUC__) && !defined(__VXWORKS__) +#include +#else +typedef unsigned int uint32_t; +typedef unsigned char uint8_t; +#endif + +#ifndef uthash_fatal +#define uthash_fatal(msg) exit(-1) /* fatal error (out of memory,etc) */ +#endif +#ifndef uthash_malloc +#define uthash_malloc(sz) malloc(sz) /* malloc fcn */ +#endif +#ifndef uthash_free +#define uthash_free(ptr,sz) free(ptr) /* free fcn */ +#endif +#ifndef uthash_strlen +#define uthash_strlen(s) strlen(s) +#endif +#ifndef uthash_memcmp +#define uthash_memcmp(a,b,n) memcmp(a,b,n) +#endif + +#ifndef uthash_noexpand_fyi +#define uthash_noexpand_fyi(tbl) /* can be defined to log noexpand */ +#endif +#ifndef uthash_expand_fyi +#define uthash_expand_fyi(tbl) /* can be defined to log expands */ +#endif + +/* initial number of buckets */ +#define HASH_INITIAL_NUM_BUCKETS 32U /* initial number of buckets */ +#define HASH_INITIAL_NUM_BUCKETS_LOG2 5U /* lg2 of initial number of buckets */ +#define HASH_BKT_CAPACITY_THRESH 10U /* expand when bucket count reaches */ + +/* calculate the element whose hash handle address is hhp */ +#define ELMT_FROM_HH(tbl,hhp) ((void*)(((char*)(hhp)) - ((tbl)->hho))) +/* calculate the hash handle from element address elp */ +#define HH_FROM_ELMT(tbl,elp) ((UT_hash_handle *)(((char*)(elp)) + ((tbl)->hho))) + +#define HASH_VALUE(keyptr,keylen,hashv) \ +do { \ + HASH_FCN(keyptr, keylen, hashv); \ +} while (0) + +#define HASH_FIND_BYHASHVALUE(hh,head,keyptr,keylen,hashval,out) \ +do { \ + (out) = NULL; \ + if (head) { \ + unsigned _hf_bkt; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _hf_bkt); \ + if (HASH_BLOOM_TEST((head)->hh.tbl, hashval) != 0) { \ + HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ], keyptr, keylen, hashval, out); \ + } \ + } \ +} while (0) + +#define HASH_FIND(hh,head,keyptr,keylen,out) \ +do { \ + unsigned _hf_hashv; \ + HASH_VALUE(keyptr, keylen, _hf_hashv); \ + HASH_FIND_BYHASHVALUE(hh, head, keyptr, keylen, _hf_hashv, out); \ +} while (0) + +#ifdef HASH_BLOOM +#define HASH_BLOOM_BITLEN (1UL << HASH_BLOOM) +#define HASH_BLOOM_BYTELEN (HASH_BLOOM_BITLEN/8UL) + (((HASH_BLOOM_BITLEN%8UL)!=0UL) ? 1UL : 0UL) +#define HASH_BLOOM_MAKE(tbl) \ +do { \ + (tbl)->bloom_nbits = HASH_BLOOM; \ + (tbl)->bloom_bv = (uint8_t*)uthash_malloc(HASH_BLOOM_BYTELEN); \ + if (!((tbl)->bloom_bv)) { uthash_fatal( "out of memory"); } \ + memset((tbl)->bloom_bv, 0, HASH_BLOOM_BYTELEN); \ + (tbl)->bloom_sig = HASH_BLOOM_SIGNATURE; \ +} while (0) + +#define HASH_BLOOM_FREE(tbl) \ +do { \ + uthash_free((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \ +} while (0) + +#define HASH_BLOOM_BITSET(bv,idx) (bv[(idx)/8U] |= (1U << ((idx)%8U))) +#define HASH_BLOOM_BITTEST(bv,idx) (bv[(idx)/8U] & (1U << ((idx)%8U))) + +#define HASH_BLOOM_ADD(tbl,hashv) \ + HASH_BLOOM_BITSET((tbl)->bloom_bv, (hashv & (uint32_t)((1ULL << (tbl)->bloom_nbits) - 1U))) + +#define HASH_BLOOM_TEST(tbl,hashv) \ + HASH_BLOOM_BITTEST((tbl)->bloom_bv, (hashv & (uint32_t)((1ULL << (tbl)->bloom_nbits) - 1U))) + +#else +#define HASH_BLOOM_MAKE(tbl) +#define HASH_BLOOM_FREE(tbl) +#define HASH_BLOOM_ADD(tbl,hashv) +#define HASH_BLOOM_TEST(tbl,hashv) (1) +#define HASH_BLOOM_BYTELEN 0U +#endif + +#define HASH_MAKE_TABLE(hh,head) \ +do { \ + (head)->hh.tbl = (UT_hash_table*)uthash_malloc( \ + sizeof(UT_hash_table)); \ + if (!((head)->hh.tbl)) { uthash_fatal( "out of memory"); } \ + memset((head)->hh.tbl, 0, sizeof(UT_hash_table)); \ + (head)->hh.tbl->tail = &((head)->hh); \ + (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS; \ + (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2; \ + (head)->hh.tbl->hho = (char*)(&(head)->hh) - (char*)(head); \ + (head)->hh.tbl->buckets = (UT_hash_bucket*)uthash_malloc( \ + HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \ + if (! (head)->hh.tbl->buckets) { uthash_fatal( "out of memory"); } \ + memset((head)->hh.tbl->buckets, 0, \ + HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \ + HASH_BLOOM_MAKE((head)->hh.tbl); \ + (head)->hh.tbl->signature = HASH_SIGNATURE; \ +} while (0) + +#define HASH_REPLACE_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,replaced,cmpfcn) \ +do { \ + (replaced) = NULL; \ + HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \ + if (replaced) { \ + HASH_DELETE(hh, head, replaced); \ + } \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn); \ +} while (0) + +#define HASH_REPLACE_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add,replaced) \ +do { \ + (replaced) = NULL; \ + HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \ + if (replaced) { \ + HASH_DELETE(hh, head, replaced); \ + } \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add); \ +} while (0) + +#define HASH_REPLACE(hh,head,fieldname,keylen_in,add,replaced) \ +do { \ + unsigned _hr_hashv; \ + HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ + HASH_REPLACE_BYHASHVALUE(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced); \ +} while (0) + +#define HASH_REPLACE_INORDER(hh,head,fieldname,keylen_in,add,replaced,cmpfcn) \ +do { \ + unsigned _hr_hashv; \ + HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ + HASH_REPLACE_BYHASHVALUE_INORDER(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced, cmpfcn); \ +} while (0) + +#define HASH_APPEND_LIST(hh, head, add) \ +do { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail); \ + (head)->hh.tbl->tail->next = (add); \ + (head)->hh.tbl->tail = &((add)->hh); \ +} while (0) + +#define HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh,head,keyptr,keylen_in,hashval,add,cmpfcn) \ +do { \ + unsigned _ha_bkt; \ + (add)->hh.hashv = (hashval); \ + (add)->hh.key = (char*) (keyptr); \ + (add)->hh.keylen = (unsigned) (keylen_in); \ + if (!(head)) { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = NULL; \ + (head) = (add); \ + HASH_MAKE_TABLE(hh, head); \ + } else { \ + struct UT_hash_handle *_hs_iter = &(head)->hh; \ + (add)->hh.tbl = (head)->hh.tbl; \ + do { \ + if (cmpfcn(DECLTYPE(head) ELMT_FROM_HH((head)->hh.tbl, _hs_iter), add) > 0) \ + break; \ + } while ((_hs_iter = _hs_iter->next)); \ + if (_hs_iter) { \ + (add)->hh.next = _hs_iter; \ + if (((add)->hh.prev = _hs_iter->prev)) { \ + HH_FROM_ELMT((head)->hh.tbl, _hs_iter->prev)->next = (add); \ + } else { \ + (head) = (add); \ + } \ + _hs_iter->prev = (add); \ + } else { \ + HASH_APPEND_LIST(hh, head, add); \ + } \ + } \ + (head)->hh.tbl->num_items++; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ + HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], &(add)->hh); \ + HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ + HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ + HASH_FSCK(hh, head); \ +} while (0) + +#define HASH_ADD_KEYPTR_INORDER(hh,head,keyptr,keylen_in,add,cmpfcn) \ +do { \ + unsigned _hs_hashv; \ + HASH_VALUE(keyptr, keylen_in, _hs_hashv); \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, keyptr, keylen_in, _hs_hashv, add, cmpfcn); \ +} while (0) + +#define HASH_ADD_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,cmpfcn) \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn) + +#define HASH_ADD_INORDER(hh,head,fieldname,keylen_in,add,cmpfcn) \ + HASH_ADD_KEYPTR_INORDER(hh, head, &((add)->fieldname), keylen_in, add, cmpfcn) + +#define HASH_ADD_KEYPTR_BYHASHVALUE(hh,head,keyptr,keylen_in,hashval,add) \ +do { \ + unsigned _ha_bkt; \ + (add)->hh.hashv = (hashval); \ + (add)->hh.key = (char*) (keyptr); \ + (add)->hh.keylen = (unsigned) (keylen_in); \ + if (!(head)) { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = NULL; \ + (head) = (add); \ + HASH_MAKE_TABLE(hh, head); \ + } else { \ + (add)->hh.tbl = (head)->hh.tbl; \ + HASH_APPEND_LIST(hh, head, add); \ + } \ + (head)->hh.tbl->num_items++; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ + HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], &(add)->hh); \ + HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ + HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ + HASH_FSCK(hh, head); \ +} while (0) + +#define HASH_ADD_KEYPTR(hh,head,keyptr,keylen_in,add) \ +do { \ + unsigned _ha_hashv; \ + HASH_VALUE(keyptr, keylen_in, _ha_hashv); \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, keyptr, keylen_in, _ha_hashv, add); \ +} while (0) + +#define HASH_ADD_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add) \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add) + +#define HASH_ADD(hh,head,fieldname,keylen_in,add) \ + HASH_ADD_KEYPTR(hh, head, &((add)->fieldname), keylen_in, add) + +#define HASH_TO_BKT(hashv,num_bkts,bkt) \ +do { \ + bkt = ((hashv) & ((num_bkts) - 1U)); \ +} while (0) + +/* delete "delptr" from the hash table. + * "the usual" patch-up process for the app-order doubly-linked-list. + * The use of _hd_hh_del below deserves special explanation. + * These used to be expressed using (delptr) but that led to a bug + * if someone used the same symbol for the head and deletee, like + * HASH_DELETE(hh,users,users); + * We want that to work, but by changing the head (users) below + * we were forfeiting our ability to further refer to the deletee (users) + * in the patch-up process. Solution: use scratch space to + * copy the deletee pointer, then the latter references are via that + * scratch pointer rather than through the repointed (users) symbol. + */ +#define HASH_DELETE(hh,head,delptr) \ +do { \ + struct UT_hash_handle *_hd_hh_del; \ + if ( ((delptr)->hh.prev == NULL) && ((delptr)->hh.next == NULL) ) { \ + uthash_free((head)->hh.tbl->buckets, \ + (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket) ); \ + HASH_BLOOM_FREE((head)->hh.tbl); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + head = NULL; \ + } else { \ + unsigned _hd_bkt; \ + _hd_hh_del = &((delptr)->hh); \ + if ((delptr) == ELMT_FROM_HH((head)->hh.tbl,(head)->hh.tbl->tail)) { \ + (head)->hh.tbl->tail = \ + (UT_hash_handle*)((ptrdiff_t)((delptr)->hh.prev) + \ + (head)->hh.tbl->hho); \ + } \ + if ((delptr)->hh.prev != NULL) { \ + ((UT_hash_handle*)((ptrdiff_t)((delptr)->hh.prev) + \ + (head)->hh.tbl->hho))->next = (delptr)->hh.next; \ + } else { \ + DECLTYPE_ASSIGN(head,(delptr)->hh.next); \ + } \ + if (_hd_hh_del->next != NULL) { \ + ((UT_hash_handle*)((ptrdiff_t)_hd_hh_del->next + \ + (head)->hh.tbl->hho))->prev = \ + _hd_hh_del->prev; \ + } \ + HASH_TO_BKT( _hd_hh_del->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \ + HASH_DEL_IN_BKT(hh,(head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del); \ + (head)->hh.tbl->num_items--; \ + } \ + HASH_FSCK(hh,head); \ +} while (0) + + +/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */ +#define HASH_FIND_STR(head,findstr,out) \ + HASH_FIND(hh,head,findstr,(unsigned)uthash_strlen(findstr),out) +#define HASH_ADD_STR(head,strfield,add) \ + HASH_ADD(hh,head,strfield[0],(unsigned)uthash_strlen(add->strfield),add) +#define HASH_REPLACE_STR(head,strfield,add,replaced) \ + HASH_REPLACE(hh,head,strfield[0],(unsigned)uthash_strlen(add->strfield),add,replaced) +#define HASH_FIND_INT(head,findint,out) \ + HASH_FIND(hh,head,findint,sizeof(int),out) +#define HASH_ADD_INT(head,intfield,add) \ + HASH_ADD(hh,head,intfield,sizeof(int),add) +#define HASH_REPLACE_INT(head,intfield,add,replaced) \ + HASH_REPLACE(hh,head,intfield,sizeof(int),add,replaced) +#define HASH_FIND_PTR(head,findptr,out) \ + HASH_FIND(hh,head,findptr,sizeof(void *),out) +#define HASH_ADD_PTR(head,ptrfield,add) \ + HASH_ADD(hh,head,ptrfield,sizeof(void *),add) +#define HASH_REPLACE_PTR(head,ptrfield,add,replaced) \ + HASH_REPLACE(hh,head,ptrfield,sizeof(void *),add,replaced) +#define HASH_DEL(head,delptr) \ + HASH_DELETE(hh,head,delptr) + +/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined. + * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined. + */ +#ifdef HASH_DEBUG +#define HASH_OOPS(...) do { fprintf(stderr,__VA_ARGS__); exit(-1); } while (0) +#define HASH_FSCK(hh,head) \ +do { \ + struct UT_hash_handle *_thh; \ + if (head) { \ + unsigned _bkt_i; \ + unsigned _count; \ + char *_prev; \ + _count = 0; \ + for( _bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; _bkt_i++) { \ + unsigned _bkt_count = 0; \ + _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head; \ + _prev = NULL; \ + while (_thh) { \ + if (_prev != (char*)(_thh->hh_prev)) { \ + HASH_OOPS("invalid hh_prev %p, actual %p\n", \ + _thh->hh_prev, _prev ); \ + } \ + _bkt_count++; \ + _prev = (char*)(_thh); \ + _thh = _thh->hh_next; \ + } \ + _count += _bkt_count; \ + if ((head)->hh.tbl->buckets[_bkt_i].count != _bkt_count) { \ + HASH_OOPS("invalid bucket count %u, actual %u\n", \ + (head)->hh.tbl->buckets[_bkt_i].count, _bkt_count); \ + } \ + } \ + if (_count != (head)->hh.tbl->num_items) { \ + HASH_OOPS("invalid hh item count %u, actual %u\n", \ + (head)->hh.tbl->num_items, _count ); \ + } \ + /* traverse hh in app order; check next/prev integrity, count */ \ + _count = 0; \ + _prev = NULL; \ + _thh = &(head)->hh; \ + while (_thh) { \ + _count++; \ + if (_prev !=(char*)(_thh->prev)) { \ + HASH_OOPS("invalid prev %p, actual %p\n", \ + _thh->prev, _prev ); \ + } \ + _prev = (char*)ELMT_FROM_HH((head)->hh.tbl, _thh); \ + _thh = ( _thh->next ? (UT_hash_handle*)((char*)(_thh->next) + \ + (head)->hh.tbl->hho) : NULL ); \ + } \ + if (_count != (head)->hh.tbl->num_items) { \ + HASH_OOPS("invalid app item count %u, actual %u\n", \ + (head)->hh.tbl->num_items, _count ); \ + } \ + } \ +} while (0) +#else +#define HASH_FSCK(hh,head) +#endif + +/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to + * the descriptor to which this macro is defined for tuning the hash function. + * The app can #include to get the prototype for write(2). */ +#ifdef HASH_EMIT_KEYS +#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) \ +do { \ + unsigned _klen = fieldlen; \ + write(HASH_EMIT_KEYS, &_klen, sizeof(_klen)); \ + write(HASH_EMIT_KEYS, keyptr, (unsigned long)fieldlen); \ +} while (0) +#else +#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) +#endif + +/* default to Jenkin's hash unless overridden e.g. DHASH_FUNCTION=HASH_SAX */ +#ifdef HASH_FUNCTION +#define HASH_FCN HASH_FUNCTION +#else +#define HASH_FCN HASH_JEN +#endif + +/* The Bernstein hash function, used in Perl prior to v5.6. Note (x<<5+x)=x*33. */ +#define HASH_BER(key,keylen,hashv) \ +do { \ + unsigned _hb_keylen=(unsigned)keylen; \ + const unsigned char *_hb_key=(const unsigned char*)(key); \ + (hashv) = 0; \ + while (_hb_keylen-- != 0U) { \ + (hashv) = (((hashv) << 5) + (hashv)) + *_hb_key++; \ + } \ +} while (0) + + +/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at + * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */ +#define HASH_SAX(key,keylen,hashv) \ +do { \ + unsigned _sx_i; \ + const unsigned char *_hs_key=(const unsigned char*)(key); \ + hashv = 0; \ + for(_sx_i=0; _sx_i < keylen; _sx_i++) { \ + hashv ^= (hashv << 5) + (hashv >> 2) + _hs_key[_sx_i]; \ + } \ +} while (0) +/* FNV-1a variation */ +#define HASH_FNV(key,keylen,hashv) \ +do { \ + unsigned _fn_i; \ + const unsigned char *_hf_key=(const unsigned char*)(key); \ + hashv = 2166136261U; \ + for(_fn_i=0; _fn_i < keylen; _fn_i++) { \ + hashv = hashv ^ _hf_key[_fn_i]; \ + hashv = hashv * 16777619U; \ + } \ +} while (0) + +#define HASH_OAT(key,keylen,hashv) \ +do { \ + unsigned _ho_i; \ + const unsigned char *_ho_key=(const unsigned char*)(key); \ + hashv = 0; \ + for(_ho_i=0; _ho_i < keylen; _ho_i++) { \ + hashv += _ho_key[_ho_i]; \ + hashv += (hashv << 10); \ + hashv ^= (hashv >> 6); \ + } \ + hashv += (hashv << 3); \ + hashv ^= (hashv >> 11); \ + hashv += (hashv << 15); \ +} while (0) + +#define HASH_JEN_MIX(a,b,c) \ +do { \ + a -= b; a -= c; a ^= ( c >> 13 ); \ + b -= c; b -= a; b ^= ( a << 8 ); \ + c -= a; c -= b; c ^= ( b >> 13 ); \ + a -= b; a -= c; a ^= ( c >> 12 ); \ + b -= c; b -= a; b ^= ( a << 16 ); \ + c -= a; c -= b; c ^= ( b >> 5 ); \ + a -= b; a -= c; a ^= ( c >> 3 ); \ + b -= c; b -= a; b ^= ( a << 10 ); \ + c -= a; c -= b; c ^= ( b >> 15 ); \ +} while (0) + +#define HASH_JEN(key,keylen,hashv) \ +do { \ + unsigned _hj_i,_hj_j,_hj_k; \ + unsigned const char *_hj_key=(unsigned const char*)(key); \ + hashv = 0xfeedbeefu; \ + _hj_i = _hj_j = 0x9e3779b9u; \ + _hj_k = (unsigned)(keylen); \ + while (_hj_k >= 12U) { \ + _hj_i += (_hj_key[0] + ( (unsigned)_hj_key[1] << 8 ) \ + + ( (unsigned)_hj_key[2] << 16 ) \ + + ( (unsigned)_hj_key[3] << 24 ) ); \ + _hj_j += (_hj_key[4] + ( (unsigned)_hj_key[5] << 8 ) \ + + ( (unsigned)_hj_key[6] << 16 ) \ + + ( (unsigned)_hj_key[7] << 24 ) ); \ + hashv += (_hj_key[8] + ( (unsigned)_hj_key[9] << 8 ) \ + + ( (unsigned)_hj_key[10] << 16 ) \ + + ( (unsigned)_hj_key[11] << 24 ) ); \ + \ + HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ + \ + _hj_key += 12; \ + _hj_k -= 12U; \ + } \ + hashv += (unsigned)(keylen); \ + switch ( _hj_k ) { \ + case 11: hashv += ( (unsigned)_hj_key[10] << 24 ); /* FALLTHROUGH */ \ + case 10: hashv += ( (unsigned)_hj_key[9] << 16 ); /* FALLTHROUGH */ \ + case 9: hashv += ( (unsigned)_hj_key[8] << 8 ); /* FALLTHROUGH */ \ + case 8: _hj_j += ( (unsigned)_hj_key[7] << 24 ); /* FALLTHROUGH */ \ + case 7: _hj_j += ( (unsigned)_hj_key[6] << 16 ); /* FALLTHROUGH */ \ + case 6: _hj_j += ( (unsigned)_hj_key[5] << 8 ); /* FALLTHROUGH */ \ + case 5: _hj_j += _hj_key[4]; /* FALLTHROUGH */ \ + case 4: _hj_i += ( (unsigned)_hj_key[3] << 24 ); /* FALLTHROUGH */ \ + case 3: _hj_i += ( (unsigned)_hj_key[2] << 16 ); /* FALLTHROUGH */ \ + case 2: _hj_i += ( (unsigned)_hj_key[1] << 8 ); /* FALLTHROUGH */ \ + case 1: _hj_i += _hj_key[0]; \ + } \ + HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ +} while (0) + +/* The Paul Hsieh hash function */ +#undef get16bits +#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__) \ + || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__) +#define get16bits(d) (*((const uint16_t *) (d))) +#endif + +#if !defined (get16bits) +#define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8) \ + +(uint32_t)(((const uint8_t *)(d))[0]) ) +#endif +#define HASH_SFH(key,keylen,hashv) \ +do { \ + unsigned const char *_sfh_key=(unsigned const char*)(key); \ + uint32_t _sfh_tmp, _sfh_len = (uint32_t)keylen; \ + \ + unsigned _sfh_rem = _sfh_len & 3U; \ + _sfh_len >>= 2; \ + hashv = 0xcafebabeu; \ + \ + /* Main loop */ \ + for (;_sfh_len > 0U; _sfh_len--) { \ + hashv += get16bits (_sfh_key); \ + _sfh_tmp = ((uint32_t)(get16bits (_sfh_key+2)) << 11) ^ hashv; \ + hashv = (hashv << 16) ^ _sfh_tmp; \ + _sfh_key += 2U*sizeof (uint16_t); \ + hashv += hashv >> 11; \ + } \ + \ + /* Handle end cases */ \ + switch (_sfh_rem) { \ + case 3: hashv += get16bits (_sfh_key); \ + hashv ^= hashv << 16; \ + hashv ^= (uint32_t)(_sfh_key[sizeof (uint16_t)]) << 18; \ + hashv += hashv >> 11; \ + break; \ + case 2: hashv += get16bits (_sfh_key); \ + hashv ^= hashv << 11; \ + hashv += hashv >> 17; \ + break; \ + case 1: hashv += *_sfh_key; \ + hashv ^= hashv << 10; \ + hashv += hashv >> 1; \ + } \ + \ + /* Force "avalanching" of final 127 bits */ \ + hashv ^= hashv << 3; \ + hashv += hashv >> 5; \ + hashv ^= hashv << 4; \ + hashv += hashv >> 17; \ + hashv ^= hashv << 25; \ + hashv += hashv >> 6; \ +} while (0) + +#ifdef HASH_USING_NO_STRICT_ALIASING +/* The MurmurHash exploits some CPU's (x86,x86_64) tolerance for unaligned reads. + * For other types of CPU's (e.g. Sparc) an unaligned read causes a bus error. + * MurmurHash uses the faster approach only on CPU's where we know it's safe. + * + * Note the preprocessor built-in defines can be emitted using: + * + * gcc -m64 -dM -E - < /dev/null (on gcc) + * cc -## a.c (where a.c is a simple test file) (Sun Studio) + */ +#if (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86)) +#define MUR_GETBLOCK(p,i) p[i] +#else /* non intel */ +#define MUR_PLUS0_ALIGNED(p) (((unsigned long)p & 3UL) == 0UL) +#define MUR_PLUS1_ALIGNED(p) (((unsigned long)p & 3UL) == 1UL) +#define MUR_PLUS2_ALIGNED(p) (((unsigned long)p & 3UL) == 2UL) +#define MUR_PLUS3_ALIGNED(p) (((unsigned long)p & 3UL) == 3UL) +#define WP(p) ((uint32_t*)((unsigned long)(p) & ~3UL)) +#if (defined(__BIG_ENDIAN__) || defined(SPARC) || defined(__ppc__) || defined(__ppc64__)) +#define MUR_THREE_ONE(p) ((((*WP(p))&0x00ffffff) << 8) | (((*(WP(p)+1))&0xff000000) >> 24)) +#define MUR_TWO_TWO(p) ((((*WP(p))&0x0000ffff) <<16) | (((*(WP(p)+1))&0xffff0000) >> 16)) +#define MUR_ONE_THREE(p) ((((*WP(p))&0x000000ff) <<24) | (((*(WP(p)+1))&0xffffff00) >> 8)) +#else /* assume little endian non-intel */ +#define MUR_THREE_ONE(p) ((((*WP(p))&0xffffff00) >> 8) | (((*(WP(p)+1))&0x000000ff) << 24)) +#define MUR_TWO_TWO(p) ((((*WP(p))&0xffff0000) >>16) | (((*(WP(p)+1))&0x0000ffff) << 16)) +#define MUR_ONE_THREE(p) ((((*WP(p))&0xff000000) >>24) | (((*(WP(p)+1))&0x00ffffff) << 8)) +#endif +#define MUR_GETBLOCK(p,i) (MUR_PLUS0_ALIGNED(p) ? ((p)[i]) : \ + (MUR_PLUS1_ALIGNED(p) ? MUR_THREE_ONE(p) : \ + (MUR_PLUS2_ALIGNED(p) ? MUR_TWO_TWO(p) : \ + MUR_ONE_THREE(p)))) +#endif +#define MUR_ROTL32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +#define MUR_FMIX(_h) \ +do { \ + _h ^= _h >> 16; \ + _h *= 0x85ebca6bu; \ + _h ^= _h >> 13; \ + _h *= 0xc2b2ae35u; \ + _h ^= _h >> 16; \ +} while (0) + +#define HASH_MUR(key,keylen,hashv) \ +do { \ + const uint8_t *_mur_data = (const uint8_t*)(key); \ + const int _mur_nblocks = (int)(keylen) / 4; \ + uint32_t _mur_h1 = 0xf88D5353u; \ + uint32_t _mur_c1 = 0xcc9e2d51u; \ + uint32_t _mur_c2 = 0x1b873593u; \ + uint32_t _mur_k1 = 0; \ + const uint8_t *_mur_tail; \ + const uint32_t *_mur_blocks = (const uint32_t*)(_mur_data+(_mur_nblocks*4)); \ + int _mur_i; \ + for(_mur_i = -_mur_nblocks; _mur_i!=0; _mur_i++) { \ + _mur_k1 = MUR_GETBLOCK(_mur_blocks,_mur_i); \ + _mur_k1 *= _mur_c1; \ + _mur_k1 = MUR_ROTL32(_mur_k1,15); \ + _mur_k1 *= _mur_c2; \ + \ + _mur_h1 ^= _mur_k1; \ + _mur_h1 = MUR_ROTL32(_mur_h1,13); \ + _mur_h1 = (_mur_h1*5U) + 0xe6546b64u; \ + } \ + _mur_tail = (const uint8_t*)(_mur_data + (_mur_nblocks*4)); \ + _mur_k1=0; \ + switch((keylen) & 3U) { \ + case 3: _mur_k1 ^= (uint32_t)_mur_tail[2] << 16; /* FALLTHROUGH */ \ + case 2: _mur_k1 ^= (uint32_t)_mur_tail[1] << 8; /* FALLTHROUGH */ \ + case 1: _mur_k1 ^= (uint32_t)_mur_tail[0]; \ + _mur_k1 *= _mur_c1; \ + _mur_k1 = MUR_ROTL32(_mur_k1,15); \ + _mur_k1 *= _mur_c2; \ + _mur_h1 ^= _mur_k1; \ + } \ + _mur_h1 ^= (uint32_t)(keylen); \ + MUR_FMIX(_mur_h1); \ + hashv = _mur_h1; \ +} while (0) +#endif /* HASH_USING_NO_STRICT_ALIASING */ + +/* iterate over items in a known bucket to find desired item */ +#define HASH_FIND_IN_BKT(tbl,hh,head,keyptr,keylen_in,hashval,out) \ +do { \ + if ((head).hh_head != NULL) { \ + DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (head).hh_head)); \ + } else { \ + (out) = NULL; \ + } \ + while ((out) != NULL) { \ + if ((out)->hh.hashv == (hashval) && (out)->hh.keylen == (keylen_in)) { \ + if (uthash_memcmp((out)->hh.key, keyptr, keylen_in) == 0) { \ + break; \ + } \ + } \ + if ((out)->hh.hh_next != NULL) { \ + DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (out)->hh.hh_next)); \ + } else { \ + (out) = NULL; \ + } \ + } \ +} while (0) + +/* add an item to a bucket */ +#define HASH_ADD_TO_BKT(head,addhh) \ +do { \ + head.count++; \ + (addhh)->hh_next = head.hh_head; \ + (addhh)->hh_prev = NULL; \ + if (head.hh_head != NULL) { (head).hh_head->hh_prev = (addhh); } \ + (head).hh_head=addhh; \ + if ((head.count >= ((head.expand_mult+1U) * HASH_BKT_CAPACITY_THRESH)) \ + && ((addhh)->tbl->noexpand != 1U)) { \ + HASH_EXPAND_BUCKETS((addhh)->tbl); \ + } \ +} while (0) + +/* remove an item from a given bucket */ +#define HASH_DEL_IN_BKT(hh,head,hh_del) \ + (head).count--; \ + if ((head).hh_head == hh_del) { \ + (head).hh_head = hh_del->hh_next; \ + } \ + if (hh_del->hh_prev) { \ + hh_del->hh_prev->hh_next = hh_del->hh_next; \ + } \ + if (hh_del->hh_next) { \ + hh_del->hh_next->hh_prev = hh_del->hh_prev; \ + } + +/* Bucket expansion has the effect of doubling the number of buckets + * and redistributing the items into the new buckets. Ideally the + * items will distribute more or less evenly into the new buckets + * (the extent to which this is true is a measure of the quality of + * the hash function as it applies to the key domain). + * + * With the items distributed into more buckets, the chain length + * (item count) in each bucket is reduced. Thus by expanding buckets + * the hash keeps a bound on the chain length. This bounded chain + * length is the essence of how a hash provides constant time lookup. + * + * The calculation of tbl->ideal_chain_maxlen below deserves some + * explanation. First, keep in mind that we're calculating the ideal + * maximum chain length based on the *new* (doubled) bucket count. + * In fractions this is just n/b (n=number of items,b=new num buckets). + * Since the ideal chain length is an integer, we want to calculate + * ceil(n/b). We don't depend on floating point arithmetic in this + * hash, so to calculate ceil(n/b) with integers we could write + * + * ceil(n/b) = (n/b) + ((n%b)?1:0) + * + * and in fact a previous version of this hash did just that. + * But now we have improved things a bit by recognizing that b is + * always a power of two. We keep its base 2 log handy (call it lb), + * so now we can write this with a bit shift and logical AND: + * + * ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0) + * + */ +#define HASH_EXPAND_BUCKETS(tbl) \ +do { \ + unsigned _he_bkt; \ + unsigned _he_bkt_i; \ + struct UT_hash_handle *_he_thh, *_he_hh_nxt; \ + UT_hash_bucket *_he_new_buckets, *_he_newbkt; \ + _he_new_buckets = (UT_hash_bucket*)uthash_malloc( \ + 2UL * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \ + if (!_he_new_buckets) { uthash_fatal( "out of memory"); } \ + memset(_he_new_buckets, 0, \ + 2UL * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \ + tbl->ideal_chain_maxlen = \ + (tbl->num_items >> (tbl->log2_num_buckets+1U)) + \ + (((tbl->num_items & ((tbl->num_buckets*2U)-1U)) != 0U) ? 1U : 0U); \ + tbl->nonideal_items = 0; \ + for(_he_bkt_i = 0; _he_bkt_i < tbl->num_buckets; _he_bkt_i++) \ + { \ + _he_thh = tbl->buckets[ _he_bkt_i ].hh_head; \ + while (_he_thh != NULL) { \ + _he_hh_nxt = _he_thh->hh_next; \ + HASH_TO_BKT( _he_thh->hashv, tbl->num_buckets*2U, _he_bkt); \ + _he_newbkt = &(_he_new_buckets[ _he_bkt ]); \ + if (++(_he_newbkt->count) > tbl->ideal_chain_maxlen) { \ + tbl->nonideal_items++; \ + _he_newbkt->expand_mult = _he_newbkt->count / \ + tbl->ideal_chain_maxlen; \ + } \ + _he_thh->hh_prev = NULL; \ + _he_thh->hh_next = _he_newbkt->hh_head; \ + if (_he_newbkt->hh_head != NULL) { _he_newbkt->hh_head->hh_prev = \ + _he_thh; } \ + _he_newbkt->hh_head = _he_thh; \ + _he_thh = _he_hh_nxt; \ + } \ + } \ + uthash_free( tbl->buckets, tbl->num_buckets*sizeof(struct UT_hash_bucket) ); \ + tbl->num_buckets *= 2U; \ + tbl->log2_num_buckets++; \ + tbl->buckets = _he_new_buckets; \ + tbl->ineff_expands = (tbl->nonideal_items > (tbl->num_items >> 1)) ? \ + (tbl->ineff_expands+1U) : 0U; \ + if (tbl->ineff_expands > 1U) { \ + tbl->noexpand=1; \ + uthash_noexpand_fyi(tbl); \ + } \ + uthash_expand_fyi(tbl); \ +} while (0) + + +/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */ +/* Note that HASH_SORT assumes the hash handle name to be hh. + * HASH_SRT was added to allow the hash handle name to be passed in. */ +#define HASH_SORT(head,cmpfcn) HASH_SRT(hh,head,cmpfcn) +#define HASH_SRT(hh,head,cmpfcn) \ +do { \ + unsigned _hs_i; \ + unsigned _hs_looping,_hs_nmerges,_hs_insize,_hs_psize,_hs_qsize; \ + struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail; \ + if (head != NULL) { \ + _hs_insize = 1; \ + _hs_looping = 1; \ + _hs_list = &((head)->hh); \ + while (_hs_looping != 0U) { \ + _hs_p = _hs_list; \ + _hs_list = NULL; \ + _hs_tail = NULL; \ + _hs_nmerges = 0; \ + while (_hs_p != NULL) { \ + _hs_nmerges++; \ + _hs_q = _hs_p; \ + _hs_psize = 0; \ + for ( _hs_i = 0; _hs_i < _hs_insize; _hs_i++ ) { \ + _hs_psize++; \ + _hs_q = (UT_hash_handle*)((_hs_q->next != NULL) ? \ + ((void*)((char*)(_hs_q->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + if (! (_hs_q) ) { break; } \ + } \ + _hs_qsize = _hs_insize; \ + while ((_hs_psize > 0U) || ((_hs_qsize > 0U) && (_hs_q != NULL))) {\ + if (_hs_psize == 0U) { \ + _hs_e = _hs_q; \ + _hs_q = (UT_hash_handle*)((_hs_q->next != NULL) ? \ + ((void*)((char*)(_hs_q->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + _hs_qsize--; \ + } else if ( (_hs_qsize == 0U) || (_hs_q == NULL) ) { \ + _hs_e = _hs_p; \ + if (_hs_p != NULL){ \ + _hs_p = (UT_hash_handle*)((_hs_p->next != NULL) ? \ + ((void*)((char*)(_hs_p->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + } \ + _hs_psize--; \ + } else if (( \ + cmpfcn(DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_p)), \ + DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_q))) \ + ) <= 0) { \ + _hs_e = _hs_p; \ + if (_hs_p != NULL){ \ + _hs_p = (UT_hash_handle*)((_hs_p->next != NULL) ? \ + ((void*)((char*)(_hs_p->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + } \ + _hs_psize--; \ + } else { \ + _hs_e = _hs_q; \ + _hs_q = (UT_hash_handle*)((_hs_q->next != NULL) ? \ + ((void*)((char*)(_hs_q->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + _hs_qsize--; \ + } \ + if ( _hs_tail != NULL ) { \ + _hs_tail->next = ((_hs_e != NULL) ? \ + ELMT_FROM_HH((head)->hh.tbl,_hs_e) : NULL); \ + } else { \ + _hs_list = _hs_e; \ + } \ + if (_hs_e != NULL) { \ + _hs_e->prev = ((_hs_tail != NULL) ? \ + ELMT_FROM_HH((head)->hh.tbl,_hs_tail) : NULL); \ + } \ + _hs_tail = _hs_e; \ + } \ + _hs_p = _hs_q; \ + } \ + if (_hs_tail != NULL){ \ + _hs_tail->next = NULL; \ + } \ + if ( _hs_nmerges <= 1U ) { \ + _hs_looping=0; \ + (head)->hh.tbl->tail = _hs_tail; \ + DECLTYPE_ASSIGN(head,ELMT_FROM_HH((head)->hh.tbl, _hs_list)); \ + } \ + _hs_insize *= 2U; \ + } \ + HASH_FSCK(hh,head); \ + } \ +} while (0) + +/* This function selects items from one hash into another hash. + * The end result is that the selected items have dual presence + * in both hashes. There is no copy of the items made; rather + * they are added into the new hash through a secondary hash + * hash handle that must be present in the structure. */ +#define HASH_SELECT(hh_dst, dst, hh_src, src, cond) \ +do { \ + unsigned _src_bkt, _dst_bkt; \ + void *_last_elt=NULL, *_elt; \ + UT_hash_handle *_src_hh, *_dst_hh, *_last_elt_hh=NULL; \ + ptrdiff_t _dst_hho = ((char*)(&(dst)->hh_dst) - (char*)(dst)); \ + if (src != NULL) { \ + for(_src_bkt=0; _src_bkt < (src)->hh_src.tbl->num_buckets; _src_bkt++) { \ + for(_src_hh = (src)->hh_src.tbl->buckets[_src_bkt].hh_head; \ + _src_hh != NULL; \ + _src_hh = _src_hh->hh_next) { \ + _elt = ELMT_FROM_HH((src)->hh_src.tbl, _src_hh); \ + if (cond(_elt)) { \ + _dst_hh = (UT_hash_handle*)(((char*)_elt) + _dst_hho); \ + _dst_hh->key = _src_hh->key; \ + _dst_hh->keylen = _src_hh->keylen; \ + _dst_hh->hashv = _src_hh->hashv; \ + _dst_hh->prev = _last_elt; \ + _dst_hh->next = NULL; \ + if (_last_elt_hh != NULL) { _last_elt_hh->next = _elt; } \ + if (dst == NULL) { \ + DECLTYPE_ASSIGN(dst,_elt); \ + HASH_MAKE_TABLE(hh_dst,dst); \ + } else { \ + _dst_hh->tbl = (dst)->hh_dst.tbl; \ + } \ + HASH_TO_BKT(_dst_hh->hashv, _dst_hh->tbl->num_buckets, _dst_bkt); \ + HASH_ADD_TO_BKT(_dst_hh->tbl->buckets[_dst_bkt],_dst_hh); \ + (dst)->hh_dst.tbl->num_items++; \ + _last_elt = _elt; \ + _last_elt_hh = _dst_hh; \ + } \ + } \ + } \ + } \ + HASH_FSCK(hh_dst,dst); \ +} while (0) + +#define HASH_CLEAR(hh,head) \ +do { \ + if (head != NULL) { \ + uthash_free((head)->hh.tbl->buckets, \ + (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket)); \ + HASH_BLOOM_FREE((head)->hh.tbl); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + (head)=NULL; \ + } \ +} while (0) + +#define HASH_OVERHEAD(hh,head) \ + ((head != NULL) ? ( \ + (size_t)(((head)->hh.tbl->num_items * sizeof(UT_hash_handle)) + \ + ((head)->hh.tbl->num_buckets * sizeof(UT_hash_bucket)) + \ + sizeof(UT_hash_table) + \ + (HASH_BLOOM_BYTELEN))) : 0U) + +#ifdef NO_DECLTYPE +#define HASH_ITER(hh,head,el,tmp) \ +for(((el)=(head)), ((*(char**)(&(tmp)))=(char*)((head!=NULL)?(head)->hh.next:NULL)); \ + (el) != NULL; ((el)=(tmp)), ((*(char**)(&(tmp)))=(char*)((tmp!=NULL)?(tmp)->hh.next:NULL))) +#else +#define HASH_ITER(hh,head,el,tmp) \ +for(((el)=(head)), ((tmp)=DECLTYPE(el)((head!=NULL)?(head)->hh.next:NULL)); \ + (el) != NULL; ((el)=(tmp)), ((tmp)=DECLTYPE(el)((tmp!=NULL)?(tmp)->hh.next:NULL))) +#endif + +/* obtain a count of items in the hash */ +#define HASH_COUNT(head) HASH_CNT(hh,head) +#define HASH_CNT(hh,head) ((head != NULL)?((head)->hh.tbl->num_items):0U) + +typedef struct UT_hash_bucket { + struct UT_hash_handle *hh_head; + unsigned count; + + /* expand_mult is normally set to 0. In this situation, the max chain length + * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If + * the bucket's chain exceeds this length, bucket expansion is triggered). + * However, setting expand_mult to a non-zero value delays bucket expansion + * (that would be triggered by additions to this particular bucket) + * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH. + * (The multiplier is simply expand_mult+1). The whole idea of this + * multiplier is to reduce bucket expansions, since they are expensive, in + * situations where we know that a particular bucket tends to be overused. + * It is better to let its chain length grow to a longer yet-still-bounded + * value, than to do an O(n) bucket expansion too often. + */ + unsigned expand_mult; + +} UT_hash_bucket; + +/* random signature used only to find hash tables in external analysis */ +#define HASH_SIGNATURE 0xa0111fe1u +#define HASH_BLOOM_SIGNATURE 0xb12220f2u + +typedef struct UT_hash_table { + UT_hash_bucket *buckets; + unsigned num_buckets, log2_num_buckets; + unsigned num_items; + struct UT_hash_handle *tail; /* tail hh in app order, for fast append */ + ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */ + + /* in an ideal situation (all buckets used equally), no bucket would have + * more than ceil(#items/#buckets) items. that's the ideal chain length. */ + unsigned ideal_chain_maxlen; + + /* nonideal_items is the number of items in the hash whose chain position + * exceeds the ideal chain maxlen. these items pay the penalty for an uneven + * hash distribution; reaching them in a chain traversal takes >ideal steps */ + unsigned nonideal_items; + + /* ineffective expands occur when a bucket doubling was performed, but + * afterward, more than half the items in the hash had nonideal chain + * positions. If this happens on two consecutive expansions we inhibit any + * further expansion, as it's not helping; this happens when the hash + * function isn't a good fit for the key domain. When expansion is inhibited + * the hash will still work, albeit no longer in constant time. */ + unsigned ineff_expands, noexpand; + + uint32_t signature; /* used only to find hash tables in external analysis */ +#ifdef HASH_BLOOM + uint32_t bloom_sig; /* used only to test bloom exists in external analysis */ + uint8_t *bloom_bv; + uint8_t bloom_nbits; +#endif + +} UT_hash_table; + +typedef struct UT_hash_handle { + struct UT_hash_table *tbl; + void *prev; /* prev element in app order */ + void *next; /* next element in app order */ + struct UT_hash_handle *hh_prev; /* previous hh in bucket order */ + struct UT_hash_handle *hh_next; /* next hh in bucket order */ + void *key; /* ptr to enclosing struct's key */ + unsigned keylen; /* enclosing struct's key len */ + unsigned hashv; /* result of hash-fcn(key) */ +} UT_hash_handle; + +#endif /* UTHASH_H */ diff --git a/src/common/thirdparty/utlist.h b/src/common/thirdparty/utlist.h new file mode 100644 index 000000000..9b5534ffb --- /dev/null +++ b/src/common/thirdparty/utlist.h @@ -0,0 +1,895 @@ +/* +Copyright (c) 2007-2016, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef UTLIST_H +#define UTLIST_H + +#define UTLIST_VERSION 2.0.1 + +#include + +/* + * This file contains macros to manipulate singly and doubly-linked lists. + * + * 1. LL_ macros: singly-linked lists. + * 2. DL_ macros: doubly-linked lists. + * 3. CDL_ macros: circular doubly-linked lists. + * + * To use singly-linked lists, your structure must have a "next" pointer. + * To use doubly-linked lists, your structure must "prev" and "next" pointers. + * Either way, the pointer to the head of the list must be initialized to NULL. + * + * ----------------.EXAMPLE ------------------------- + * struct item { + * int id; + * struct item *prev, *next; + * } + * + * struct item *list = NULL: + * + * int main() { + * struct item *item; + * ... allocate and populate item ... + * DL_APPEND(list, item); + * } + * -------------------------------------------------- + * + * For doubly-linked lists, the append and delete macros are O(1) + * For singly-linked lists, append and delete are O(n) but prepend is O(1) + * The sort macro is O(n log(n)) for all types of single/double/circular lists. + */ + +/* These macros use decltype or the earlier __typeof GNU extension. + As decltype is only available in newer compilers (VS2010 or gcc 4.3+ + when compiling c++ code), this code uses whatever method is needed + or, for VS2008 where neither is available, uses casting workarounds. */ +#ifdef _MSC_VER /* MS compiler */ +#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ +#define LDECLTYPE(x) decltype(x) +#else /* VS2008 or older (or VS2010 in C mode) */ +#define NO_DECLTYPE +#endif +#elif defined(__ICCARM__) +#define NO_DECLTYPE +#else /* GNU, Sun and other compilers */ +#define LDECLTYPE(x) __typeof(x) +#endif + +/* for VS2008 we use some workarounds to get around the lack of decltype, + * namely, we always reassign our tmp variable to the list head if we need + * to dereference its prev/next pointers, and save/restore the real head.*/ +#ifdef NO_DECLTYPE +#define IF_NO_DECLTYPE(x) x +#define LDECLTYPE(x) char* +#define _SV(elt,list) _tmp = (char*)(list); {char **_alias = (char**)&(list); *_alias = (elt); } +#define _NEXT(elt,list,next) ((char*)((list)->next)) +#define _NEXTASGN(elt,list,to,next) { char **_alias = (char**)&((list)->next); *_alias=(char*)(to); } +/* #define _PREV(elt,list,prev) ((char*)((list)->prev)) */ +#define _PREVASGN(elt,list,to,prev) { char **_alias = (char**)&((list)->prev); *_alias=(char*)(to); } +#define _RS(list) { char **_alias = (char**)&(list); *_alias=_tmp; } +#define _CASTASGN(a,b) { char **_alias = (char**)&(a); *_alias=(char*)(b); } +#else +#define IF_NO_DECLTYPE(x) +#define _SV(elt,list) +#define _NEXT(elt,list,next) ((elt)->next) +#define _NEXTASGN(elt,list,to,next) ((elt)->next)=(to) +/* #define _PREV(elt,list,prev) ((elt)->prev) */ +#define _PREVASGN(elt,list,to,prev) ((elt)->prev)=(to) +#define _RS(list) +#define _CASTASGN(a,b) (a)=(b) +#endif + +/****************************************************************************** + * The sort macro is an adaptation of Simon Tatham's O(n log(n)) mergesort * + * Unwieldy variable names used here to avoid shadowing passed-in variables. * + *****************************************************************************/ +#define LL_SORT(list, cmp) \ + LL_SORT2(list, cmp, next) + +#define LL_SORT2(list, cmp, next) \ +do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + _CASTASGN(_ls_p,list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + _SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \ + if (!_ls_q) break; \ + } \ + _ls_qsize = _ls_insize; \ + while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + } else if (_ls_qsize == 0 || !_ls_q) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + } else if (cmp(_ls_p,_ls_q) <= 0) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + } else { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + } \ + if (_ls_tail) { \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \ + } else { \ + _CASTASGN(list,_ls_e); \ + } \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + if (_ls_tail) { \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \ + } \ + if (_ls_nmerges <= 1) { \ + _ls_looping=0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ +} while (0) + + +#define DL_SORT(list, cmp) \ + DL_SORT2(list, cmp, prev, next) + +#define DL_SORT2(list, cmp, prev, next) \ +do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + _CASTASGN(_ls_p,list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + _SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \ + if (!_ls_q) break; \ + } \ + _ls_qsize = _ls_insize; \ + while ((_ls_psize > 0) || ((_ls_qsize > 0) && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + } else if ((_ls_qsize == 0) || (!_ls_q)) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + } else if (cmp(_ls_p,_ls_q) <= 0) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + } else { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + } \ + if (_ls_tail) { \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \ + } else { \ + _CASTASGN(list,_ls_e); \ + } \ + _SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + _CASTASGN((list)->prev, _ls_tail); \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \ + if (_ls_nmerges <= 1) { \ + _ls_looping=0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ +} while (0) + +#define CDL_SORT(list, cmp) \ + CDL_SORT2(list, cmp, prev, next) + +#define CDL_SORT2(list, cmp, prev, next) \ +do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + LDECLTYPE(list) _ls_oldhead; \ + LDECLTYPE(list) _tmp; \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + _CASTASGN(_ls_p,list); \ + _CASTASGN(_ls_oldhead,list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + _SV(_ls_q,list); \ + if (_NEXT(_ls_q,list,next) == _ls_oldhead) { \ + _ls_q = NULL; \ + } else { \ + _ls_q = _NEXT(_ls_q,list,next); \ + } \ + _RS(list); \ + if (!_ls_q) break; \ + } \ + _ls_qsize = _ls_insize; \ + while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \ + } else if (_ls_qsize == 0 || !_ls_q) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \ + } else if (cmp(_ls_p,_ls_q) <= 0) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \ + } else { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \ + } \ + if (_ls_tail) { \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \ + } else { \ + _CASTASGN(list,_ls_e); \ + } \ + _SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + _CASTASGN((list)->prev,_ls_tail); \ + _CASTASGN(_tmp,list); \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_tmp,next); _RS(list); \ + if (_ls_nmerges <= 1) { \ + _ls_looping=0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ +} while (0) + +/****************************************************************************** + * singly linked list macros (non-circular) * + *****************************************************************************/ +#define LL_PREPEND(head,add) \ + LL_PREPEND2(head,add,next) + +#define LL_PREPEND2(head,add,next) \ +do { \ + (add)->next = (head); \ + (head) = (add); \ +} while (0) + +#define LL_CONCAT(head1,head2) \ + LL_CONCAT2(head1,head2,next) + +#define LL_CONCAT2(head1,head2,next) \ +do { \ + LDECLTYPE(head1) _tmp; \ + if (head1) { \ + _tmp = (head1); \ + while (_tmp->next) { _tmp = _tmp->next; } \ + _tmp->next=(head2); \ + } else { \ + (head1)=(head2); \ + } \ +} while (0) + +#define LL_APPEND(head,add) \ + LL_APPEND2(head,add,next) + +#define LL_APPEND2(head,add,next) \ +do { \ + LDECLTYPE(head) _tmp; \ + (add)->next=NULL; \ + if (head) { \ + _tmp = (head); \ + while (_tmp->next) { _tmp = _tmp->next; } \ + _tmp->next=(add); \ + } else { \ + (head)=(add); \ + } \ +} while (0) + +#define LL_DELETE(head,del) \ + LL_DELETE2(head,del,next) + +#define LL_DELETE2(head,del,next) \ +do { \ + LDECLTYPE(head) _tmp; \ + if ((head) == (del)) { \ + (head)=(head)->next; \ + } else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (del))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (del)->next; \ + } \ + } \ +} while (0) + +#define LL_COUNT(head,el,counter) \ + LL_COUNT2(head,el,counter,next) \ + +#define LL_COUNT2(head,el,counter,next) \ +do { \ + (counter) = 0; \ + LL_FOREACH2(head,el,next) { ++(counter); } \ +} while (0) + +#define LL_FOREACH(head,el) \ + LL_FOREACH2(head,el,next) + +#define LL_FOREACH2(head,el,next) \ + for ((el) = (head); el; (el) = (el)->next) + +#define LL_FOREACH_SAFE(head,el,tmp) \ + LL_FOREACH_SAFE2(head,el,tmp,next) + +#define LL_FOREACH_SAFE2(head,el,tmp,next) \ + for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) + +#define LL_SEARCH_SCALAR(head,out,field,val) \ + LL_SEARCH_SCALAR2(head,out,field,val,next) + +#define LL_SEARCH_SCALAR2(head,out,field,val,next) \ +do { \ + LL_FOREACH2(head,out,next) { \ + if ((out)->field == (val)) break; \ + } \ +} while (0) + +#define LL_SEARCH(head,out,elt,cmp) \ + LL_SEARCH2(head,out,elt,cmp,next) + +#define LL_SEARCH2(head,out,elt,cmp,next) \ +do { \ + LL_FOREACH2(head,out,next) { \ + if ((cmp(out,elt))==0) break; \ + } \ +} while (0) + +#define LL_REPLACE_ELEM2(head, el, add, next) \ +do { \ + LDECLTYPE(head) _tmp; \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (el))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (add); \ + } \ + } \ +} while (0) + +#define LL_REPLACE_ELEM(head, el, add) \ + LL_REPLACE_ELEM2(head, el, add, next) + +#define LL_PREPEND_ELEM2(head, el, add, next) \ +do { \ + if (el) { \ + LDECLTYPE(head) _tmp; \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (el))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (add); \ + } \ + } \ + } else { \ + LL_APPEND2(head, add, next); \ + } \ +} while (0) \ + +#define LL_PREPEND_ELEM(head, el, add) \ + LL_PREPEND_ELEM2(head, el, add, next) + +#define LL_APPEND_ELEM2(head, el, add, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (el)->next = (add); \ + } else { \ + LL_PREPEND2(head, add, next); \ + } \ +} while (0) \ + +#define LL_APPEND_ELEM(head, el, add) \ + LL_APPEND_ELEM2(head, el, add, next) + +#ifdef NO_DECLTYPE +/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ + +#undef LL_CONCAT2 +#define LL_CONCAT2(head1,head2,next) \ +do { \ + char *_tmp; \ + if (head1) { \ + _tmp = (char*)(head1); \ + while ((head1)->next) { (head1) = (head1)->next; } \ + (head1)->next = (head2); \ + _RS(head1); \ + } else { \ + (head1)=(head2); \ + } \ +} while (0) + +#undef LL_APPEND2 +#define LL_APPEND2(head,add,next) \ +do { \ + if (head) { \ + (add)->next = head; /* use add->next as a temp variable */ \ + while ((add)->next->next) { (add)->next = (add)->next->next; } \ + (add)->next->next=(add); \ + } else { \ + (head)=(add); \ + } \ + (add)->next=NULL; \ +} while (0) + +#undef LL_DELETE2 +#define LL_DELETE2(head,del,next) \ +do { \ + if ((head) == (del)) { \ + (head)=(head)->next; \ + } else { \ + char *_tmp = (char*)(head); \ + while ((head)->next && ((head)->next != (del))) { \ + (head) = (head)->next; \ + } \ + if ((head)->next) { \ + (head)->next = ((del)->next); \ + } \ + _RS(head); \ + } \ +} while (0) + +#undef LL_REPLACE_ELEM2 +#define LL_REPLACE_ELEM2(head, el, add, next) \ +do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + (add)->next = head; \ + while ((add)->next->next && ((add)->next->next != (el))) { \ + (add)->next = (add)->next->next; \ + } \ + if ((add)->next->next) { \ + (add)->next->next = (add); \ + } \ + } \ + (add)->next = (el)->next; \ +} while (0) + +#undef LL_PREPEND_ELEM2 +#define LL_PREPEND_ELEM2(head, el, add, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + (add)->next = (head); \ + while ((add)->next->next && ((add)->next->next != (el))) { \ + (add)->next = (add)->next->next; \ + } \ + if ((add)->next->next) { \ + (add)->next->next = (add); \ + } \ + } \ + (add)->next = (el); \ + } else { \ + LL_APPEND2(head, add, next); \ + } \ +} while (0) \ + +#endif /* NO_DECLTYPE */ + +/****************************************************************************** + * doubly linked list macros (non-circular) * + *****************************************************************************/ +#define DL_PREPEND(head,add) \ + DL_PREPEND2(head,add,prev,next) + +#define DL_PREPEND2(head,add,prev,next) \ +do { \ + (add)->next = (head); \ + if (head) { \ + (add)->prev = (head)->prev; \ + (head)->prev = (add); \ + } else { \ + (add)->prev = (add); \ + } \ + (head) = (add); \ +} while (0) + +#define DL_APPEND(head,add) \ + DL_APPEND2(head,add,prev,next) + +#define DL_APPEND2(head,add,prev,next) \ +do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (head)->prev->next = (add); \ + (head)->prev = (add); \ + (add)->next = NULL; \ + } else { \ + (head)=(add); \ + (head)->prev = (head); \ + (head)->next = NULL; \ + } \ +} while (0) + +#define DL_CONCAT(head1,head2) \ + DL_CONCAT2(head1,head2,prev,next) + +#define DL_CONCAT2(head1,head2,prev,next) \ +do { \ + LDECLTYPE(head1) _tmp; \ + if (head2) { \ + if (head1) { \ + _CASTASGN(_tmp, (head2)->prev); \ + (head2)->prev = (head1)->prev; \ + (head1)->prev->next = (head2); \ + _CASTASGN((head1)->prev, _tmp); \ + } else { \ + (head1)=(head2); \ + } \ + } \ +} while (0) + +#define DL_DELETE(head,del) \ + DL_DELETE2(head,del,prev,next) + +#define DL_DELETE2(head,del,prev,next) \ +do { \ + assert((del)->prev != NULL); \ + if ((del)->prev == (del)) { \ + (head)=NULL; \ + } else if ((del)==(head)) { \ + (del)->next->prev = (del)->prev; \ + (head) = (del)->next; \ + } else { \ + (del)->prev->next = (del)->next; \ + if ((del)->next) { \ + (del)->next->prev = (del)->prev; \ + } else { \ + (head)->prev = (del)->prev; \ + } \ + } \ +} while (0) + +#define DL_COUNT(head,el,counter) \ + DL_COUNT2(head,el,counter,next) \ + +#define DL_COUNT2(head,el,counter,next) \ +do { \ + (counter) = 0; \ + DL_FOREACH2(head,el,next) { ++(counter); } \ +} while (0) + +#define DL_FOREACH(head,el) \ + DL_FOREACH2(head,el,next) + +#define DL_FOREACH2(head,el,next) \ + for ((el) = (head); el; (el) = (el)->next) + +/* this version is safe for deleting the elements during iteration */ +#define DL_FOREACH_SAFE(head,el,tmp) \ + DL_FOREACH_SAFE2(head,el,tmp,next) + +#define DL_FOREACH_SAFE2(head,el,tmp,next) \ + for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) + +/* these are identical to their singly-linked list counterparts */ +#define DL_SEARCH_SCALAR LL_SEARCH_SCALAR +#define DL_SEARCH LL_SEARCH +#define DL_SEARCH_SCALAR2 LL_SEARCH_SCALAR2 +#define DL_SEARCH2 LL_SEARCH2 + +#define DL_REPLACE_ELEM2(head, el, add, prev, next) \ +do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + (add)->next = (el)->next; \ + if ((el)->next == NULL) { \ + (add)->prev = (add); \ + } else { \ + (add)->prev = (el)->prev; \ + (add)->next->prev = (add); \ + } \ + } else { \ + (add)->next = (el)->next; \ + (add)->prev = (el)->prev; \ + (add)->prev->next = (add); \ + if ((el)->next == NULL) { \ + (head)->prev = (add); \ + } else { \ + (add)->next->prev = (add); \ + } \ + } \ +} while (0) + +#define DL_REPLACE_ELEM(head, el, add) \ + DL_REPLACE_ELEM2(head, el, add, prev, next) + +#define DL_PREPEND_ELEM2(head, el, add, prev, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + (add)->prev = (el)->prev; \ + (el)->prev = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + (add)->prev->next = (add); \ + } \ + } else { \ + DL_APPEND2(head, add, prev, next); \ + } \ +} while (0) \ + +#define DL_PREPEND_ELEM(head, el, add) \ + DL_PREPEND_ELEM2(head, el, add, prev, next) + +#define DL_APPEND_ELEM2(head, el, add, prev, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (add)->prev = (el); \ + (el)->next = (add); \ + if ((add)->next) { \ + (add)->next->prev = (add); \ + } else { \ + (head)->prev = (add); \ + } \ + } else { \ + DL_PREPEND2(head, add, prev, next); \ + } \ +} while (0) \ + +#define DL_APPEND_ELEM(head, el, add) \ + DL_APPEND_ELEM2(head, el, add, prev, next) + +/****************************************************************************** + * circular doubly linked list macros * + *****************************************************************************/ +#define CDL_APPEND(head,add) \ + CDL_APPEND2(head,add,prev,next) + +#define CDL_APPEND2(head,add,prev,next) \ +do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (add)->prev->next = (add); \ + } else { \ + (add)->prev = (add); \ + (add)->next = (add); \ + (head) = (add); \ + } \ +} while (0) + +#define CDL_PREPEND(head,add) \ + CDL_PREPEND2(head,add,prev,next) + +#define CDL_PREPEND2(head,add,prev,next) \ +do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (add)->prev->next = (add); \ + } else { \ + (add)->prev = (add); \ + (add)->next = (add); \ + } \ + (head) = (add); \ +} while (0) + +#define CDL_DELETE(head,del) \ + CDL_DELETE2(head,del,prev,next) + +#define CDL_DELETE2(head,del,prev,next) \ +do { \ + if (((head)==(del)) && ((head)->next == (head))) { \ + (head) = NULL; \ + } else { \ + (del)->next->prev = (del)->prev; \ + (del)->prev->next = (del)->next; \ + if ((del) == (head)) (head)=(del)->next; \ + } \ +} while (0) + +#define CDL_COUNT(head,el,counter) \ + CDL_COUNT2(head,el,counter,next) \ + +#define CDL_COUNT2(head, el, counter,next) \ +do { \ + (counter) = 0; \ + CDL_FOREACH2(head,el,next) { ++(counter); } \ +} while (0) + +#define CDL_FOREACH(head,el) \ + CDL_FOREACH2(head,el,next) + +#define CDL_FOREACH2(head,el,next) \ + for ((el)=(head);el;(el)=(((el)->next==(head)) ? NULL : (el)->next)) + +#define CDL_FOREACH_SAFE(head,el,tmp1,tmp2) \ + CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next) + +#define CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next) \ + for ((el) = (head), (tmp1) = (head) ? (head)->prev : NULL; \ + (el) && ((tmp2) = (el)->next, 1); \ + (el) = ((el) == (tmp1) ? NULL : (tmp2))) + +#define CDL_SEARCH_SCALAR(head,out,field,val) \ + CDL_SEARCH_SCALAR2(head,out,field,val,next) + +#define CDL_SEARCH_SCALAR2(head,out,field,val,next) \ +do { \ + CDL_FOREACH2(head,out,next) { \ + if ((out)->field == (val)) break; \ + } \ +} while (0) + +#define CDL_SEARCH(head,out,elt,cmp) \ + CDL_SEARCH2(head,out,elt,cmp,next) + +#define CDL_SEARCH2(head,out,elt,cmp,next) \ +do { \ + CDL_FOREACH2(head,out,next) { \ + if ((cmp(out,elt))==0) break; \ + } \ +} while (0) + +#define CDL_REPLACE_ELEM2(head, el, add, prev, next) \ +do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((el)->next == (el)) { \ + (add)->next = (add); \ + (add)->prev = (add); \ + (head) = (add); \ + } else { \ + (add)->next = (el)->next; \ + (add)->prev = (el)->prev; \ + (add)->next->prev = (add); \ + (add)->prev->next = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + } \ +} while (0) + +#define CDL_REPLACE_ELEM(head, el, add) \ + CDL_REPLACE_ELEM2(head, el, add, prev, next) + +#define CDL_PREPEND_ELEM2(head, el, add, prev, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + (add)->prev = (el)->prev; \ + (el)->prev = (add); \ + (add)->prev->next = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + } else { \ + CDL_APPEND2(head, add, prev, next); \ + } \ +} while (0) + +#define CDL_PREPEND_ELEM(head, el, add) \ + CDL_PREPEND_ELEM2(head, el, add, prev, next) + +#define CDL_APPEND_ELEM2(head, el, add, prev, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (add)->prev = (el); \ + (el)->next = (add); \ + (add)->next->prev = (add); \ + } else { \ + CDL_PREPEND2(head, add, prev, next); \ + } \ +} while (0) + +#define CDL_APPEND_ELEM(head, el, add) \ + CDL_APPEND_ELEM2(head, el, add, prev, next) + +#endif /* UTLIST_H */ diff --git a/src/common/thirdparty/utstring.h b/src/common/thirdparty/utstring.h new file mode 100644 index 000000000..debe5f3df --- /dev/null +++ b/src/common/thirdparty/utstring.h @@ -0,0 +1,398 @@ +/* +Copyright (c) 2008-2016, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* a dynamic string implementation using macros + */ +#ifndef UTSTRING_H +#define UTSTRING_H + +#define UTSTRING_VERSION 2.0.1 + +#ifdef __GNUC__ +#define _UNUSED_ __attribute__ ((__unused__)) +#else +#define _UNUSED_ +#endif + +#include +#include +#include +#include + +#ifndef oom +#define oom() exit(-1) +#endif + +typedef struct { + char *d; + size_t n; /* allocd size */ + size_t i; /* index of first unused byte */ +} UT_string; + +#define utstring_reserve(s,amt) \ +do { \ + if (((s)->n - (s)->i) < (size_t)(amt)) { \ + char *utstring_tmp = (char*)realloc( \ + (s)->d, (s)->n + (amt)); \ + if (utstring_tmp == NULL) oom(); \ + (s)->d = utstring_tmp; \ + (s)->n += (amt); \ + } \ +} while(0) + +#define utstring_init(s) \ +do { \ + (s)->n = 0; (s)->i = 0; (s)->d = NULL; \ + utstring_reserve(s,100); \ + (s)->d[0] = '\0'; \ +} while(0) + +#define utstring_done(s) \ +do { \ + if ((s)->d != NULL) free((s)->d); \ + (s)->n = 0; \ +} while(0) + +#define utstring_free(s) \ +do { \ + utstring_done(s); \ + free(s); \ +} while(0) + +#define utstring_new(s) \ +do { \ + s = (UT_string*)calloc(sizeof(UT_string),1); \ + if (!s) oom(); \ + utstring_init(s); \ +} while(0) + +#define utstring_renew(s) \ +do { \ + if (s) { \ + utstring_clear(s); \ + } else { \ + utstring_new(s); \ + } \ +} while(0) + +#define utstring_clear(s) \ +do { \ + (s)->i = 0; \ + (s)->d[0] = '\0'; \ +} while(0) + +#define utstring_bincpy(s,b,l) \ +do { \ + utstring_reserve((s),(l)+1); \ + if (l) memcpy(&(s)->d[(s)->i], b, l); \ + (s)->i += (l); \ + (s)->d[(s)->i]='\0'; \ +} while(0) + +#define utstring_concat(dst,src) \ +do { \ + utstring_reserve((dst),((src)->i)+1); \ + if ((src)->i) memcpy(&(dst)->d[(dst)->i], (src)->d, (src)->i); \ + (dst)->i += (src)->i; \ + (dst)->d[(dst)->i]='\0'; \ +} while(0) + +#define utstring_len(s) ((unsigned)((s)->i)) + +#define utstring_body(s) ((s)->d) + +_UNUSED_ static void utstring_printf_va(UT_string *s, const char *fmt, va_list ap) { + int n; + va_list cp; + for (;;) { +#ifdef _WIN32 + cp = ap; +#else + va_copy(cp, ap); +#endif + n = vsnprintf (&s->d[s->i], s->n-s->i, fmt, cp); + va_end(cp); + + if ((n > -1) && ((size_t) n < (s->n-s->i))) { + s->i += n; + return; + } + + /* Else try again with more space. */ + if (n > -1) utstring_reserve(s,n+1); /* exact */ + else utstring_reserve(s,(s->n)*2); /* 2x */ + } +} +#ifdef __GNUC__ +/* support printf format checking (2=the format string, 3=start of varargs) */ +static void utstring_printf(UT_string *s, const char *fmt, ...) + __attribute__ (( format( printf, 2, 3) )); +#endif +_UNUSED_ static void utstring_printf(UT_string *s, const char *fmt, ...) { + va_list ap; + va_start(ap,fmt); + utstring_printf_va(s,fmt,ap); + va_end(ap); +} + +/******************************************************************************* + * begin substring search functions * + ******************************************************************************/ +/* Build KMP table from left to right. */ +_UNUSED_ static void _utstring_BuildTable( + const char *P_Needle, + size_t P_NeedleLen, + long *P_KMP_Table) +{ + long i, j; + + i = 0; + j = i - 1; + P_KMP_Table[i] = j; + while (i < (long) P_NeedleLen) + { + while ( (j > -1) && (P_Needle[i] != P_Needle[j]) ) + { + j = P_KMP_Table[j]; + } + i++; + j++; + if (i < (long) P_NeedleLen) + { + if (P_Needle[i] == P_Needle[j]) + { + P_KMP_Table[i] = P_KMP_Table[j]; + } + else + { + P_KMP_Table[i] = j; + } + } + else + { + P_KMP_Table[i] = j; + } + } + + return; +} + + +/* Build KMP table from right to left. */ +_UNUSED_ static void _utstring_BuildTableR( + const char *P_Needle, + size_t P_NeedleLen, + long *P_KMP_Table) +{ + long i, j; + + i = P_NeedleLen - 1; + j = i + 1; + P_KMP_Table[i + 1] = j; + while (i >= 0) + { + while ( (j < (long) P_NeedleLen) && (P_Needle[i] != P_Needle[j]) ) + { + j = P_KMP_Table[j + 1]; + } + i--; + j--; + if (i >= 0) + { + if (P_Needle[i] == P_Needle[j]) + { + P_KMP_Table[i + 1] = P_KMP_Table[j + 1]; + } + else + { + P_KMP_Table[i + 1] = j; + } + } + else + { + P_KMP_Table[i + 1] = j; + } + } + + return; +} + + +/* Search data from left to right. ( Multiple search mode. ) */ +_UNUSED_ static long _utstring_find( + const char *P_Haystack, + size_t P_HaystackLen, + const char *P_Needle, + size_t P_NeedleLen, + long *P_KMP_Table) +{ + long i, j; + long V_FindPosition = -1; + + /* Search from left to right. */ + i = j = 0; + while ( (j < (int)P_HaystackLen) && (((P_HaystackLen - j) + i) >= P_NeedleLen) ) + { + while ( (i > -1) && (P_Needle[i] != P_Haystack[j]) ) + { + i = P_KMP_Table[i]; + } + i++; + j++; + if (i >= (int)P_NeedleLen) + { + /* Found. */ + V_FindPosition = j - i; + break; + } + } + + return V_FindPosition; +} + + +/* Search data from right to left. ( Multiple search mode. ) */ +_UNUSED_ static long _utstring_findR( + const char *P_Haystack, + size_t P_HaystackLen, + const char *P_Needle, + size_t P_NeedleLen, + long *P_KMP_Table) +{ + long i, j; + long V_FindPosition = -1; + + /* Search from right to left. */ + j = (P_HaystackLen - 1); + i = (P_NeedleLen - 1); + while ( (j >= 0) && (j >= i) ) + { + while ( (i < (int)P_NeedleLen) && (P_Needle[i] != P_Haystack[j]) ) + { + i = P_KMP_Table[i + 1]; + } + i--; + j--; + if (i < 0) + { + /* Found. */ + V_FindPosition = j + 1; + break; + } + } + + return V_FindPosition; +} + + +/* Search data from left to right. ( One time search mode. ) */ +_UNUSED_ static long utstring_find( + UT_string *s, + long P_StartPosition, /* Start from 0. -1 means last position. */ + const char *P_Needle, + size_t P_NeedleLen) +{ + long V_StartPosition; + long V_HaystackLen; + long *V_KMP_Table; + long V_FindPosition = -1; + + if (P_StartPosition < 0) + { + V_StartPosition = s->i + P_StartPosition; + } + else + { + V_StartPosition = P_StartPosition; + } + V_HaystackLen = s->i - V_StartPosition; + if ( (V_HaystackLen >= (long) P_NeedleLen) && (P_NeedleLen > 0) ) + { + V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1)); + if (V_KMP_Table != NULL) + { + _utstring_BuildTable(P_Needle, P_NeedleLen, V_KMP_Table); + + V_FindPosition = _utstring_find(s->d + V_StartPosition, + V_HaystackLen, + P_Needle, + P_NeedleLen, + V_KMP_Table); + if (V_FindPosition >= 0) + { + V_FindPosition += V_StartPosition; + } + + free(V_KMP_Table); + } + } + + return V_FindPosition; +} + + +/* Search data from right to left. ( One time search mode. ) */ +_UNUSED_ static long utstring_findR( + UT_string *s, + long P_StartPosition, /* Start from 0. -1 means last position. */ + const char *P_Needle, + size_t P_NeedleLen) +{ + long V_StartPosition; + long V_HaystackLen; + long *V_KMP_Table; + long V_FindPosition = -1; + + if (P_StartPosition < 0) + { + V_StartPosition = s->i + P_StartPosition; + } + else + { + V_StartPosition = P_StartPosition; + } + V_HaystackLen = V_StartPosition + 1; + if ( (V_HaystackLen >= (long) P_NeedleLen) && (P_NeedleLen > 0) ) + { + V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1)); + if (V_KMP_Table != NULL) + { + _utstring_BuildTableR(P_Needle, P_NeedleLen, V_KMP_Table); + + V_FindPosition = _utstring_findR(s->d, + V_HaystackLen, + P_Needle, + P_NeedleLen, + V_KMP_Table); + + free(V_KMP_Table); + } + } + + return V_FindPosition; +} +/******************************************************************************* + * end substring search functions * + ******************************************************************************/ + +#endif /* UTSTRING_H */ diff --git a/src/computation_graph.cc b/src/computation_graph.cc deleted file mode 100644 index 2aa696286..000000000 --- a/src/computation_graph.cc +++ /dev/null @@ -1,27 +0,0 @@ -#include "computation_graph.h" - -OperationId ComputationGraph::add_operation(std::unique_ptr operation) { - OperationId operationid = operations_.size(); - OperationId creator_operationid = operation->creator_operationid(); - RAY_CHECK_EQ(spawned_operations_.size(), operationid, "ComputationGraph is attempting to call add_operation, but spawned_operations_.size() != operationid."); - operations_.emplace_back(std::move(operation)); - if (creator_operationid != NO_OPERATION && creator_operationid != ROOT_OPERATION) { - spawned_operations_[creator_operationid].push_back(operationid); - } - spawned_operations_.push_back(std::vector()); - return operationid; -} - -const Task& ComputationGraph::get_task(OperationId operationid) { - RAY_CHECK_NEQ(operationid, ROOT_OPERATION, "ComputationGraph attempting to get_task with operationid == ROOT_OPERATION"); - RAY_CHECK_NEQ(operationid, NO_OPERATION, "ComputationGraph attempting to get_task with operationid == NO_OPERATION"); - RAY_CHECK_LT(operationid, operations_.size(), "ComputationGraph attempting to get_task with operationid " << operationid << ", but operationid >= operations_.size()."); - RAY_CHECK(operations_[operationid]->has_task(), "Calling get_task with operationid " << operationid << ", but this corresponds to a put not a task."); - return operations_[operationid]->task(); -} - -void ComputationGraph::to_protobuf(CompGraph* computation_graph) { - for (OperationId id = 0; id < operations_.size(); ++id) { - computation_graph->add_operation()->CopyFrom(*operations_[id]); - } -} diff --git a/src/computation_graph.h b/src/computation_graph.h deleted file mode 100644 index 2918569b8..000000000 --- a/src/computation_graph.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef RAY_COMPUTATIONGRAPH_H -#define RAY_COMPUTATIONGRAPH_H - -#include -#include - -#include "ray/ray.h" - -#include "graph.pb.h" -#include "types.pb.h" - -// used to represent the root operation (that is, the driver code) -const OperationId ROOT_OPERATION = std::numeric_limits::max(); -// used to represent the absence of an operation -const OperationId NO_OPERATION = std::numeric_limits::max() - 1; - -class ComputationGraph { -public: - // Add an operation to the computation graph, this returns the OperationId for - // the new operation. This method takes ownership over operation. - OperationId add_operation(std::unique_ptr operation); - // Return the task corresponding to a particular OperationId. If operationid - // corresponds to a put, then fail. - const Task& get_task(OperationId operationid); - // Serialize the computation graph to ProtoBuf and store it in computation_graph - void to_protobuf(CompGraph* computation_graph); -private: - // maps an OperationId to the corresponding task or put - std::vector > operations_; - // spawned_operations_[operationid] is a vector of the OperationIds of the - // operations spawned by the task with OperationId operationid - std::vector > spawned_operations_; -}; - -#endif diff --git a/src/ipc.cc b/src/ipc.cc deleted file mode 100644 index 1b492cc07..000000000 --- a/src/ipc.cc +++ /dev/null @@ -1,202 +0,0 @@ -#include "ipc.h" - -#if defined(__unix__) || defined(__linux__) -#include -#endif - -#include -#include "ray/ray.h" -#include "utils.h" - -ObjHandle::ObjHandle(SegmentId segmentid, size_t size, IpcPointer ipcpointer, size_t metadata_offset) - : segmentid_(segmentid), size_(size), ipcpointer_(ipcpointer), metadata_offset_(metadata_offset) -{} - -MessageQueue<>::MessageQueue() : create_(false) { } - -MessageQueue<>::~MessageQueue() { - if (!name_.empty() && create_) { - // Only remove the message queue if we created it. - RAY_LOG(RAY_DEBUG, "Removing message queue " << name_.c_str() << ", create = " << create_); - bip::message_queue::remove(name_.c_str()); - } -} - -MessageQueue<>::MessageQueue(MessageQueue&& other) { - *this = std::move(other); -} - -MessageQueue<>& MessageQueue<>::operator=(MessageQueue&& other) { - name_ = std::move(other.name_); - create_ = other.create_; - queue_ = std::move(other.queue_); - other.name_.clear(); // It is unclear if this is guaranteed, but we need it to hold for the destructor. See: https://stackoverflow.com/a/17735913 - - return *this; -} - - -bool MessageQueue<>::connect(const std::string& name, bool create, size_t message_size, size_t message_capacity) { - name_ = name; - name_.insert(0, "ray-{BC200A09-2465-431D-AEC7-2F8530B04535}-"); -#if defined(WIN32) || defined(_WIN32) - std::replace(name_.begin(), name_.end(), ':', '-'); -#endif - try { - if (create) { - bip::message_queue::remove(name_.c_str()); // remove queue if it has not been properly removed from last run - queue_ = std::unique_ptr(new bip::message_queue(bip::create_only, name_.c_str(), message_capacity, message_size)); - create_ = true; // Only set create_ = true on success. - } - else { - queue_ = std::unique_ptr(new bip::message_queue(bip::open_only, name_.c_str())); - } - } - catch (bip::interprocess_exception &ex) { - RAY_CHECK(false, "name = " << name_ << ", create = " << create << ", boost::interprocess exception: " << ex.what()); - } - return true; -} -bool MessageQueue<>::connected() { - return queue_ != NULL; -} - -bool MessageQueue<>::send(const void * object, size_t size) { - bool succeeded; - try { - // This will return true if the message was successfully sent and false if - // the message queue is full. - succeeded = queue_->try_send(object, size, 0); - } - catch (bip::interprocess_exception &ex) { - RAY_CHECK(false, "boost::interprocess exception: " << ex.what()); - } - return succeeded; -} - -bool MessageQueue<>::receive(void * object, size_t size) { - unsigned int priority; - bip::message_queue::size_type recvd_size; - try { - queue_->receive(object, size, recvd_size, priority); - } - catch (bip::interprocess_exception &ex) { - RAY_CHECK(false, "boost::interprocess exception: " << ex.what()); - } - return true; -} - -MemorySegmentPool::MemorySegmentPool(ObjStoreId objstoreid, std::string& objstore_address, bool create) : objstoreid_(objstoreid), objstore_address_(objstore_address), create_mode_(create) { - std::string::iterator split_point = split_ip_address(objstore_address); - objstore_port_.assign(split_point, objstore_address.end()); -} - -// creates a memory segment if it is not already there; if the pool is in create mode, -// space is allocated, if it is in open mode, the shared memory is mapped into the process -void MemorySegmentPool::open_segment(SegmentId segmentid, size_t size) { - RAY_LOG(RAY_DEBUG, "Opening segmentid " << segmentid << " on object store " << objstoreid_ << " with port " << objstore_port_ << " with create_mode_ = " << create_mode_); - RAY_CHECK(segmentid == segments_.size() || !create_mode_, "Object store " << objstoreid_ << " with port " << objstore_port_ << " is attempting to open segmentid " << segmentid << " on the object store, but segments_.size() = " << segments_.size()); - if (segmentid >= segments_.size()) { // resize and initialize segments_ - int current_size = segments_.size(); - segments_.resize(segmentid + 1); - for (int i = current_size; i < segments_.size(); ++i) { - segments_[i].first = nullptr; - segments_[i].second = SegmentStatusType::UNOPENED; - } - } - if (segments_[segmentid].second == SegmentStatusType::OPENED) { - return; - } - RAY_CHECK_NEQ(segments_[segmentid].second, SegmentStatusType::CLOSED, "Attempting to open segmentid " << segmentid << ", but segments_[segmentid].second == SegmentStatusType::CLOSED."); - std::string segment_name = get_segment_name(segmentid); - if (create_mode_) { - assert(size > 0); - bip::shared_memory_object::remove(segment_name.c_str()); // remove segment if it has not been properly removed from last run - size_t new_size = (size / page_size_ + 2) * page_size_; // additional room for boost's bookkeeping - segments_[segmentid] = std::make_pair(std::unique_ptr(new bip::managed_shared_memory(bip::create_only, segment_name.c_str(), new_size)), SegmentStatusType::OPENED); - } else { - segments_[segmentid] = std::make_pair(std::unique_ptr(new bip::managed_shared_memory(bip::open_only, segment_name.c_str())), SegmentStatusType::OPENED); - } -} - -void MemorySegmentPool::unmap_segment(SegmentId segmentid) { - segments_[segmentid].first.reset(); - segments_[segmentid].second = SegmentStatusType::UNOPENED; -} - -void MemorySegmentPool::close_segment(SegmentId segmentid) { - RAY_LOG(RAY_DEBUG, "closing segmentid " << segmentid); - std::string segment_name = get_segment_name(segmentid); - bip::shared_memory_object::remove(segment_name.c_str()); - segments_[segmentid].first.reset(); - segments_[segmentid].second = SegmentStatusType::CLOSED; -} - -ObjHandle MemorySegmentPool::allocate(size_t size) { - RAY_CHECK(create_mode_, "Attempting to call allocate, but create_mode_ is false"); - // TODO(pcm): at the moment, this always creates a new segment, this will be changed - SegmentId segmentid = segments_.size(); - open_segment(segmentid, size); - objstore_memcheck(size); - void* ptr = segments_[segmentid].first->allocate(size); - auto handle = segments_[segmentid].first->get_handle_from_address(ptr); - return ObjHandle(segmentid, size, handle); -} - -void MemorySegmentPool::deallocate(ObjHandle pointer) { - SegmentId segmentid = pointer.segmentid(); - void* ptr = segments_[segmentid].first->get_address_from_handle(pointer.ipcpointer()); - segments_[segmentid].first->deallocate(ptr); - close_segment(segmentid); -} - -// returns address of the object refered to by the handle, needs to be called on -// the process that will use the address -uint8_t* MemorySegmentPool::get_address(ObjHandle pointer) { - RAY_CHECK(!create_mode_ || segments_[pointer.segmentid()].second == SegmentStatusType::OPENED, "Object store " << objstoreid_ << " is attempting to call get_address on segmentid " << pointer.segmentid() << ", which has not been opened yet."); - if (!create_mode_) { - open_segment(pointer.segmentid()); - } - bip::managed_shared_memory* segment = segments_[pointer.segmentid()].first.get(); - return static_cast(segment->get_address_from_handle(pointer.ipcpointer())); -} - -// returns the name of the segment -std::string MemorySegmentPool::get_segment_name(SegmentId segmentid) { - return std::string("ray-{BC200A09-2465-431D-AEC7-2F8530B04535}-objstore-") + std::to_string(objstoreid_) + "-" + objstore_port_ + std::string("-segment-") + std::to_string(segmentid); -} - -MemorySegmentPool::~MemorySegmentPool() { - destroy_segments(); -} - -void MemorySegmentPool::objstore_memcheck(int64_t size) { -#if defined(__unix__) || defined(__linux__) - struct statvfs buffer; - statvfs("/dev/shm/", &buffer); - if (size + 100 > buffer.f_bsize * buffer.f_bavail) { - MemorySegmentPool::destroy_segments(); - RAY_LOG(RAY_FATAL, "Not enough memory for allocating object in objectstore."); - } -#endif -} - -void MemorySegmentPool::destroy_segments() { - for (size_t segmentid = 0; segmentid < segments_.size(); ++segmentid) { - std::string segment_name = get_segment_name(segmentid); - segments_[segmentid].first.reset(); - bip::shared_memory_object::remove(segment_name.c_str()); - } -} -#if defined(WIN32) || defined(_WIN32) -namespace boost { - namespace interprocess { - namespace ipcdetail { - windows_bootstamp windows_intermodule_singleton::get() { - // HACK: Only do this for Windows as there seems to be no better workaround. Possibly undefined behavior! - return reinterpret_cast(std::string("BOOTSTAMP")); - } - } - } -} -#endif diff --git a/src/ipc.h b/src/ipc.h deleted file mode 100644 index 03300f4ba..000000000 --- a/src/ipc.h +++ /dev/null @@ -1,142 +0,0 @@ -#ifndef RAY_IPC_H -#define RAY_IPC_H - -#include -#include - -#if defined(WIN32) || defined(_WIN32) -#include -namespace boost { - namespace interprocess { - namespace ipcdetail { - struct windows_bootstamp; - template<> - class windows_intermodule_singleton { - public: - static windows_bootstamp get(); - }; - } - } -} -#endif - -#include -#include - -#include "ray/ray.h" - -namespace bip = boost::interprocess; - -// Methods for inter process communication (abstracts from the shared memory implementation) - -// Message Queues: Exchanging objects of type T between processes on a node - -template -class MessageQueue; - -template<> -class MessageQueue<> { -public: - ~MessageQueue(); - MessageQueue(); - MessageQueue(MessageQueue&& other); - MessageQueue& operator=(MessageQueue&& other); - bool connected(); -protected: - bool connect(const std::string& name, bool create, size_t message_size, size_t message_capacity); - bool send(const void* object, size_t size);; - bool receive(void* object, size_t size); -private: - std::string name_; - bool create_; - std::unique_ptr queue_; -}; - -template -class MessageQueue : public MessageQueue<> { -public: - bool connect(const std::string& name, bool create, size_t capacity = 1000) { return MessageQueue<>::connect(name, create, sizeof(T), capacity); } - bool send(const T* object) { return MessageQueue<>::send(object, sizeof(*object)); }; - bool receive(T* object) { return MessageQueue<>::receive(object, sizeof(*object)); } -}; - -// Object Queues - -// For communicating between object store and workers, the following -// messages can be sent: - -// ALLOC: workerid, objectid, size -> objhandle: -// worker requests an allocation from the object store -// GET: workerid, objectid -> objhandle: -// worker requests an object from the object store -// WORKER_DONE: workerid, objectid -> (): -// worker tells the object store that an object has been finalized -// ALIAS_DONE: objectid -> (): -// objstore tells itself that it has finalized something (perhaps an alias) - -enum ObjRequestType {ALLOC = 0, GET = 1, WORKER_DONE = 2, ALIAS_DONE = 3}; - -struct ObjRequest { - WorkerId workerid; // worker that sends the request - ObjRequestType type; // do we want to allocate a new object or get a handle? - ObjectID objectid; // object ID of the object to be returned/allocated - int64_t size; // if allocate, that's the size of the object - int64_t metadata_offset; // if sending 'WORKER_DONE', that's the location of the metadata relative to the beginning of the object -}; - -typedef size_t SegmentId; // index into a memory segment table -typedef bip::managed_shared_memory::handle_t IpcPointer; - -// Object handle: Handle to object that can be passed around between processes -// that are connected to the same object store - -class ObjHandle { -public: - ObjHandle(SegmentId segmentid = 0, size_t size = 0, IpcPointer ipcpointer = IpcPointer(), size_t metadata_offset = 0); - SegmentId segmentid() { return segmentid_; } - size_t size() { return size_; } - IpcPointer ipcpointer() { return ipcpointer_; } - size_t metadata_offset() { return metadata_offset_; } - void set_metadata_offset(size_t metadata_offset) {metadata_offset_ = metadata_offset; } -private: - SegmentId segmentid_; // which shared memory file the object is stored in - IpcPointer ipcpointer_; // pointer to the beginning of the object, exchangeable between processes - size_t size_; // total size of the object - size_t metadata_offset_; // offset of the metadata that describes this object -}; - -// Memory segment pool: A collection of shared memory segments -// used in two modes: -// \item on the object store it is used with create = true, in this case the -// segments are allocated -// \item on the worker it is used in open mode, with create = false, in this case -// the segments, which have been created by the object store, are just mapped -// into memory - -enum SegmentStatusType {UNOPENED = 0, OPENED = 1, CLOSED = 2}; - -class MemorySegmentPool { -public: - MemorySegmentPool(ObjStoreId objstoreid, std::string& objstore_address, bool create); // can be used in two modes: create mode and open mode (see above) - ~MemorySegmentPool(); - ObjHandle allocate(size_t nbytes); // allocate memory, potentially creating a new segment (only run on object store) - void deallocate(ObjHandle pointer); // deallocate object, potentially deallocating a new segment (only run on object store) - uint8_t* get_address(ObjHandle pointer); // get address of shared object - std::string get_segment_name(SegmentId segmentid); // get the name of a segment - void unmap_segment(SegmentId segmentid); // unmap a memory segment from a client (only to be called by clients) - void destroy_segments(); - void objstore_memcheck(int64_t size); -private: - void open_segment(SegmentId segmentid, size_t size = 0); // create a segment or map an existing one into memory - void close_segment(SegmentId segmentid); // close a segment - bool create_mode_; // true in the object stores, false on the workers - ObjStoreId objstoreid_; // the identity of the associated object store - // The address of the object store. - std::string objstore_address_; - // The port of the object store. This is used to help avoid name collisions. - std::string objstore_port_; - size_t page_size_ = bip::mapped_region::get_page_size(); - std::vector, SegmentStatusType> > segments_; -}; - -#endif diff --git a/src/objstore.cc b/src/objstore.cc deleted file mode 100644 index 8f5c82931..000000000 --- a/src/objstore.cc +++ /dev/null @@ -1,375 +0,0 @@ -#include "objstore.h" - -#include -#include "utils.h" - -const size_t ObjStoreService::CHUNK_SIZE = 8 * 1024; - -// this method needs to be protected by a objstore_lock_ -// TODO(rkn): Make sure that we do not in fact need the objstore_lock_. We want multiple deliveries to be able to happen simultaneously. -void ObjStoreService::get_data_from(ObjectID objectid, ObjStore::Stub& stub) { - RAY_LOG(RAY_DEBUG, "Objstore " << objstoreid_ << " is beginning to get objectid " << objectid); - ObjChunk chunk; - ClientContext context; - StreamObjToRequest stream_request; - stream_request.set_objectid(objectid); - std::unique_ptr > reader(stub.StreamObjTo(&context, stream_request)); - - size_t total_size = 0; - ObjHandle handle; - if (reader->Read(&chunk)) { - total_size = chunk.total_size(); - handle = alloc(objectid, total_size); - } - size_t num_bytes = 0; - segmentpool_lock_.lock(); - uint8_t* data = segmentpool_->get_address(handle); - segmentpool_lock_.unlock(); - do { - RAY_CHECK_LE(num_bytes + chunk.data().size(), total_size, "The reader attempted to stream too many bytes."); - std::memcpy(data, chunk.data().c_str(), chunk.data().size()); - data += chunk.data().size(); - num_bytes += chunk.data().size(); - } while (reader->Read(&chunk)); - RAY_CHECK_GRPC(reader->Finish()); - - // finalize object - RAY_CHECK_EQ(num_bytes, total_size, "Streamed objectid " << objectid << ", but num_bytes != total_size"); - object_ready(objectid, chunk.metadata_offset()); - RAY_LOG(RAY_DEBUG, "finished streaming data, objectid was " << objectid << " and size was " << num_bytes); -} - -ObjStoreService::ObjStoreService(std::shared_ptr scheduler_channel) - : scheduler_stub_(Scheduler::NewStub(scheduler_channel)) { -} - -void ObjStoreService::register_objstore(const std::string& objstore_address, const std::string& recv_queue_name) { - // Create the queue that will be used by workers to send requests to the - // object store. - RAY_LOG(RAY_INFO, "Object store is creating queue with name " << recv_queue_name); - RAY_CHECK(recv_queue_.connect(recv_queue_name, true), "error connecting recv_queue_"); - objstore_address_ = objstore_address; - // Register the object store with the scheduler. - ClientContext context; - RegisterObjStoreRequest request; - request.set_objstore_address(objstore_address); - RegisterObjStoreReply reply; - RAY_CHECK_GRPC(scheduler_stub_->RegisterObjStore(&context, request, &reply)); - objstoreid_ = reply.objstoreid(); - segmentpool_ = std::make_shared(objstoreid_, objstore_address_, true); -} - -// this method needs to be protected by a objstores_lock_ -ObjStore::Stub& ObjStoreService::get_objstore_stub(const std::string& objstore_address) { - auto iter = objstores_.find(objstore_address); - if (iter != objstores_.end()) - return *(iter->second); - auto channel = grpc::CreateChannel(objstore_address, grpc::InsecureChannelCredentials()); - objstores_.emplace(objstore_address, ObjStore::NewStub(channel)); - return *objstores_[objstore_address]; -} - -Status ObjStoreService::StartDelivery(ServerContext* context, const StartDeliveryRequest* request, AckReply* reply) { - // TODO(rkn): We're pushing the delivery task onto a new thread so that this method can return immediately. This matters - // because the scheduler holds a lock while DeliverObj is being called. The correct solution is to make DeliverObj - // an asynchronous call (and similarly with the rest of the object store service methods). - std::string address = request->objstore_address(); - ObjectID objectid = request->objectid(); - { - std::lock_guard memory_lock(memory_lock_); - if (objectid >= memory_.size()) { - memory_.resize(objectid + 1, std::make_pair(ObjHandle(), MemoryStatusType::NOT_PRESENT)); - } - if (memory_[objectid].second == MemoryStatusType::NOT_PRESENT) { - } - else { - RAY_CHECK_NEQ(memory_[objectid].second, MemoryStatusType::DEALLOCATED, "Objstore " << objstoreid_ << " is attempting to get objectid " << objectid << ", but memory_[objectid] == DEALLOCATED."); - RAY_LOG(RAY_DEBUG, "Objstore " << objstoreid_ << " already has objectid " << objectid << " or it is already being shipped, so no need to get it again."); - return Status::OK; - } - memory_[objectid].second = MemoryStatusType::PRE_ALLOCED; - } - delivery_threads_.push_back(std::make_shared([this, address, objectid]() { - std::lock_guard objstores_lock(objstores_lock_); - ObjStore::Stub& stub = get_objstore_stub(address); - get_data_from(objectid, stub); - })); - return Status::OK; -} - -Status ObjStoreService::ObjStoreInfo(ServerContext* context, const ObjStoreInfoRequest* request, ObjStoreInfoReply* reply) { - std::lock_guard memory_lock(memory_lock_); - for (size_t i = 0; i < memory_.size(); ++i) { - if (memory_[i].second == MemoryStatusType::READY) { // is the object available? - reply->add_objectid(i); - } - } - /* - for (int i = 0; i < request->objectid_size(); ++i) { - ObjectID objectid = request->objectid(i); - Obj* obj = new Obj(); - std::string data(memory_[objectid].ptr.data, memory_[objectid].ptr.len); // copies, but for debugging should be ok - obj->ParseFromString(data); - reply->mutable_obj()->AddAllocated(obj); - } - */ - return Status::OK; -} - -Status ObjStoreService::StreamObjTo(ServerContext* context, const StreamObjToRequest* request, ServerWriter* writer) { - RAY_LOG(RAY_DEBUG, "begin to stream data from object store " << objstoreid_); - ObjChunk chunk; - ObjectID objectid = request->objectid(); - memory_lock_.lock(); - RAY_CHECK_LT(objectid, memory_.size(), "Objstore " << objstoreid_ << " is attempting to use objectid " << objectid << " in StreamObjTo, but this objectid is not present in the object store."); - RAY_CHECK_EQ(memory_[objectid].second, MemoryStatusType::READY, "Objstore " << objstoreid_ << " is attempting to stream objectid " << objectid << ", but memory_[objectid].second != MemoryStatusType::READY."); - ObjHandle handle = memory_[objectid].first; - memory_lock_.unlock(); // TODO(rkn): Make sure we don't still need to hold on to this lock. - segmentpool_lock_.lock(); - const uint8_t* head = segmentpool_->get_address(handle); - segmentpool_lock_.unlock(); - size_t size = handle.size(); - for (size_t i = 0; i < size; i += CHUNK_SIZE) { - chunk.set_metadata_offset(handle.metadata_offset()); - chunk.set_total_size(size); - chunk.set_data(head + i, std::min(CHUNK_SIZE, size - i)); - RAY_CHECK(writer->Write(chunk), "stream connection prematurely closed") - } - return Status::OK; -} - -Status ObjStoreService::NotifyAlias(ServerContext* context, const NotifyAliasRequest* request, AckReply* reply) { - // NotifyAlias assumes that the objstore already holds canonical_objectid - ObjectID alias_objectid = request->alias_objectid(); - ObjectID canonical_objectid = request->canonical_objectid(); - RAY_LOG(RAY_DEBUG, "Aliasing objectid " << alias_objectid << " with objectid " << canonical_objectid); - { - std::lock_guard memory_lock(memory_lock_); - RAY_CHECK_LT(canonical_objectid, memory_.size(), "Attempting to alias objectid " << alias_objectid << " with objectid " << canonical_objectid << ", but objectid " << canonical_objectid << " is not in the objstore.") - RAY_CHECK_NEQ(memory_[canonical_objectid].second, MemoryStatusType::NOT_READY, "Attempting to alias objectid " << alias_objectid << " with objectid " << canonical_objectid << ", but objectid " << canonical_objectid << " is not ready yet in the objstore.") - RAY_CHECK_NEQ(memory_[canonical_objectid].second, MemoryStatusType::NOT_PRESENT, "Attempting to alias objectid " << alias_objectid << " with objectid " << canonical_objectid << ", but objectid " << canonical_objectid << " is not present in the objstore.") - RAY_CHECK_NEQ(memory_[canonical_objectid].second, MemoryStatusType::DEALLOCATED, "Attempting to alias objectid " << alias_objectid << " with objectid " << canonical_objectid << ", but objectid " << canonical_objectid << " has already been deallocated.") - if (alias_objectid >= memory_.size()) { - memory_.resize(alias_objectid + 1, std::make_pair(ObjHandle(), MemoryStatusType::NOT_PRESENT)); - } - memory_[alias_objectid].first = memory_[canonical_objectid].first; - memory_[alias_objectid].second = MemoryStatusType::READY; - } - ObjRequest done_request; - done_request.type = ObjRequestType::ALIAS_DONE; - done_request.objectid = alias_objectid; - RAY_CHECK(recv_queue_.send(&done_request), "Failed to send message from the object store to itself because the message queue was full."); - return Status::OK; -} - -Status ObjStoreService::DeallocateObject(ServerContext* context, const DeallocateObjectRequest* request, AckReply* reply) { - ObjectID canonical_objectid = request->canonical_objectid(); - RAY_LOG(RAY_INFO, "Deallocating canonical_objectid " << canonical_objectid); - std::lock_guard memory_lock(memory_lock_); - RAY_CHECK_EQ(memory_[canonical_objectid].second, MemoryStatusType::READY, "Attempting to deallocate canonical_objectid " << canonical_objectid << ", but memory_[canonical_objectid].second = " << memory_[canonical_objectid].second); - RAY_CHECK_LT(canonical_objectid, memory_.size(), "Attempting to deallocate canonical_objectid " << canonical_objectid << ", but it is not in the objstore."); - segmentpool_lock_.lock(); - segmentpool_->deallocate(memory_[canonical_objectid].first); - segmentpool_lock_.unlock(); - memory_[canonical_objectid].second = MemoryStatusType::DEALLOCATED; - return Status::OK; -} - -// This table describes how the memory status changes in response to requests. -// -// MemoryStatus | ObjRequest | New MemoryStatus | action performed -// -------------+-------------+------------------+---------------------------- -// NOT_PRESENT | ALLOC | NOT_READY | allocate object -// NOT_READY | WORKER_DONE | READY | send ObjReady to scheduler -// NOT_READY | GET | NOT_READY | add to get queue -// READY | GET | READY | return handle -// READY | DEALLOC | DEALLOCATED | deallocate -// -------------+-------------+------------------+---------------------------- -void ObjStoreService::process_objstore_request(const ObjRequest request) { - switch (request.type) { - case ObjRequestType::ALIAS_DONE: { - process_gets_for_objectid(request.objectid); - } - break; - default: { - RAY_CHECK(false, "Attempting to process request of type " << request.type << ". This code should be unreachable."); - } - } -} - -void ObjStoreService::process_worker_request(const ObjRequest request) { - if (request.workerid >= send_queues_.size()) { - send_queues_.resize(request.workerid + 1); - } - if (!send_queues_[request.workerid].connected()) { - std::string queue_name = std::string("queue:") + objstore_address_ + std::string(":worker:") + std::to_string(request.workerid) + std::string(":obj"); - RAY_CHECK(send_queues_[request.workerid].connect(queue_name, false), "error connecting receive_queue_"); - } - { - std::lock_guard memory_lock(memory_lock_); - if (request.objectid >= memory_.size()) { - memory_.resize(request.objectid + 1, std::make_pair(ObjHandle(), MemoryStatusType::NOT_PRESENT)); - } - } - switch (request.type) { - case ObjRequestType::ALLOC: { - ObjHandle handle = alloc(request.objectid, request.size); // This method acquires memory_lock_ - RAY_CHECK(send_queues_[request.workerid].send(&handle), "Failed to send message from the object store to the worker with id " << request.workerid << " because the message queue was full."); - } - break; - case ObjRequestType::GET: { - std::lock_guard memory_lock(memory_lock_); - std::pair& item = memory_[request.objectid]; - if (item.second == MemoryStatusType::READY) { - RAY_LOG(RAY_DEBUG, "Responding to GET request: returning objectid " << request.objectid); - RAY_CHECK(send_queues_[request.workerid].send(&item.first), "Failed to send message from the object store to the worker with id " << request.workerid << " because the message queue was full."); - } else if (item.second == MemoryStatusType::NOT_READY || item.second == MemoryStatusType::NOT_PRESENT || item.second == MemoryStatusType::PRE_ALLOCED) { - std::lock_guard lock(get_queue_lock_); - get_queue_.push_back(std::make_pair(request.workerid, request.objectid)); - } else { - RAY_CHECK(false, "A worker requested objectid " << request.objectid << ", but memory_[objectid].second = " << memory_[request.objectid].second); - } - } - break; - case ObjRequestType::WORKER_DONE: { - object_ready(request.objectid, request.metadata_offset); // This method acquires memory_lock_ - } - break; - default: { - RAY_CHECK(false, "Attempting to process request of type " << request.type << ". This code should be unreachable."); - } - } -} - -void ObjStoreService::process_requests() { - // TODO(rkn): Should memory_lock_ be used in this method? - ObjRequest request; - while (true) { - RAY_CHECK(recv_queue_.receive(&request), "error receiving over IPC"); - switch (request.type) { - case ObjRequestType::ALLOC: { - RAY_LOG(RAY_VERBOSE, "Request (worker " << request.workerid << " to objstore " << objstoreid_ << "): Allocate object with objectid " << request.objectid << " and size " << request.size); - process_worker_request(request); - } - break; - case ObjRequestType::GET: { - RAY_LOG(RAY_VERBOSE, "Request (worker " << request.workerid << " to objstore " << objstoreid_ << "): Get object with objectid " << request.objectid); - process_worker_request(request); - } - break; - case ObjRequestType::WORKER_DONE: { - RAY_LOG(RAY_VERBOSE, "Request (worker " << request.workerid << " to objstore " << objstoreid_ << "): Finalize object with objectid " << request.objectid); - process_worker_request(request); - } - break; - case ObjRequestType::ALIAS_DONE: { - process_objstore_request(request); - } - break; - default: { - RAY_CHECK(false, "Attempting to process request of type " << request.type << ". This code should be unreachable."); - } - } - } -} - -void ObjStoreService::process_gets_for_objectid(ObjectID objectid) { - std::pair& item = memory_[objectid]; - std::lock_guard get_queue_lock(get_queue_lock_); - for (size_t i = 0; i < get_queue_.size(); ++i) { - if (get_queue_[i].second == objectid) { - ObjHandle& elem = memory_[objectid].first; - RAY_CHECK(send_queues_[get_queue_[i].first].send(&item.first), "Failed to send message from the object store to the worker with id " << get_queue_[i].first << " because the message queue was full."); - // Remove the get task from the queue - std::swap(get_queue_[i], get_queue_[get_queue_.size() - 1]); - get_queue_.pop_back(); - i -= 1; - } - } -} - -ObjHandle ObjStoreService::alloc(ObjectID objectid, size_t size) { - segmentpool_lock_.lock(); - ObjHandle handle = segmentpool_->allocate(size); - segmentpool_lock_.unlock(); - std::lock_guard memory_lock(memory_lock_); - RAY_LOG(RAY_VERBOSE, "Allocating space for objectid " << objectid << " on object store " << objstoreid_); - RAY_CHECK(memory_[objectid].second == MemoryStatusType::NOT_PRESENT || memory_[objectid].second == MemoryStatusType::PRE_ALLOCED, "Attempting to allocate space for objectid " << objectid << ", but memory_[objectid].second = " << memory_[objectid].second); - memory_[objectid].first = handle; - memory_[objectid].second = MemoryStatusType::NOT_READY; - return handle; -} - -void ObjStoreService::object_ready(ObjectID objectid, size_t metadata_offset) { - { - RAY_LOG(RAY_INFO, "Object with ObjectID " << objectid << " is ready."); - std::lock_guard memory_lock(memory_lock_); - std::pair& item = memory_[objectid]; - RAY_CHECK_EQ(item.second, MemoryStatusType::NOT_READY, "A worker notified the object store that objectid " << objectid << " has been written to the object store, but memory_[objectid].second != NOT_READY."); - item.first.set_metadata_offset(metadata_offset); - item.second = MemoryStatusType::READY; - } - process_gets_for_objectid(objectid); - // Tell the scheduler that the object arrived - // TODO(pcm): put this in a separate thread so we don't have to pay the latency here - ClientContext objready_context; - ObjReadyRequest objready_request; - objready_request.set_objectid(objectid); - objready_request.set_objstoreid(objstoreid_); - AckReply objready_reply; - RAY_CHECK_GRPC(scheduler_stub_->ObjReady(&objready_context, objready_request, &objready_reply)); -} - -void ObjStoreService::start_objstore_service() { - communicator_thread_ = std::thread([this]() { - RAY_LOG(RAY_INFO, "started object store communicator server"); - process_requests(); - }); -} - -void start_objstore(const char* scheduler_addr, const char* node_ip_address) { - RAY_LOG(RAY_INFO, "Starting an object store on node " << std::string(node_ip_address)); - auto scheduler_channel = grpc::CreateChannel(scheduler_addr, grpc::InsecureChannelCredentials()); - RAY_LOG(RAY_INFO, "Object store connected to scheduler " << scheduler_addr); - ObjStoreService service(scheduler_channel); - ServerBuilder builder; - // Get GRPC to assign an unused port. - int port; - builder.AddListeningPort(std::string("0.0.0.0:0"), grpc::InsecureServerCredentials(), &port); - builder.RegisterService(&service); - std::unique_ptr server(builder.BuildAndStart()); - if (server == nullptr) { - RAY_CHECK(false, "Failed to create the object store service."); - } - std::string objstore_address = std::string(node_ip_address) + ":" + std::to_string(port); - RAY_LOG(RAY_INFO, "This object store has address " << objstore_address); - std::string recv_queue_name = std::string("queue:") + objstore_address + std::string(":obj"); - service.register_objstore(objstore_address, recv_queue_name); - service.start_objstore_service(); - // Process incoming GRPC calls. These may come from the scheduler or from - // other object stores. This method does not return. - server->Wait(); -} - -RayConfig global_ray_config; - -int main(int argc, char** argv) { - RAY_CHECK_GE(argc, 3, "object store: expected at least two arguments (scheduler ip address and object store ip address)"); - - if (argc > 3) { - const char* log_file_name = get_cmd_option(argv, argv + argc, "--log-file-name"); - if (log_file_name) { - std::cout << "object store: writing to log file " << log_file_name << std::endl; - create_log_dir_or_die(log_file_name); - global_ray_config.log_to_file = true; - global_ray_config.logfile.open(log_file_name); - } else { - std::cout << "object store: writing logs to stdout; you can change this by passing --log-file-name to ./scheduler" << std::endl; - global_ray_config.log_to_file = false; - } - } - - start_objstore(argv[1], argv[2]); - - return 0; -} diff --git a/src/objstore.h b/src/objstore.h deleted file mode 100644 index 351b09068..000000000 --- a/src/objstore.h +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef RAY_OBJSTORE_H -#define RAY_OBJSTORE_H - -#include -#include -#include -#include -#include - -#include "ray/ray.h" -#include "ray.grpc.pb.h" -#include "types.pb.h" -#include "ipc.h" - -using grpc::Server; -using grpc::ServerBuilder; -using grpc::ServerReader; -using grpc::ServerContext; -using grpc::ClientContext; -using grpc::ServerWriter; -using grpc::ClientReader; -using grpc::Status; - -using grpc::Channel; - -// READY: This is used to indicate that the object has been copied from a -// worker and is ready to be used. -// NOT_READY: This is used to indicate that memory has been allocated for the -// object, but the object hasn't been copied from a worker yet. -// DEALLOCATED: This is used to indicate that the object has been deallocated. -// NOT_PRESENT: This is used to indicate that space has not been allocated for -// this object in this object store. -// PRE_ALLOCED: This is used to indicate that the memory has not yet been -// alloced, but it will be alloced soon. This is set when we call -// StartDelivery. -enum MemoryStatusType {READY = 0, NOT_READY = 1, DEALLOCATED = 2, NOT_PRESENT = 3, PRE_ALLOCED = 4}; - -class ObjStoreService final : public ObjStore::Service { -public: - ObjStoreService(std::shared_ptr scheduler_channel); - - Status StartDelivery(ServerContext* context, const StartDeliveryRequest* request, AckReply* reply) override; - Status StreamObjTo(ServerContext* context, const StreamObjToRequest* request, ServerWriter* writer) override; - Status NotifyAlias(ServerContext* context, const NotifyAliasRequest* request, AckReply* reply) override; - Status DeallocateObject(ServerContext* context, const DeallocateObjectRequest* request, AckReply* reply) override; - Status ObjStoreInfo(ServerContext* context, const ObjStoreInfoRequest* request, ObjStoreInfoReply* reply) override; - void start_objstore_service(); - void register_objstore(const std::string& objstore_address, const std::string& recv_queue_name); -private: - void get_data_from(ObjectID objectid, ObjStore::Stub& stub); - // check if we already connected to the other objstore, if yes, return reference to connection, otherwise connect - ObjStore::Stub& get_objstore_stub(const std::string& objstore_address); - void process_worker_request(const ObjRequest request); - void process_objstore_request(const ObjRequest request); - void process_requests(); - void process_gets_for_objectid(ObjectID objectid); - ObjHandle alloc(ObjectID objectid, size_t size); - void object_ready(ObjectID objectid, size_t metadata_offset); - - static const size_t CHUNK_SIZE; - std::string objstore_address_; - ObjStoreId objstoreid_; // id of this objectstore in the scheduler object store table - std::shared_ptr segmentpool_; - std::mutex segmentpool_lock_; - std::vector > memory_; // object ID -> (memory address, memory status) - std::mutex memory_lock_; - std::unordered_map> objstores_; - std::mutex objstores_lock_; - std::unique_ptr scheduler_stub_; - std::vector > get_queue_; - std::mutex get_queue_lock_; - MessageQueue recv_queue_; // This queue is used by workers to send tasks to the object store. - std::vector > send_queues_; // This maps workerid -> queue. The object store uses these queues to send replies to the relevant workers. - std::thread communicator_thread_; - - std::vector > delivery_threads_; // TODO(rkn): document - // TODO(rkn): possibly add lock, and properly remove these threads from the delivery_threads_ when the deliveries are done - -}; - -#endif diff --git a/src/photon/Makefile b/src/photon/Makefile new file mode 100644 index 000000000..81f38d06c --- /dev/null +++ b/src/photon/Makefile @@ -0,0 +1,22 @@ +CC = gcc +CFLAGS = -g -Wall --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -Icommon -Icommon/thirdparty -fPIC +BUILD = build + +all: $(BUILD)/photon_scheduler $(BUILD)/photon_client.a + +$(BUILD)/photon_client.a: photon_client.o + ar rcs $(BUILD)/photon_client.a photon_client.o + +$(BUILD)/photon_scheduler: photon.h photon_scheduler.c photon_algorithm.c common + $(CC) $(CFLAGS) -o $@ photon_scheduler.c photon_algorithm.c common/build/libcommon.a common/thirdparty/hiredis/libhiredis.a -Icommon/thirdparty/ -Icommon/ ../plasma/build/libplasma_client.a -I../plasma/src/ + +common: FORCE + git submodule update --init --recursive + cd common; make + +clean: + cd common; make clean + rm -r $(BUILD)/* + rm *.o + +FORCE: diff --git a/src/photon/build/.gitkeep b/src/photon/build/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/src/photon/lib/python/photon_extension.c b/src/photon/lib/python/photon_extension.c new file mode 100644 index 000000000..4aa199887 --- /dev/null +++ b/src/photon/lib/python/photon_extension.c @@ -0,0 +1,140 @@ +#include + +#include "common_extension.h" +#include "photon_client.h" +#include "task.h" + +PyObject *PhotonError; + +// clang-format off +typedef struct { + PyObject_HEAD + photon_conn *photon_connection; +} PyPhotonClient; +// clang-format on + +static int PyPhotonClient_init(PyPhotonClient *self, PyObject *args, + PyObject *kwds) { + char *socket_name; + if (!PyArg_ParseTuple(args, "s", &socket_name)) { + return -1; + } + self->photon_connection = photon_connect(socket_name); + return 0; +} + +static void PyPhotonClient_dealloc(PyPhotonClient *self) { + free(((PyPhotonClient *)self)->photon_connection); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject *PyPhotonClient_submit(PyObject *self, PyObject *args) { + PyObject *py_task; + if (!PyArg_ParseTuple(args, "O", &py_task)) { + return NULL; + } + photon_submit(((PyPhotonClient *)self)->photon_connection, + ((PyTask *)py_task)->spec); + Py_RETURN_NONE; +} + +// clang-format off +static PyObject *PyPhotonClient_get_task(PyObject *self) { + task_spec *task_spec; + /* Drop the global interpreter lock while we get a task because + * photon_get_task may block for a long time. */ + Py_BEGIN_ALLOW_THREADS + task_spec = photon_get_task(((PyPhotonClient *)self)->photon_connection); + Py_END_ALLOW_THREADS + return PyTask_make(task_spec); +} +// clang-format on + +static PyMethodDef PyPhotonClient_methods[] = { + {"submit", (PyCFunction)PyPhotonClient_submit, METH_VARARGS, + "Submit a task to the local scheduler."}, + {"get_task", (PyCFunction)PyPhotonClient_get_task, METH_NOARGS, + "Get a task from the local scheduler."}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject PyPhotonClientType = { + PyObject_HEAD_INIT(NULL) 0, /* ob_size */ + "photon.PhotonClient", /* tp_name */ + sizeof(PyPhotonClient), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyPhotonClient_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "PhotonClient object", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyPhotonClient_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PyPhotonClient_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +static PyMethodDef photon_methods[] = { + {"check_simple_value", check_simple_value, METH_VARARGS, + "Should the object be passed by value?"}, + {NULL} /* Sentinel */ +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif + +PyMODINIT_FUNC initphoton(void) { + PyObject *m; + + if (PyType_Ready(&PyTaskType) < 0) + return; + + if (PyType_Ready(&PyObjectIDType) < 0) + return; + + if (PyType_Ready(&PyPhotonClientType) < 0) + return; + + m = Py_InitModule3("photon", photon_methods, + "A module for the local scheduler."); + + Py_INCREF(&PyTaskType); + PyModule_AddObject(m, "Task", (PyObject *)&PyTaskType); + + Py_INCREF(&PyObjectIDType); + PyModule_AddObject(m, "ObjectID", (PyObject *)&PyObjectIDType); + + Py_INCREF(&PyPhotonClientType); + PyModule_AddObject(m, "PhotonClient", (PyObject *)&PyPhotonClientType); + + char photon_error[] = "photon.error"; + PhotonError = PyErr_NewException(photon_error, NULL, NULL); + Py_INCREF(PhotonError); + PyModule_AddObject(m, "photon_error", PhotonError); +} diff --git a/src/photon/lib/python/setup.py b/src/photon/lib/python/setup.py new file mode 100644 index 000000000..d94fd5a83 --- /dev/null +++ b/src/photon/lib/python/setup.py @@ -0,0 +1,14 @@ +from setuptools import setup, find_packages, Extension + +photon_module = Extension("photon", + sources=["photon_extension.c", "../../common/lib/python/common_extension.c"], + include_dirs=["../../", "../../common/", + "../../common/thirdparty/", + "../../common/lib/python"], + extra_objects=["../../build/photon_client.a", "../../common/build/libcommon.a"], + extra_compile_args=["--std=c99", "-Werror"]) + +setup(name="Photon", + version="0.1", + description="Photon library for Ray", + ext_modules=[photon_module]) diff --git a/src/photon/photon.h b/src/photon/photon.h new file mode 100644 index 000000000..298a27383 --- /dev/null +++ b/src/photon/photon.h @@ -0,0 +1,40 @@ +#ifndef PHOTON_H +#define PHOTON_H + +#include "common/task.h" +#include "common/state/db.h" +#include "utarray.h" +#include "uthash.h" + +enum photon_message_type { + /** Notify the local scheduler that a task has finished. */ + TASK_DONE = 64, + /** Get a new task from the local scheduler. */ + GET_TASK, + /** This is sent from the local scheduler to a worker to tell the worker to + * execute a task. */ + EXECUTE_TASK, +}; + +// clang-format off +/** Contains all information that is associated to a worker. */ +typedef struct { + int sock; +} worker; +// clang-format on + +/* These are needed to define the UT_arrays. */ +UT_icd task_ptr_icd; +UT_icd worker_icd; + +/** Resources that are exposed to the scheduling algorithm. */ +typedef struct { + /** List of workers available to this node. The index into this array + * is the worker_index and is used to identify workers throughout + * the program. */ + UT_array *workers; + /* The handle to the database. */ + db_handle *db; +} scheduler_info; + +#endif /* PHOTON_H */ diff --git a/src/photon/photon_algorithm.c b/src/photon/photon_algorithm.c new file mode 100644 index 000000000..9f3f65d78 --- /dev/null +++ b/src/photon/photon_algorithm.c @@ -0,0 +1,184 @@ +#include "photon_algorithm.h" + +#include +#include "utarray.h" + +#include "state/task_log.h" +#include "photon.h" +#include "photon_scheduler.h" + +typedef struct { + /* Object id of this object. */ + object_id object_id; + /* Handle for the uthash table. */ + UT_hash_handle handle; +} available_object; + +/** Part of the photon state that is maintained by the scheduling algorithm. */ +struct scheduler_state { + /** An array of pointers to tasks that are waiting to be scheduled. */ + UT_array *task_queue; + /** An array of worker indices corresponding to clients that are + * waiting for tasks. */ + UT_array *available_workers; + /** A hash map of the objects that are available in the local Plasma store. + * This information could be a little stale. */ + available_object *local_objects; +}; + +scheduler_state *make_scheduler_state(void) { + scheduler_state *state = malloc(sizeof(scheduler_state)); + /* Initialize an empty hash map for the cache of local available objects. */ + state->local_objects = NULL; + /* Initialize the local data structures used for queuing tasks and workers. */ + utarray_new(state->task_queue, &task_ptr_icd); + utarray_new(state->available_workers, &ut_int_icd); + return state; +} + +void free_scheduler_state(scheduler_state *s) { + utarray_free(s->task_queue); + utarray_free(s->available_workers); + free(s); +} + +/** + * Check if all of the remote object arguments for a task are available in the + * local object store. + * + * @param s The scheduler state. + * @param task Task specification of the task to check. + * @return This returns 1 if all of the remote object arguments for the task are + * present in the local object store, otherwise it returns 0. + */ +bool can_run(scheduler_state *s, task_spec *task) { + int64_t num_args = task_num_args(task); + for (int i = 0; i < num_args; ++i) { + if (task_arg_type(task, i) == ARG_BY_REF) { + object_id obj_id = *task_arg_id(task, i); + available_object *entry; + HASH_FIND(handle, s->local_objects, &obj_id, sizeof(object_id), entry); + if (entry == NULL) { + /* The object is not present locally, so this task cannot be scheduled + * right now. */ + return false; + } + } + } + return true; +} + +/** + * If there is a task whose dependencies are available locally, assign it to the + * worker. This does not remove the worker from the available worker queue. + * + * @param s The scheduler state. + * @param worker_index The index of the worker. + * @return This returns 1 if it successfully assigned a task to the worker, + * otherwise it returns 0. + */ +int find_and_schedule_task_if_possible(scheduler_info *info, + scheduler_state *state, + int worker_index) { + int found_task_to_schedule = 0; + /* Find the first task whose dependencies are available locally. */ + task_spec *spec; + task_instance **task; + int i = 0; + for (; i < utarray_len(state->task_queue); ++i) { + task = (task_instance **) utarray_eltptr(state->task_queue, i); + spec = task_instance_task_spec(*task); + if (can_run(state, spec)) { + found_task_to_schedule = 1; + break; + } + } + if (found_task_to_schedule) { + /* This task's dependencies are available locally, so assign the task to the + * worker. */ + assign_task_to_worker(info, spec, worker_index); + /* Update the task queue data structure and free the task. */ + free(*task); + utarray_erase(state->task_queue, i, 1); + } + return found_task_to_schedule; +} + +void handle_task_submitted(scheduler_info *info, + scheduler_state *s, + task_spec *task) { + /* Create a unique task instance ID. This is different from the task ID and + * is used to distinguish between potentially multiple executions of the + * task. */ + task_iid task_iid = globally_unique_id(); + task_instance *instance = + make_task_instance(task_iid, task, TASK_STATUS_WAITING, NIL_ID); + /* If this task's dependencies are available locally, and if there is an + * available worker, then assign this task to an available worker. Otherwise, + * add this task to the local task queue. */ + int schedule_locally = + (utarray_len(s->available_workers) > 0) && can_run(s, task); + if (schedule_locally) { + /* Get the last available worker in the available worker queue. */ + int *worker_index = (int *) utarray_back(s->available_workers); + /* Tell the available worker to execute the task. */ + assign_task_to_worker(info, task, *worker_index); + /* Remove the available worker from the queue and free the struct. */ + utarray_pop_back(s->available_workers); + } else { + /* Add the task to the task queue. This passes ownership of the task queue. + * And the task will be freed when it is assigned to a worker. */ + utarray_push_back(s->task_queue, &instance); + } + /* Submit the task to redis. */ + task_log_add_task(info->db, instance); + if (schedule_locally) { + /* If the task was scheduled locally, we need to free it. Otherwise, + * ownership of the task is passed to the task_queue, and it will be freed + * when it is assigned to a worker. */ + free(instance); + } +} + +void handle_worker_available(scheduler_info *info, + scheduler_state *state, + int worker_index) { + int scheduled_task = + find_and_schedule_task_if_possible(info, state, worker_index); + /* If we couldn't find a task to schedule, add the worker to the queue of + * available workers. */ + if (!scheduled_task) { + for (int *p = (int *) utarray_front(state->available_workers); p != NULL; + p = (int *) utarray_next(state->available_workers, p)) { + CHECK(*p != worker_index); + } + /* Add client_sock to a list of available workers. This struct will be freed + * when a task is assigned to this worker. */ + utarray_push_back(state->available_workers, &worker_index); + LOG_INFO("Adding worker_index %d to available workers.\n", worker_index); + } +} + +void handle_object_available(scheduler_info *info, + scheduler_state *state, + object_id object_id) { + /* TODO(rkn): When does this get freed? */ + available_object *entry = + (available_object *) malloc(sizeof(available_object)); + entry->object_id = object_id; + HASH_ADD(handle, state->local_objects, object_id, sizeof(object_id), entry); + + /* Check if we can schedule any tasks. */ + int num_tasks_scheduled = 0; + for (int *p = (int *) utarray_front(state->available_workers); p != NULL; + p = (int *) utarray_next(state->available_workers, p)) { + /* Schedule a task on this worker if possible. */ + int scheduled_task = find_and_schedule_task_if_possible(info, state, *p); + if (!scheduled_task) { + /* There are no tasks we can schedule, so exit the loop. */ + break; + } + num_tasks_scheduled += 1; + } + utarray_erase(state->available_workers, 0, num_tasks_scheduled); +} diff --git a/src/photon/photon_algorithm.h b/src/photon/photon_algorithm.h new file mode 100644 index 000000000..714de8869 --- /dev/null +++ b/src/photon/photon_algorithm.h @@ -0,0 +1,88 @@ +#ifndef PHOTON_ALGORITHM_H +#define PHOTON_ALGORITHM_H + +#include "photon.h" +#include "common/task.h" + +/* ==== The scheduling algorithm ==== + * + * This file contains declaration for all functions and data structures + * that need to be provided if you want to implement a new algorithms + * for the local scheduler. + * + */ + +/** Internal state of the scheduling algorithm. */ +typedef struct scheduler_state scheduler_state; + +/** + * Initialize the scheduler state. + * + * @return Internal state of the scheduling algorithm. + */ +scheduler_state *make_scheduler_state(void); + +/** + * Free the scheduler state. + * + * @param state Internal state of the scheduling algorithm. + * @return Void. + */ +void free_scheduler_state(scheduler_state *state); + +/** + * This function will be called when a new task is submitted by a worker for + * execution. + * + * @param info Info about resources exposed by photon to the scheduling + * algorithm. + * @param state State of the scheduling algorithm. + * @param task Task that is submitted by the worker. + * @return Void. + */ +void handle_task_submitted(scheduler_info *info, + scheduler_state *state, + task_spec *task); + +/** + * This function will be called when a task is assigned by the global scheduler + * for execution on this local scheduler. + * + * @param info Info about resources exposed by photon to the scheduling + * algorithm. + * @param state State of the scheduling algorithm. + * @param task Task that is assigned by the global scheduler. + * @return Void. + */ +void handle_task_assigned(scheduler_info *info, + scheduler_state *state, + task_spec *task); + +/** + * This function is called if a new object becomes available in the local + * plasma store. + * + * @param info Info about resources exposed by photon to the scheduling + * algorithm. + * @param state State of the scheduling algorithm. + * @param object_id ID of the object that became available. + * @return Void. + */ +void handle_object_available(scheduler_info *info, + scheduler_state *state, + object_id object_id); + +/** + * This function is called when a new worker becomes available + * + * @param info Info about resources exposed by photon to the scheduling + * algorithm. + * @param state State of the scheduling algorithm. + * @param worker_index The index of the worker that becomes available. + * @return Void. + */ +void handle_worker_available(scheduler_info *info, + scheduler_state *state, + int worker_index); + +#endif /* PHOTON_ALGORITHM_H */ diff --git a/src/photon/photon_client.c b/src/photon/photon_client.c new file mode 100644 index 000000000..1bf87f491 --- /dev/null +++ b/src/photon/photon_client.c @@ -0,0 +1,41 @@ +#include "photon_client.h" + +#include "common/io.h" +#include "common/task.h" +#include + +photon_conn *photon_connect(const char *photon_socket) { + photon_conn *result = malloc(sizeof(photon_conn)); + result->conn = connect_ipc_sock(photon_socket); + return result; +} + +void photon_submit(photon_conn *conn, task_spec *task) { + write_message(conn->conn, SUBMIT_TASK, task_size(task), (uint8_t *)task); +} + +task_spec *photon_get_task(photon_conn *conn) { + write_message(conn->conn, GET_TASK, 0, NULL); + int64_t type; + int64_t length; + uint8_t *message; + /* Receive a task from the local scheduler. This will block until the local + * scheduler gives this client a task. */ + read_message(conn->conn, &type, &length, &message); + CHECK(type == EXECUTE_TASK); + task_spec *task = (task_spec *)message; + CHECK(length == task_size(task)); + return task; +} + +void photon_task_done(photon_conn *conn) { + write_message(conn->conn, TASK_DONE, 0, NULL); +} + +void photon_disconnect(photon_conn *conn) { + write_message(conn->conn, DISCONNECT_CLIENT, 0, NULL); +} + +void photon_log_message(photon_conn *conn) { + write_message(conn->conn, LOG_MESSAGE, 0, NULL); +} diff --git a/src/photon/photon_client.h b/src/photon/photon_client.h new file mode 100644 index 000000000..76b09455c --- /dev/null +++ b/src/photon/photon_client.h @@ -0,0 +1,66 @@ +#ifndef PHOTON_CLIENT_H +#define PHOTON_CLIENT_H + +#include "common/task.h" +#include "photon.h" + +typedef struct { + /* File descriptor of the Unix domain socket that connects to photon. */ + int conn; +} photon_conn; + +/** + * Connect to the local scheduler. + * + * @param photon_socket The name of the socket to use to connect to the local + scheduler. + * @return The connection information. + */ +photon_conn *photon_connect(const char *photon_socket); + +/** + * Submit a task to the local scheduler. + * + * @param conn The connection information. + * @param task The address of the task to submit. + * @return Void. + */ +void photon_submit(photon_conn *conn, task_spec *task); + +/** + * Get next task for this client. This will block until the scheduler assigns + * a task to this worker. This allocates and returns a task, and so the task + * must be freed by the caller. + * + * @todo When does this actually get freed? + * + * @param conn The connection information. + * @return The address of the assigned task. + */ +task_spec *photon_get_task(photon_conn *conn); + +/** + * Tell the local scheduler that the client has finished executing a task. + * + * @param conn The connection information. + * @return Void. + */ +void photon_task_done(photon_conn *conn); + +/** + * Disconnect from the local scheduler. + * + * @param conn The connection information. + * @return Void. + */ +void photon_disconnect(photon_conn *conn); + +/** + * Send a log message to the local scheduler. + * + * @param conn The connection information. + * @return Void. + */ +void photon_log_message(photon_conn *conn); + +#endif diff --git a/src/photon/photon_scheduler.c b/src/photon/photon_scheduler.c new file mode 100644 index 000000000..bdd18857d --- /dev/null +++ b/src/photon/photon_scheduler.c @@ -0,0 +1,228 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "event_loop.h" +#include "io.h" +#include "photon.h" +#include "photon_algorithm.h" +#include "photon_scheduler.h" +#include "plasma_client.h" +#include "state/db.h" +#include "state/task_log.h" +#include "utarray.h" +#include "uthash.h" + +UT_icd task_ptr_icd = {sizeof(task_instance *), NULL, NULL, NULL}; +UT_icd worker_icd = {sizeof(worker), NULL, NULL, NULL}; + +/** Association between the socket fd of a worker and its worker_index. */ +typedef struct { + /** The socket fd of a worker. */ + int sock; + /** The index of the worker in scheduler_info->workers. */ + int64_t worker_index; + /** Handle for the hash table. */ + UT_hash_handle hh; +} worker_index; + +struct local_scheduler_state { + /* The local scheduler event loop. */ + event_loop *loop; + /* The Plasma client. */ + plasma_store_conn *plasma_conn; + /* Association between client socket and worker index. */ + worker_index *worker_index; + /* Info that is exposed to the scheduling algorithm. */ + scheduler_info *scheduler_info; + /* State for the scheduling algorithm. */ + scheduler_state *scheduler_state; +}; + +local_scheduler_state *init_local_scheduler(event_loop *loop, + const char *redis_addr, + int redis_port, + const char *plasma_socket_name) { + local_scheduler_state *state = malloc(sizeof(local_scheduler_state)); + state->loop = loop; + /* Connect to Plasma. This method will retry if Plasma hasn't started yet. */ + state->plasma_conn = plasma_store_connect(plasma_socket_name); + /* Subscribe to notifications about sealed objects. */ + int plasma_fd = plasma_subscribe(state->plasma_conn); + /* Add the callback that processes the notification to the event loop. */ + event_loop_add_file(loop, plasma_fd, EVENT_LOOP_READ, + process_plasma_notification, state); + state->worker_index = NULL; + /* Add scheduler info. */ + state->scheduler_info = malloc(sizeof(scheduler_info)); + utarray_new(state->scheduler_info->workers, &worker_icd); + /* Connect to Redis. */ + state->scheduler_info->db = + db_connect(redis_addr, redis_port, "photon", "", -1); + db_attach(state->scheduler_info->db, loop); + /* Add scheduler state. */ + state->scheduler_state = make_scheduler_state(); + return state; +}; + +void free_local_scheduler(local_scheduler_state *s) { + db_disconnect(s->scheduler_info->db); + free(s->scheduler_info); + free_scheduler_state(s->scheduler_state); + event_loop_destroy(s->loop); + free(s); +} + +void assign_task_to_worker(scheduler_info *info, + task_spec *task, + int worker_index) { + CHECK(worker_index < utarray_len(info->workers)); + worker *w = (worker *) utarray_eltptr(info->workers, worker_index); + write_message(w->sock, EXECUTE_TASK, task_size(task), (uint8_t *) task); +} + +void process_plasma_notification(event_loop *loop, + int client_sock, + void *context, + int events) { + local_scheduler_state *s = context; + /* Read the notification from Plasma. */ + uint8_t *message = (uint8_t *) malloc(sizeof(object_id)); + recv(client_sock, message, sizeof(object_id), 0); + object_id *obj_id = (object_id *) message; + handle_object_available(s->scheduler_info, s->scheduler_state, *obj_id); +} + +void process_message(event_loop *loop, int client_sock, void *context, + int events) { + local_scheduler_state *s = context; + + uint8_t *message; + int64_t type; + int64_t length; + read_message(client_sock, &type, &length, &message); + + LOG_DEBUG("New event of type %" PRId64, type); + + switch (type) { + case SUBMIT_TASK: { + task_spec *spec = (task_spec *) message; + CHECK(task_size(spec) == length); + handle_task_submitted(s->scheduler_info, s->scheduler_state, spec); + } break; + case TASK_DONE: { + } break; + case GET_TASK: { + worker_index *wi; + HASH_FIND_INT(s->worker_index, &client_sock, wi); + printf("worker_index is %" PRId64 "\n", wi->worker_index); + handle_worker_available(s->scheduler_info, s->scheduler_state, + wi->worker_index); + } break; + case DISCONNECT_CLIENT: { + LOG_INFO("Disconnecting client on fd %d", client_sock); + event_loop_remove_file(loop, client_sock); + } break; + case LOG_MESSAGE: { + } break; + default: + /* This code should be unreachable. */ + CHECK(0); + } + free(message); +} + +void new_client_connection(event_loop *loop, int listener_sock, void *context, + int events) { + local_scheduler_state *s = context; + int new_socket = accept_client(listener_sock); + event_loop_add_file(loop, new_socket, EVENT_LOOP_READ, process_message, s); + LOG_INFO("new connection with fd %d", new_socket); + /* Add worker to list of workers. */ + /* TODO(pcm): Where shall we free this? */ + worker_index *new_worker_index = malloc(sizeof(worker_index)); + new_worker_index->sock = new_socket; + new_worker_index->worker_index = utarray_len(s->scheduler_info->workers); + HASH_ADD_INT(s->worker_index, sock, new_worker_index); + worker worker = {.sock = new_socket}; + utarray_push_back(s->scheduler_info->workers, &worker); +} + +/* We need this code so we can clean up when we get a SIGTERM signal. */ + +local_scheduler_state *g_state; + +void signal_handler(int signal) { + if (signal == SIGTERM) { + free_local_scheduler(g_state); + exit(0); + } +} + +/* End of the cleanup code. */ + +void start_server(const char *socket_name, + const char *redis_addr, + int redis_port, + const char *plasma_socket_name) { + int fd = bind_ipc_sock(socket_name); + event_loop *loop = event_loop_create(); + g_state = + init_local_scheduler(loop, redis_addr, redis_port, plasma_socket_name); + + /* Run event loop. */ + event_loop_add_file(loop, fd, EVENT_LOOP_READ, new_client_connection, + g_state); + event_loop_run(loop); +} + +int main(int argc, char *argv[]) { + signal(SIGTERM, signal_handler); + /* Path of the listening socket of the local scheduler. */ + char *scheduler_socket_name = NULL; + /* IP address and port of redis. */ + char *redis_addr_port = NULL; + /* Socket name for the local Plasma store. */ + char *plasma_socket_name = NULL; + int c; + while ((c = getopt(argc, argv, "s:r:p:")) != -1) { + switch (c) { + case 's': + scheduler_socket_name = optarg; + break; + case 'r': + redis_addr_port = optarg; + break; + case 'p': + plasma_socket_name = optarg; + break; + default: + LOG_ERR("unknown option %c", c); + exit(-1); + } + } + if (!scheduler_socket_name) { + LOG_ERR("please specify socket for incoming connections with -s switch"); + exit(-1); + } + if (!plasma_socket_name) { + LOG_ERR("please specify socket for connecting to Plasma with -p switch"); + exit(-1); + } + /* Parse the Redis address into an IP address and a port. */ + char redis_addr[16] = {0}; + char redis_port[6] = {0}; + if (!redis_addr_port || + sscanf(redis_addr_port, "%15[0-9.]:%5[0-9]", redis_addr, redis_port) != + 2) { + LOG_ERR("need to specify redis address like 127.0.0.1:6379 with -r switch"); + exit(-1); + } + start_server(scheduler_socket_name, &redis_addr[0], atoi(redis_port), + plasma_socket_name); +} diff --git a/src/photon/photon_scheduler.h b/src/photon/photon_scheduler.h new file mode 100644 index 000000000..3ebb03432 --- /dev/null +++ b/src/photon/photon_scheduler.h @@ -0,0 +1,52 @@ +#ifndef PHOTON_SCHEDULER_H +#define PHOTON_SCHEDULER_H + +#include "task.h" +#include "event_loop.h" + +typedef struct local_scheduler_state local_scheduler_state; + +/** + * Establish a connection to a new client. + * + * @param loop Event loop of the local scheduler. + * @param listener_socket Socket the local scheduler is listening on for new + * client requests. + * @param context State of the local scheduler. + * @param events Flag for events that are available on the listener socket. + * @return Void. + */ +void new_client_connection(event_loop *loop, + int listener_sock, + void *context, + int events); + +/** + * This function can be called by the scheduling algorithm to assign a task + * to a worker. + * + * @param info + * @param task The task that is submitted to the worker. + * @param worker_index The index of the worker the task is submitted to. + * @return Void. + */ +void assign_task_to_worker(scheduler_info *info, + task_spec *task, + int worker_index); + +/** + * This is the callback that is used to process a notification from the Plasma + * store that an object has been sealed. + * + * @param loop The local scheduler's event loop. + * @param client_sock The file descriptor to read the notification from. + * @param context The local scheduler state. + * @param events + * @return Void. + */ +void process_plasma_notification(event_loop *loop, + int client_sock, + void *context, + int events); + +#endif /* PHOTON_SCHEDULER_H */ diff --git a/src/photon/test/test.py b/src/photon/test/test.py new file mode 100644 index 000000000..fb35702ad --- /dev/null +++ b/src/photon/test/test.py @@ -0,0 +1,151 @@ +from __future__ import print_function + +import os +import signal +import subprocess +import sys +import unittest +import random +import threading +import time + +import photon +import plasma + +USE_VALGRIND = False + +class TestPhotonClient(unittest.TestCase): + + def setUp(self): + # Start Redis. + redis_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../common/thirdparty/redis-3.2.3/src/redis-server") + self.p1 = subprocess.Popen([redis_executable, "--loglevel", "warning"]) + # Start Plasma. + plasma_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../plasma/build/plasma_store") + plasma_socket = "/tmp/plasma_store{}".format(random.randint(0, 10000)) + self.p2 = subprocess.Popen([plasma_executable, "-s", plasma_socket]) + time.sleep(0.1) + self.plasma_client = plasma.PlasmaClient(plasma_socket) + scheduler_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../build/photon_scheduler") + scheduler_name = "/tmp/scheduler{}".format(random.randint(0, 10000)) + command = [scheduler_executable, "-s", scheduler_name, "-r", "127.0.0.1:6379", "-p", plasma_socket] + if USE_VALGRIND: + self.p3 = subprocess.Popen(["valgrind", "--track-origins=yes", "--leak-check=full", "--show-leak-kinds=all"] + command) + else: + self.p3 = subprocess.Popen(command) + if USE_VALGRIND: + time.sleep(1.0) + else: + time.sleep(0.1) + # Connect to the scheduler. + self.photon_client = photon.PhotonClient(scheduler_name) + + def tearDown(self): + # Kill the Redis server. + self.p1.kill() + # Kill Plasma. + self.p2.kill() + # Kill the local scheduler. + if USE_VALGRIND: + self.p3.send_signal(signal.SIGTERM) + self.p3.wait() + os._exit(self.p3.returncode) + else: + self.p3.kill() + + def test_submit_and_get_task(self): + # TODO(rkn): This should be a FunctionID. + function_id = photon.ObjectID(20 * "a") + object_ids = [photon.ObjectID(20 * chr(i)) for i in range(256)] + # Create and seal the objects in the object store so that we can schedule + # all of the subsequent tasks. + for object_id in object_ids: + self.plasma_client.create(object_id.id(), 0) + self.plasma_client.seal(object_id.id()) + # Define some arguments to use for the tasks. + args_list = [ + [], + #{}, + #(), + 1 * [1], + 10 * [1], + 100 * [1], + 1000 * [1], + 1 * ["a"], + 10 * ["a"], + 100 * ["a"], + 1000 * ["a"], + [1, 1.3, 2L, 1L << 100, "hi", u"hi", [1, 2]], + object_ids[:1], + object_ids[:2], + object_ids[:3], + object_ids[:4], + object_ids[:5], + object_ids[:10], + object_ids[:100], + object_ids[:256], + [1, object_ids[0]], + [object_ids[0], "a"], + [1, object_ids[0], "a"], + [object_ids[0], 1, object_ids[1], "a"], + object_ids[:3] + [1, "hi", 2.3] + object_ids[:5], + object_ids + 100 * ["a"] + object_ids + ] + + for args in args_list: + for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: + task = photon.Task(function_id, args, num_return_vals) + # Submit a task. + self.photon_client.submit(task) + # Get the task. + new_task = self.photon_client.get_task() + self.assertEqual(task.function_id().id(), new_task.function_id().id()) + retrieved_args = new_task.arguments() + returns = new_task.returns() + self.assertEqual(len(args), len(retrieved_args)) + self.assertEqual(num_return_vals, len(returns)) + for i in range(len(retrieved_args)): + if isinstance(args[i], photon.ObjectID): + self.assertEqual(args[i].id(), retrieved_args[i].id()) + else: + self.assertEqual(args[i], retrieved_args[i]) + + # Submit all of the tasks. + for args in args_list: + for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: + task = photon.Task(function_id, args, num_return_vals) + self.photon_client.submit(task) + # Get all of the tasks. + for args in args_list: + for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: + new_task = self.photon_client.get_task() + + def test_scheduling_when_objects_ready(self): + # Create a task and submit it. + object_id = photon.ObjectID(20 * chr(0)) + # TODO(rkn): This should be a FunctionID. + function_id = photon.ObjectID(20 * "a") + task = photon.Task(function_id, [object_id], 0) + self.photon_client.submit(task) + # Launch a thread to get the task. + def get_task(): + self.photon_client.get_task() + t = threading.Thread(target=get_task) + t.start() + # Sleep to give the thread time to call get_task. + time.sleep(0.1) + # Create and seal the object ID in the object store. This should trigger a + # scheduling event. + self.plasma_client.create(object_id.id(), 0) + self.plasma_client.seal(object_id.id()) + # Wait until the thread finishes so that we know the task was scheduled. + t.join() + +if __name__ == "__main__": + if len(sys.argv) > 1: + # pop the argument so we don't mess with unittest's own argument parser + arg = sys.argv.pop() + if arg == "valgrind": + USE_VALGRIND = True + print("Using valgrind for tests") + unittest.main(verbosity=2) diff --git a/src/plasma/Makefile b/src/plasma/Makefile new file mode 100644 index 000000000..0ad78eb74 --- /dev/null +++ b/src/plasma/Makefile @@ -0,0 +1,40 @@ +CC = gcc +CFLAGS = -g -Wall --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -I. -Icommon -Icommon/thirdparty +BUILD = build + +all: $(BUILD)/plasma_store $(BUILD)/plasma_manager $(BUILD)/plasma_client.so $(BUILD)/example $(BUILD)/libplasma_client.a + +debug: FORCE +debug: CFLAGS += -DRAY_COMMON_DEBUG=1 +debug: all + +clean: + cd common; make clean + rm -r $(BUILD)/* + +$(BUILD)/plasma_store: src/plasma_store.c src/plasma.h src/fling.h src/fling.c src/malloc.c src/malloc.h thirdparty/dlmalloc.c common + $(CC) $(CFLAGS) src/plasma_store.c src/fling.c src/malloc.c common/build/libcommon.a -o $(BUILD)/plasma_store + +$(BUILD)/plasma_manager: src/plasma_manager.c src/plasma.h src/plasma_client.c src/fling.h src/fling.c common + $(CC) $(CFLAGS) src/plasma_manager.c src/plasma_client.c src/fling.c common/build/libcommon.a common/thirdparty/hiredis/libhiredis.a -o $(BUILD)/plasma_manager + +$(BUILD)/plasma_client.so: src/plasma_client.c src/fling.h src/fling.c common + $(CC) $(CFLAGS) src/plasma_client.c src/fling.c common/build/libcommon.a -fPIC -shared -o $(BUILD)/plasma_client.so + +$(BUILD)/libplasma_client.a: src/plasma_client.o src/fling.o + ar rcs $@ $^ + +$(BUILD)/example: src/plasma_client.c src/plasma.h src/example.c src/fling.h src/fling.c common + $(CC) $(CFLAGS) src/plasma_client.c src/example.c src/fling.c common/build/libcommon.a -o $(BUILD)/example + +common: FORCE + git submodule update --init --recursive + cd common; make + +# Set the request timeout low for testing purposes. +test: CFLAGS += -DRAY_TIMEOUT=50 +test: FORCE + cd common; make redis +test: all + +FORCE: diff --git a/src/plasma/build/.gitkeep b/src/plasma/build/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/src/plasma/doc/plasma-doxy-config b/src/plasma/doc/plasma-doxy-config new file mode 100644 index 000000000..9c291f838 --- /dev/null +++ b/src/plasma/doc/plasma-doxy-config @@ -0,0 +1,2473 @@ +# Doxyfile 1.8.13 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all text +# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv +# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv +# for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Plasma" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = NO + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = YES + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 2 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines. + +ALIASES = + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding "class=itcl::class" +# will allow you to use the command class in the itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, Javascript, +# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: +# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: +# Fortran. In the later case the parser tries to guess whether the code is fixed +# or free formatted code, this is the default for Fortran type files), VHDL. For +# instance to make doxygen treat .inc files as Fortran files (default is PHP), +# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 0. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 0 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = NO + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# (class|struct|union) declarations. If set to NO, these declarations will be +# included in the documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES, upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. +# The default value is: system dependent. + +CASE_SENSE_NAMES = NO + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = YES + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT = ../src + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: http://www.gnu.org/software/libiconv) for the list of +# possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. + +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.pyw \ + *.f90 \ + *.f95 \ + *.f03 \ + *.f08 \ + *.f \ + *.for \ + *.tcl \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = ../src/utarray.h ../src/uthash.h + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = * + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# +# +# where is the value of the INPUT_FILTER tag, and is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# function all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see http://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in +# which the alphabetical index list will be split. +# Minimum value: 1, maximum value: 20, default value: 5. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = NO + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: http://developer.apple.com/tools/xcode/), introduced with +# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a +# Makefile in the HTML output directory. Running make will produce the docset in +# that directory and running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the master .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- +# folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location of Qt's +# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the +# generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = NO + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# http://www.mathjax.org) which uses client side Javascript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from http://www.mathjax.org before deployment. +# The default value is: http://cdn.mathjax.org/mathjax/latest. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use + S +# (what the is depends on the OS and browser, but it is typically +# , /