diff --git a/src/common/Makefile b/src/common/Makefile new file mode 100644 index 000000000..6982b7945 --- /dev/null +++ b/src/common/Makefile @@ -0,0 +1,46 @@ +CC = gcc +CFLAGS = -g -Wall --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -fPIC -I. -Ithirdparty -Ithirdparty/ae -Wno-typedef-redefinition -Werror +BUILD = build + +all: hiredis $(BUILD)/libcommon.a + +$(BUILD)/libcommon.a: event_loop.o common.o task.o io.o state/redis.o thirdparty/ae/ae.o + ar rcs $@ $^ + +$(BUILD)/common_tests: test/common_tests.c $(BUILD)/libcommon.a + $(CC) -o $@ test/common_tests.c $(BUILD)/libcommon.a $(CFLAGS) + +$(BUILD)/db_tests: hiredis test/db_tests.c $(BUILD)/libcommon.a + $(CC) -o $@ test/db_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS) + +$(BUILD)/io_tests: test/io_tests.c $(BUILD)/libcommon.a + $(CC) -o $@ $^ $(CFLAGS) + +$(BUILD)/task_tests: test/task_tests.c $(BUILD)/libcommon.a + $(CC) -o $@ $^ $(CFLAGS) + +$(BUILD)/redis_tests: hiredis test/redis_tests.c $(BUILD)/libcommon.a logging.h + $(CC) -o $@ test/redis_tests.c logging.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS) + +clean: + rm -f *.o state/*.o test/*.o thirdparty/ae/*.o + rm -rf $(BUILD)/* + +redis: + cd thirdparty ; bash ./build-redis.sh + +hiredis: + git submodule update --init --recursive -- "thirdparty/hiredis" ; cd thirdparty/hiredis ; make + +test: hiredis redis $(BUILD)/common_tests $(BUILD)/db_tests $(BUILD)/io_tests $(BUILD)/task_tests $(BUILD)/redis_tests FORCE + ./thirdparty/redis-3.2.3/src/redis-server & + sleep 1s ; ./build/common_tests ; ./build/db_tests ; ./build/io_tests ; ./build/task_tests ; ./build/redis_tests + +valgrind: test + valgrind --leak-check=full --error-exitcode=1 ./build/common_tests + valgrind --leak-check=full --error-exitcode=1 ./build/db_tests + valgrind --leak-check=full --error-exitcode=1 ./build/io_tests + valgrind --leak-check=full --error-exitcode=1 ./build/task_tests + valgrind --leak-check=full --error-exitcode=1 ./build/redis_tests + +FORCE: diff --git a/src/common/build/.gitkeep b/src/common/build/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/src/common/common.c b/src/common/common.c new file mode 100644 index 000000000..d9fe1e951 --- /dev/null +++ b/src/common/common.c @@ -0,0 +1,36 @@ +#include "common.h" + +#include +#include +#include +#include +#include + +const unique_id NIL_ID = {{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}}; + +unique_id globally_unique_id(void) { + /* Use /dev/urandom for "real" randomness. */ + int fd; + if ((fd = open("/dev/urandom", O_RDONLY)) == -1) { + LOG_ERR("Could not generate random number"); + } + unique_id result; + read(fd, &result.id[0], UNIQUE_ID_SIZE); + close(fd); + return result; +} + +char *sha1_to_hex(const unsigned char *sha1, char *buffer) { + static const char hex[] = "0123456789abcdef"; + char *buf = buffer; + + for (int i = 0; i < UNIQUE_ID_SIZE; i++) { + unsigned int val = *sha1++; + *buf++ = hex[val >> 4]; + *buf++ = hex[val & 0xf]; + } + *buf = '\0'; + + return buffer; +} diff --git a/src/common/common.h b/src/common/common.h new file mode 100644 index 000000000..f09e91cd5 --- /dev/null +++ b/src/common/common.h @@ -0,0 +1,61 @@ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include + +#ifndef RAY_COMMON_DEBUG +#define LOG_DEBUG(M, ...) +#else +#define LOG_DEBUG(M, ...) \ + fprintf(stderr, "[DEBUG] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) +#endif + +#define LOG_ERR(M, ...) \ + fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, \ + errno == 0 ? "None" : strerror(errno), ##__VA_ARGS__) + +#define LOG_INFO(M, ...) \ + fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) + +#define CHECK(COND) \ + do { \ + if (!(COND)) { \ + LOG_ERR("Check failure: %s", #COND); \ + exit(-1); \ + } \ + } while (0); + +#define CHECKM(COND, M, ...) \ + do { \ + if (!(COND)) { \ + LOG_ERR("Check failure: %s \n" M, #COND, ##__VA_ARGS__); \ + exit(-1); \ + } \ + } while (0); + +#define UNIQUE_ID_SIZE 20 + +/* Cleanup method for running tests with the greatest library. + * Runs the test, then clears the Redis database. */ +#define RUN_REDIS_TEST(context, test) \ + RUN_TEST(test); \ + freeReplyObject(redisCommand(context, "FLUSHALL")); + +typedef struct { unsigned char id[UNIQUE_ID_SIZE]; } unique_id; + +extern const unique_id NIL_ID; + +/* Generate a globally unique ID. */ +unique_id globally_unique_id(void); + +/* Convert a 20 byte sha1 hash to a hexdecimal string. This function assumes + * that buffer points to an already allocated char array of size 2 * + * UNIQUE_ID_SIZE + 1 */ +char *sha1_to_hex(const unsigned char *sha1, char *buffer); + +typedef unique_id object_id; + +#endif diff --git a/src/common/doc/tasks.md b/src/common/doc/tasks.md new file mode 100644 index 000000000..b56f89783 --- /dev/null +++ b/src/common/doc/tasks.md @@ -0,0 +1,33 @@ +# Task specifications, task instances and task logs + +A *task specification* contains all information that is needed for computing +the results of a task: + +- The function ID of the function that executes the task +- The arguments (either object IDs for pass by reference +or values for pass by value) +- The IDs of the result objects + +From these, a task ID can be computed which is also stored in the task +specification. + +A *task instance* represents one execution of a task specification. +It consists of: + +- A scheduling state (WAITING, SCHEDULED, RUNNING, DONE) +- The target node where the task is scheduled or executed +- A unique task instance ID that identifies the particular execution + of the task. + +The task data structures are defined in `common/task.h`. + +The *task log* is a mapping from the task instance ID to a sequence of +updates to the status of the task instance. It is updated by various parts +of the system: + +1. The local scheduler writes it with status WAITING when submits a task to the global scheduler +2. The global scheduler appends an update WAITING -> SCHEDULED together with the node ID when assigning the task to a local scheduler +3. The local scheduler appends an update SCHEDULED -> RUNNING when it assigns a task to a worker +4. The local scheduler appends an update RUNNING -> DONE when the task finishes execution + +The task log is defined in `common/state/task_log.h`. diff --git a/src/common/event_loop.c b/src/common/event_loop.c new file mode 100644 index 000000000..7e7835445 --- /dev/null +++ b/src/common/event_loop.c @@ -0,0 +1,62 @@ +#include "event_loop.h" + +#include "common.h" +#include + +#define INITIAL_EVENT_LOOP_SIZE 1024 + +event_loop *event_loop_create() { + return aeCreateEventLoop(INITIAL_EVENT_LOOP_SIZE); +} + +void event_loop_destroy(event_loop *loop) { + /* Clean up timer events. This is to make valgrind happy. */ + aeTimeEvent *te = loop->timeEventHead; + while (te) { + aeTimeEvent *next = te->next; + free(te); + te = next; + } + aeDeleteEventLoop(loop); +} + +void event_loop_add_file(event_loop *loop, + int fd, + int events, + event_loop_file_handler handler, + void *context) { + /* Try to add the file descriptor. */ + int err = aeCreateFileEvent(loop, fd, events, handler, context); + /* If it cannot be added, increase the size of the event loop. */ + if (err == AE_ERR && errno == ERANGE) { + err = aeResizeSetSize(loop, 3 * aeGetSetSize(loop) / 2); + CHECK(err == AE_OK); + err = aeCreateFileEvent(loop, fd, events, handler, context); + } + /* In any case, test if there were errors. */ + CHECK(err == AE_OK); +} + +void event_loop_remove_file(event_loop *loop, int fd) { + aeDeleteFileEvent(loop, fd, EVENT_LOOP_READ | EVENT_LOOP_WRITE); +} + +int64_t event_loop_add_timer(event_loop *loop, + int64_t milliseconds, + event_loop_timer_handler handler, + void *context) { + return aeCreateTimeEvent(loop, milliseconds, handler, context, NULL); +} + +void event_loop_remove_timer(event_loop *loop, timer_id timer_id) { + int err = aeDeleteTimeEvent(loop, timer_id); + CHECK(err == AE_OK); /* timer id found? */ +} + +void event_loop_run(event_loop *loop) { + aeMain(loop); +} + +void event_loop_stop(event_loop *loop) { + aeStop(loop); +} diff --git a/src/common/event_loop.h b/src/common/event_loop.h new file mode 100644 index 000000000..7f0659ba7 --- /dev/null +++ b/src/common/event_loop.h @@ -0,0 +1,77 @@ +#ifndef EVENT_LOOP_H +#define EVENT_LOOP_H + +#include +#include "ae/ae.h" + +typedef long long timer_id; + +typedef aeEventLoop event_loop; + +/* File descriptor is readable. */ +#define EVENT_LOOP_READ AE_READABLE + +/* File descriptor is writable. */ +#define EVENT_LOOP_WRITE AE_WRITABLE + +/* Constant specifying that the timer is done and it will be removed. */ +#define EVENT_LOOP_TIMER_DONE AE_NOMORE + +/* Signature of the handler that will be called when there is a new event + * on the file descriptor that this handler has been registered for. The + * context is the one that was passed into add_file by the user. The + * events parameter indicates which event is available on the file, + * it can be EVENT_LOOP_READ or EVENT_LOOP_WRITE. */ +typedef void (*event_loop_file_handler)(event_loop *loop, + int fd, + void *context, + int events); + +/* This handler will be called when a timer times out. The id of the timer + * as well as the context that was specified when registering this handler + * are passed as arguments. The return is the number of milliseconds the + * timer shall be reset to or EVENT_LOOP_TIMER_DONE if the timer shall + * not be triggered again. */ +typedef int (*event_loop_timer_handler)(event_loop *loop, + timer_id timer_id, + void *context); + +/* Create and return a new event loop. */ +event_loop *event_loop_create(); + +/* Deallocate space associated with the event loop that was created + * with the "create" function. */ +void event_loop_destroy(event_loop *loop); + +/* Register a handler that will be called any time a new event happens on + * a file descriptor. Can specify a context that will be passed as an + * argument to the handler. Currently there can only be one handler per file. + * The events parameter specifies which events we listen to: EVENT_LOOP_READ + * or EVENT_LOOP_WRITE. */ +void event_loop_add_file(event_loop *loop, + int fd, + int events, + event_loop_file_handler handler, + void *context); + +/* Remove a registered file event handler from the event loop. */ +void event_loop_remove_file(event_loop *loop, int fd); + +/* Register a handler that will be called after a time slice of + * "milliseconds" milliseconds. Can specify a context that will be passed + * as an argument to the handler. Return the id of the time event. */ +int64_t event_loop_add_timer(event_loop *loop, + int64_t milliseconds, + event_loop_timer_handler handler, + void *context); + +/* Remove a registered time event handler from the event loop. */ +void event_loop_remove_timer(event_loop *loop, timer_id timer_id); + +/* Run the event loop. */ +void event_loop_run(event_loop *loop); + +/* Stop the event loop. */ +void event_loop_stop(event_loop *loop); + +#endif diff --git a/src/common/io.c b/src/common/io.c new file mode 100644 index 000000000..1f1125bc3 --- /dev/null +++ b/src/common/io.c @@ -0,0 +1,333 @@ +#include "io.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" + +/** + * Binds to an Internet socket at the given port. Removes any existing file at + * the pathname. Returns a non-blocking file descriptor for the socket, or -1 + * if an error occurred. + * + * @note Since the returned file descriptor is non-blocking, it is not + * recommended to use the Linux read and write calls directly, since these + * might read or write a partial message. Instead, use the provided + * write_message and read_message methods. + * + * @param port The port to bind to. + * @return A non-blocking file descriptor for the socket, or -1 if an error + * occurs. + */ +int bind_inet_sock(const int port) { + struct sockaddr_in name; + int socket_fd = socket(PF_INET, SOCK_STREAM, 0); + if (socket_fd < 0) { + LOG_ERR("socket() failed for port %d.", port); + return -1; + } + name.sin_family = AF_INET; + name.sin_port = htons(port); + name.sin_addr.s_addr = htonl(INADDR_ANY); + int on = 1; + /* TODO(pcm): http://stackoverflow.com/q/1150635 */ + if (ioctl(socket_fd, FIONBIO, (char *) &on) < 0) { + LOG_ERR("ioctl failed"); + close(socket_fd); + return -1; + } + if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0) { + LOG_ERR("setsockopt failed for port %d", port); + close(socket_fd); + return -1; + } + if (bind(socket_fd, (struct sockaddr *) &name, sizeof(name)) < 0) { + LOG_ERR("Bind failed for port %d", port); + close(socket_fd); + return -1; + } + if (listen(socket_fd, 5) == -1) { + LOG_ERR("Could not listen to socket %d", port); + close(socket_fd); + return -1; + } + return socket_fd; +} + +/** + * Binds to a Unix domain streaming socket at the given + * pathname. Removes any existing file at the pathname. + * + * @param socket_pathname The pathname for the socket. + * @return A blocking file descriptor for the socket, or -1 if an error + * occurs. + */ +int bind_ipc_sock(const char *socket_pathname) { + struct sockaddr_un socket_address; + int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (socket_fd < 0) { + LOG_ERR("socket() failed for pathname %s.", socket_pathname); + return -1; + } + /* Tell the system to allow the port to be reused. */ + int on = 1; + if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, + sizeof(on)) < 0) { + LOG_ERR("setsockopt failed for pathname %s", socket_pathname); + close(socket_fd); + return -1; + } + + unlink(socket_pathname); + memset(&socket_address, 0, sizeof(struct sockaddr_un)); + socket_address.sun_family = AF_UNIX; + if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) { + LOG_ERR("Socket pathname is too long."); + close(socket_fd); + return -1; + } + strncpy(socket_address.sun_path, socket_pathname, + strlen(socket_pathname) + 1); + + if (bind(socket_fd, (struct sockaddr *) &socket_address, + sizeof(struct sockaddr_un)) != 0) { + LOG_ERR("Bind failed for pathname %s.", socket_pathname); + close(socket_fd); + return -1; + } + if (listen(socket_fd, 5) == -1) { + LOG_ERR("Could not listen to socket %s", socket_pathname); + close(socket_fd); + return -1; + } + return socket_fd; +} + +/** + * Connects to a Unix domain streaming socket at the given + * pathname. Returns a file descriptor for the socket, or -1 if + * an error occurred. + */ +int connect_ipc_sock(const char *socket_pathname) { + struct sockaddr_un socket_address; + int socket_fd; + + socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (socket_fd < 0) { + LOG_ERR("socket() failed for pathname %s.", socket_pathname); + return -1; + } + + memset(&socket_address, 0, sizeof(struct sockaddr_un)); + socket_address.sun_family = AF_UNIX; + if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) { + LOG_ERR("Socket pathname is too long."); + return -1; + } + strncpy(socket_address.sun_path, socket_pathname, + strlen(socket_pathname) + 1); + + if (connect(socket_fd, (struct sockaddr *) &socket_address, + sizeof(struct sockaddr_un)) != 0) { + LOG_ERR("Connection to socket failed for pathname %s.", socket_pathname); + return -1; + } + + return socket_fd; +} + +/** + * Accept a new client connection on the given socket + * descriptor. Returns a descriptor for the new socket. + */ +int accept_client(int socket_fd) { + int client_fd = accept(socket_fd, NULL, NULL); + if (client_fd < 0) { + LOG_ERR("Error reading from socket."); + return -1; + } + return client_fd; +} + +/** + * Write a sequence of bytes into a file descriptor. This will block until one + * of the following happens: (1) there is an error (2) end of file, or (3) all + * length bytes have been written. + * + * @param fd The file descriptor to write to. It can be non-blocking. + * @param cursor The cursor pointing to the beginning of the bytes to send. + * @param length The size of the bytes sequence to write. + * @return int Whether there was an error while writing. 0 corresponds to + * success and -1 corresponds to an error (errno will be set). + */ +int write_bytes(int fd, uint8_t *cursor, size_t length) { + ssize_t nbytes = 0; + while (length > 0) { + /* While we haven't written the whole message, write to the file + * descriptor, advance the cursor, and decrease the amount left to write. */ + nbytes = write(fd, cursor, length); + if (nbytes < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + continue; + } + /* TODO(swang): Return the error instead of exiting. */ + /* Force an exit if there was any other type of error. */ + CHECK(nbytes < 0); + } + if (nbytes == 0) { + return -1; + } + cursor += nbytes; + length -= nbytes; + } + return 0; +} + +/** + * Write a sequence of bytes on a file descriptor. The bytes should then be read + * by read_message. + * + * @param fd The file descriptor to write to. It can be non-blocking. + * @param type The type of the message to send. + * @param length The size in bytes of the bytes parameter. + * @param bytes The address of the message to send. + * @return int Whether there was an error while writing. 0 corresponds to + * success and -1 corresponds to an error (errno will be set). + */ +int write_message(int fd, int64_t type, int64_t length, uint8_t *bytes) { + int closed; + closed = write_bytes(fd, (uint8_t *) &type, sizeof(type)); + if (closed) { + return closed; + } + closed = write_bytes(fd, (uint8_t *) &length, sizeof(length)); + if (closed) { + return closed; + } + closed = write_bytes(fd, bytes, length * sizeof(char)); + if (closed) { + return closed; + } + return 0; +} + +/** + * Read a sequence of bytes from a file descriptor into a buffer. This will + * block until one of the following happens: (1) there is an error (2) end of + * file, or (3) all length bytes have been written. + * + * @note The buffer pointed to by cursor must already have length number of + * bytes allocated before calling this method. + * + * @param fd The file descriptor to read from. It can be non-blocking. + * @param cursor The cursor pointing to the beginning of the buffer. + * @param length The size of the byte sequence to read. + * @return int Whether there was an error while writing. 0 corresponds to + * success and -1 corresponds to an error (errno will be set). + */ +int read_bytes(int fd, uint8_t *cursor, size_t length) { + ssize_t nbytes = 0; + while (length > 0) { + /* While we haven't read the whole message, read from the file descriptor, + * advance the cursor, and decrease the amount left to read. */ + nbytes = read(fd, cursor, length); + if (nbytes < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + continue; + } + /* Force an exit if there was any other type of error. */ + CHECK(nbytes < 0); + } + if (nbytes == 0) { + return -1; + } + cursor += nbytes; + length -= nbytes; + } + return 0; +} + +/** + * Read a sequence of bytes written by write_message from a file descriptor. + * This allocates space for the message. + * + * @note The caller must free the memory. + * + * @param fd The file descriptor to read from. It can be non-blocking. + * @param type The type of the message that is read will be written at this + address. If there was an error while reading, this will be + DISCONNECT_CLIENT. + * @param length The size in bytes of the message that is read will be written + at this address. This size does not include the bytes used to encode + the type and length. If there was an error while reading, this will + be 0. + * @param bytes The address at which to write the pointer to the bytes that are + read and allocated by this function. If there was an error while + reading, this will be NULL. + + * @return Void. + */ +void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes) { + int closed = read_bytes(fd, (uint8_t *) type, sizeof(int64_t)); + if (closed) { + goto disconnected; + } + closed = read_bytes(fd, (uint8_t *) length, sizeof(int64_t)); + if (closed) { + goto disconnected; + } + *bytes = malloc(*length * sizeof(uint8_t)); + closed = read_bytes(fd, *bytes, *length); + if (closed) { + free(*bytes); + goto disconnected; + } + return; + +disconnected: + /* Handle the case in which the socket is closed. */ + *type = DISCONNECT_CLIENT; + *length = 0; + *bytes = NULL; + return; +} + +/* Write a null-terminated string to a file descriptor. */ +void write_log_message(int fd, char *message) { + /* Account for the \0 at the end of the string. */ + write_message(fd, LOG_MESSAGE, strlen(message) + 1, (uint8_t *) message); +} + +/* Reads a null-terminated string from the file descriptor that has been + * written by write_log_message. Allocates and returns a pointer to the string. + * NOTE: Caller must free the memory! */ +char *read_log_message(int fd) { + uint8_t *bytes; + int64_t type; + int64_t length; + read_message(fd, &type, &length, &bytes); + CHECK(type == LOG_MESSAGE); + return (char *) bytes; +} + +void write_formatted_log_message(int socket_fd, const char *format, ...) { + UT_string *cmd; + va_list ap; + + utstring_new(cmd); + va_start(ap, format); + utstring_printf_va(cmd, format, ap); + va_end(ap); + + write_log_message(socket_fd, utstring_body(cmd)); + utstring_free(cmd); +} diff --git a/src/common/io.h b/src/common/io.h new file mode 100644 index 000000000..362b85fc1 --- /dev/null +++ b/src/common/io.h @@ -0,0 +1,32 @@ +#ifndef IO_H +#define IO_H + +#include + +enum common_message_type { + /** Disconnect a client. */ + DISCONNECT_CLIENT, + /** Log a message from a client. */ + LOG_MESSAGE, + /** Submit a task to the local scheduler. */ + SUBMIT_TASK, +}; + +/* Helper functions for socket communication. */ + +int bind_inet_sock(const int port); +int bind_ipc_sock(const char *socket_pathname); +int connect_ipc_sock(const char *socket_pathname); + +int accept_client(int socket_fd); + +/* Reading and writing data */ + +int write_message(int fd, int64_t type, int64_t length, uint8_t *bytes); +void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes); + +void write_log_message(int fd, char *message); +void write_formatted_log_message(int fd, const char *format, ...); +char *read_log_message(int fd); + +#endif diff --git a/src/common/lib/python/common_extension.c b/src/common/lib/python/common_extension.c new file mode 100644 index 000000000..14f3292d9 --- /dev/null +++ b/src/common/lib/python/common_extension.c @@ -0,0 +1,344 @@ +#include +#include "node.h" + +#include "common_extension.h" +#include "task.h" +#include "utarray.h" +#include "utstring.h" + +PyObject *CommonError; + +#define MARSHAL_VERSION 2 + +/* Define the PyObjectID class. */ + +int PyObjectToUniqueID(PyObject *object, object_id *objectid) { + if (PyObject_IsInstance(object, (PyObject *) &PyObjectIDType)) { + *objectid = ((PyObjectID *) object)->object_id; + return 1; + } else { + PyErr_SetString(PyExc_TypeError, "must be an ObjectID"); + return 0; + } +} + +static int PyObjectID_init(PyObjectID *self, PyObject *args, PyObject *kwds) { + const char *data; + int size; + if (!PyArg_ParseTuple(args, "s#", &data, &size)) { + return -1; + } + if (size != UNIQUE_ID_SIZE) { + PyErr_SetString(CommonError, + "ObjectID: object id string needs to have length 20"); + return -1; + } + memcpy(&self->object_id.id[0], data, UNIQUE_ID_SIZE); + return 0; +} + +/* Create a PyObjectID from C. */ +PyObject *PyObjectID_make(object_id object_id) { + PyObjectID *result = PyObject_New(PyObjectID, &PyObjectIDType); + result = (PyObjectID *) PyObject_Init((PyObject *) result, &PyObjectIDType); + result->object_id = object_id; + return (PyObject *) result; +} + +static PyObject *PyObjectID_id(PyObject *self) { + PyObjectID *s = (PyObjectID *) self; + return PyString_FromStringAndSize((char *) &s->object_id.id[0], + UNIQUE_ID_SIZE); +} + +static PyObject *PyObjectID___reduce__(PyObjectID *self) { + PyErr_SetString(CommonError, "ObjectID objects cannot be serialized."); + return NULL; +} + +static PyMethodDef PyObjectID_methods[] = { + {"id", (PyCFunction) PyObjectID_id, METH_NOARGS, + "Return the hash associated with this ObjectID"}, + {"__reduce__", (PyCFunction) PyObjectID___reduce__, METH_NOARGS, + "Say how to pickle this ObjectID. This raises an exception to prevent" + "object IDs from being serialized."}, + {NULL} /* Sentinel */ +}; + +static PyMemberDef PyObjectID_members[] = { + {NULL} /* Sentinel */ +}; + +PyTypeObject PyObjectIDType = { + PyObject_HEAD_INIT(NULL) 0, /* ob_size */ + "common.ObjectID", /* tp_name */ + sizeof(PyObjectID), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "ObjectID object", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyObjectID_methods, /* tp_methods */ + PyObjectID_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc) PyObjectID_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +/* Define the PyTask class. */ + +static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) { + function_id function_id; + /* Arguments of the task (can be PyObjectIDs or Python values). */ + PyObject *arguments; + /* Array of pointers to string representations of pass-by-value args. */ + UT_array *val_repr_ptrs; + utarray_new(val_repr_ptrs, &ut_ptr_icd); + int num_returns; + if (!PyArg_ParseTuple(args, "O&Oi", &PyObjectToUniqueID, &function_id, + &arguments, &num_returns)) { + return -1; + } + size_t size = PyList_Size(arguments); + /* Determine the size of pass by value data in bytes. */ + size_t value_data_bytes = 0; + for (size_t i = 0; i < size; ++i) { + PyObject *arg = PyList_GetItem(arguments, i); + if (!PyObject_IsInstance(arg, (PyObject *) &PyObjectIDType)) { + PyObject *data = PyMarshal_WriteObjectToString(arg, MARSHAL_VERSION); + value_data_bytes += PyString_Size(data); + utarray_push_back(val_repr_ptrs, &data); + } + } + /* Construct the task specification. */ + int val_repr_index = 0; + self->spec = + alloc_task_spec(function_id, size, num_returns, value_data_bytes); + /* Add the task arguments. */ + for (size_t i = 0; i < size; ++i) { + PyObject *arg = PyList_GetItem(arguments, i); + if (PyObject_IsInstance(arg, (PyObject *) &PyObjectIDType)) { + task_args_add_ref(self->spec, ((PyObjectID *) arg)->object_id); + } else { + PyObject *data = + *((PyObject **) utarray_eltptr(val_repr_ptrs, val_repr_index)); + task_args_add_val(self->spec, (uint8_t *) PyString_AS_STRING(data), + PyString_GET_SIZE(data)); + Py_DECREF(data); + val_repr_index += 1; + } + } + utarray_free(val_repr_ptrs); + /* Generate and add the object IDs for the return values. */ + for (size_t i = 0; i < num_returns; ++i) { + /* TODO(rkn): Later, this should be computed as a deterministic hash of (1) + * the contents of the task, (2) the index i, and (3) a counter of the + * number of tasks launched so far by the parent task. For now, we generate + * it randomly. */ + *task_return(self->spec, i) = globally_unique_id(); + } + return 0; +} + +static void PyTask_dealloc(PyTask *self) { + free_task_spec(self->spec); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject *PyTask_function_id(PyObject *self) { + function_id function_id = *task_function(((PyTask *) self)->spec); + return PyObjectID_make(function_id); +} + +static PyObject *PyTask_arguments(PyObject *self) { + int64_t num_args = task_num_args(((PyTask *) self)->spec); + PyObject *arg_list = PyList_New((Py_ssize_t) num_args); + task_spec *task = ((PyTask *) self)->spec; + for (int i = 0; i < num_args; ++i) { + if (task_arg_type(task, i) == ARG_BY_REF) { + object_id object_id = *task_arg_id(task, i); + PyList_SetItem(arg_list, i, PyObjectID_make(object_id)); + } else { + PyObject *s = + PyMarshal_ReadObjectFromString((char *) task_arg_val(task, i), + (Py_ssize_t) task_arg_length(task, i)); + PyList_SetItem(arg_list, i, s); + } + } + return arg_list; +} + +static PyObject *PyTask_returns(PyObject *self) { + int64_t num_returns = task_num_returns(((PyTask *) self)->spec); + PyObject *return_id_list = PyList_New((Py_ssize_t) num_returns); + task_spec *task = ((PyTask *) self)->spec; + for (int i = 0; i < num_returns; ++i) { + object_id object_id = *task_return(task, i); + PyList_SetItem(return_id_list, i, PyObjectID_make(object_id)); + } + return return_id_list; +} + +static PyMethodDef PyTask_methods[] = { + {"function_id", (PyCFunction) PyTask_function_id, METH_NOARGS, + "Return the function ID for this task."}, + {"arguments", (PyCFunction) PyTask_arguments, METH_NOARGS, + "Return the arguments for the task."}, + {"returns", (PyCFunction) PyTask_returns, METH_NOARGS, + "Return the object IDs for the return values of the task."}, + {NULL} /* Sentinel */ +}; + +PyTypeObject PyTaskType = { + PyObject_HEAD_INIT(NULL) 0, /* ob_size */ + "task.Task", /* tp_name */ + sizeof(PyTask), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor) PyTask_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Task object", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyTask_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc) PyTask_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +/* Create a PyTask from a C struct. The resulting PyTask takes ownership of the + * task_spec and will deallocate the task_spec in the PyTask destructor. */ +PyObject *PyTask_make(task_spec *task_spec) { + PyTask *result = PyObject_New(PyTask, &PyTaskType); + result = (PyTask *) PyObject_Init((PyObject *) result, &PyTaskType); + result->spec = task_spec; + return (PyObject *) result; +} + +/* Define the methods for the module. */ + +#define SIZE_LIMIT 100 +#define NUM_ELEMENTS_LIMIT 1000 + +/** + * This method checks if a Python object is sufficiently simple that it can be + * serialized and passed by value as an argument to a task (without being put in + * the object store). The details of which objects are sufficiently simple are + * defined by this method and are not particularly important. But for + * performance reasons, it is better to place "small" objects in the task itself + * and "large" objects in the object store. + * + * @param value The Python object in question. + * @param num_elements_contained If this method returns 1, then the number of + * objects recursively contained within this object will be added to the + * value at this address. This is used to make sure that we do not + * serialize objects that are too large. + * @return 0 if the object cannot be serialized in the task and 1 if it can. + */ +int is_simple_value(PyObject *value, int *num_elements_contained) { + *num_elements_contained += 1; + if (*num_elements_contained >= NUM_ELEMENTS_LIMIT) { + return 0; + } + if (PyInt_Check(value) || PyLong_Check(value) || value == Py_False || + value == Py_True || PyFloat_Check(value) || value == Py_None) { + return 1; + } + if (PyString_CheckExact(value)) { + *num_elements_contained += PyString_Size(value); + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + if (PyUnicode_CheckExact(value)) { + *num_elements_contained += PyUnicode_GET_SIZE(value); + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + if (PyList_CheckExact(value) && PyList_Size(value) < SIZE_LIMIT) { + for (size_t i = 0; i < PyList_Size(value); ++i) { + if (!is_simple_value(PyList_GetItem(value, i), num_elements_contained)) { + return 0; + } + } + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + if (PyDict_CheckExact(value) && PyDict_Size(value) < SIZE_LIMIT) { + PyObject *key, *val; + Py_ssize_t pos = 0; + while (PyDict_Next(value, &pos, &key, &val)) { + if (!is_simple_value(key, num_elements_contained) || + !is_simple_value(val, num_elements_contained)) { + return 0; + } + } + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + if (PyTuple_CheckExact(value) && PyTuple_Size(value) < SIZE_LIMIT) { + for (size_t i = 0; i < PyTuple_Size(value); ++i) { + if (!is_simple_value(PyTuple_GetItem(value, i), num_elements_contained)) { + return 0; + } + } + return (*num_elements_contained < NUM_ELEMENTS_LIMIT); + } + return 0; +} + +PyObject *check_simple_value(PyObject *self, PyObject *args) { + PyObject *value; + if (!PyArg_ParseTuple(args, "O", &value)) { + return NULL; + } + int num_elements_contained = 0; + if (is_simple_value(value, &num_elements_contained)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} diff --git a/src/common/lib/python/common_extension.h b/src/common/lib/python/common_extension.h new file mode 100644 index 000000000..1fce38e42 --- /dev/null +++ b/src/common/lib/python/common_extension.h @@ -0,0 +1,37 @@ +#ifndef COMMON_EXTENSION_H +#define COMMON_EXTENSION_H + +#include +#include "marshal.h" +#include "structmember.h" + +#include "common.h" +#include "task.h" + +extern PyObject *CommonError; + +// clang-format off +typedef struct { + PyObject_HEAD + object_id object_id; +} PyObjectID; + +typedef struct { + PyObject_HEAD + task_spec *spec; +} PyTask; +// clang-format on + +extern PyTypeObject PyObjectIDType; + +extern PyTypeObject PyTaskType; + +int PyObjectToUniqueID(PyObject *object, object_id *objectid); + +PyObject *PyObjectID_make(object_id object_id); + +PyObject *check_simple_value(PyObject *self, PyObject *args); + +PyObject *PyTask_make(task_spec *task_spec); + +#endif /* COMMON_EXTENSION_H */ diff --git a/src/common/lib/python/common_module.c b/src/common/lib/python/common_module.c new file mode 100644 index 000000000..d5222cd87 --- /dev/null +++ b/src/common/lib/python/common_module.c @@ -0,0 +1,38 @@ +#include +#include "node.h" + +#include "common_extension.h" + +static PyMethodDef common_methods[] = { + {"check_simple_value", check_simple_value, METH_VARARGS, + "Should the object be passed by value?"}, + {NULL} /* Sentinel */ +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif + +PyMODINIT_FUNC initcommon(void) { + PyObject *m; + + if (PyType_Ready(&PyTaskType) < 0) + return; + + if (PyType_Ready(&PyObjectIDType) < 0) + return; + + m = Py_InitModule3("common", common_methods, + "A module for common types. This is used for testing."); + + Py_INCREF(&PyTaskType); + PyModule_AddObject(m, "Task", (PyObject *) &PyTaskType); + + Py_INCREF(&PyObjectIDType); + PyModule_AddObject(m, "ObjectID", (PyObject *) &PyObjectIDType); + + char common_error[] = "common.error"; + CommonError = PyErr_NewException(common_error, NULL, NULL); + Py_INCREF(CommonError); + PyModule_AddObject(m, "common_error", CommonError); +} diff --git a/src/common/lib/python/setup.py b/src/common/lib/python/setup.py new file mode 100644 index 000000000..8ef8a1abd --- /dev/null +++ b/src/common/lib/python/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup, find_packages, Extension + +common_module = Extension("common", + sources=["common_module.c", "common_extension.c"], + include_dirs=["../../", "../../thirdparty"], + extra_objects=["../../build/libcommon.a"], + extra_compile_args=["--std=c99", "-Werror"]) + +setup(name="Common", + version="0.1", + description="Common library for Ray", + ext_modules=[common_module]) diff --git a/src/common/logging.c b/src/common/logging.c new file mode 100644 index 000000000..09cad7856 --- /dev/null +++ b/src/common/logging.c @@ -0,0 +1,80 @@ +#include "logging.h" + +#include +#include +#include +#include + +#include "state/redis.h" +#include "io.h" + +static const char *log_levels[5] = {"DEBUG", "INFO", "WARN", "ERROR", "FATAL"}; +static const char *log_fmt = + "HMSET log:%s:%s:%s log_level %s event_type %s message %s timestamp %s"; + +struct ray_logger_impl { + /* String that identifies this client type. */ + const char *client_type; + /* Suppress all log messages below this level. */ + int log_level; + /* Whether or not we have a direct connection to Redis. */ + int is_direct; + /* Either a db_handle or a socket to a process with a db_handle, + * depending on the is_direct flag. */ + void *conn; +}; + +ray_logger *init_ray_logger(const char *client_type, + int log_level, + int is_direct, + void *conn) { + ray_logger *logger = malloc(sizeof(ray_logger)); + logger->client_type = client_type; + logger->log_level = log_level; + logger->is_direct = is_direct; + logger->conn = conn; + return logger; +} + +void free_ray_logger(ray_logger *logger) { + free(logger); +} + +void ray_log(ray_logger *logger, + int log_level, + const char *event_type, + const char *message) { + if (log_level < logger->log_level) { + return; + } + if (log_level < RAY_DEBUG || log_level > RAY_FATAL) { + return; + } + struct timeval tv; + UT_string *timestamp; + utstring_new(timestamp); + gettimeofday(&tv, NULL); + utstring_printf(timestamp, "%ld.%ld", tv.tv_sec, (long) tv.tv_usec); + + UT_string *origin_id; + utstring_new(origin_id); + if (logger->is_direct) { + db_handle *db = (db_handle *) logger->conn; + utstring_printf(origin_id, "%" PRId64 ":%s", db->client_id, ""); + redisAsyncCommand(db->context, NULL, NULL, log_fmt, + utstring_body(timestamp), logger->client_type, + utstring_body(origin_id), log_levels[log_level], + event_type, message, utstring_body(timestamp)); + } else { + /* If we don't own a Redis connection, we leave our client + * ID to be filled in by someone else. */ + utstring_printf(origin_id, "%s:%s", "%ld", "%ld"); + int *socket_fd = (int *) logger->conn; + write_formatted_log_message(*socket_fd, log_fmt, utstring_body(timestamp), + logger->client_type, utstring_body(origin_id), + log_levels[log_level], event_type, message, + utstring_body(timestamp)); + } + utstring_free(origin_id); + utstring_free(timestamp); +} diff --git a/src/common/logging.h b/src/common/logging.h new file mode 100644 index 000000000..4ef7c8fcb --- /dev/null +++ b/src/common/logging.h @@ -0,0 +1,39 @@ +#ifndef LOGGING_H +#define LOGGING_H + +#define RAY_VERBOSE -1 +#define RAY_DEBUG 0 +#define RAY_INFO 1 +#define RAY_WARNING 2 +#define RAY_ERROR 3 +#define RAY_FATAL 4 + +/* Entity types. */ +#define RAY_FUNCTION "FUNCTION" +#define RAY_OBJECT "OBJECT" +#define RAY_TASK "TASK" + +typedef struct ray_logger_impl ray_logger; + +/* Initialize a Ray logger for the given client type and logging level. If the + * is_direct flag is set, the logger will treat the given connection as a + * direct connection to the log. Otherwise, it will treat it as a socket to + * another process with a connection to the log. + * NOTE: User is responsible for freeing the returned logger. */ +ray_logger *init_ray_logger(const char *client_type, + int log_level, + int is_direct, + void *conn); + +/* Free the logger. This does not free the connection to the log. */ +void free_ray_logger(ray_logger *logger); + +/* Log an event at the given log level with the given event_type. + * NOTE: message cannot contain spaces! JSON format is recommended. + * TODO: Support spaces in messages. */ +void ray_log(ray_logger *logger, + int log_level, + const char *event_type, + const char *message); + +#endif diff --git a/src/common/state/db.h b/src/common/state/db.h new file mode 100644 index 000000000..50536e271 --- /dev/null +++ b/src/common/state/db.h @@ -0,0 +1,32 @@ +#ifndef DB_H +#define DB_H + +#include "event_loop.h" + +typedef struct db_handle_impl db_handle; + +/* Connect to the global system store at address and port. Returns + * a handle to the database, which must be freed with db_disconnect + * after use. */ +db_handle *db_connect(const char *db_address, + int db_port, + const char *client_type, + const char *client_addr, + int client_port); + +/* Attach global system store connection to event loop. */ +void db_attach(db_handle *db, event_loop *loop); + +/* Disconnect from the global system store. */ +void db_disconnect(db_handle *db); + +/** + * Returns the client ID, according to the database. + * + * @param db The handle to the database. + * @returns int The client ID for this connection to the database. If + * this client has no connection to the database, returns -1. + */ +int get_client_id(db_handle *db); + +#endif diff --git a/src/common/state/object_table.h b/src/common/state/object_table.h new file mode 100644 index 000000000..bab54bc7b --- /dev/null +++ b/src/common/state/object_table.h @@ -0,0 +1,25 @@ +#include "common.h" +#include "db.h" + +/* The callback that is called when the result of a lookup + * in the object table comes back. The callback should free + * the manager_vector array, but NOT the strings they are pointing to. */ +typedef void (*lookup_callback)(object_id object_id, + int manager_count, + const char *manager_vector[], + void *context); + +/* Register a new object with the directory. */ +/* TODO(pcm): Retry, print for each attempt. */ +void object_table_add(db_handle *db, object_id object_id); + +/* Remove object from the directory. */ +void object_table_remove(db_handle *db, + object_id object_id, + const char *manager); + +/* Look up entry from the directory */ +void object_table_lookup(db_handle *db, + object_id object_id, + lookup_callback callback, + void *context); diff --git a/src/common/state/redis.c b/src/common/state/redis.c new file mode 100644 index 000000000..d53b0bf15 --- /dev/null +++ b/src/common/state/redis.c @@ -0,0 +1,244 @@ +/* Redis implementation of the global state store */ + +#include + +#include +#include "hiredis/adapters/ae.h" +#include "utstring.h" + +#include "common.h" +#include "db.h" +#include "object_table.h" +#include "task_log.h" +#include "event_loop.h" +#include "redis.h" +#include "io.h" + +#define LOG_REDIS_ERR(context, M, ...) \ + fprintf(stderr, "[ERROR] (%s:%d: message: %s) " M "\n", __FILE__, __LINE__, \ + context->errstr, ##__VA_ARGS__) + +#define CHECK_REDIS_CONNECT(CONTEXT_TYPE, context, M, ...) \ + do { \ + CONTEXT_TYPE *_context = (context); \ + if (!_context) { \ + LOG_ERR("could not allocate redis context"); \ + exit(-1); \ + } \ + if (_context->err) { \ + LOG_REDIS_ERR(_context, M, ##__VA_ARGS__); \ + exit(-1); \ + } \ + } while (0); + +db_handle *db_connect(const char *address, + int port, + const char *client_type, + const char *client_addr, + int client_port) { + db_handle *db = malloc(sizeof(db_handle)); + /* Sync connection for initial handshake */ + redisReply *reply; + long long num_clients; + redisContext *context = redisConnect(address, port); + CHECK_REDIS_CONNECT(redisContext, context, "could not connect to redis %s:%d", + address, port); + /* Add new client using optimistic locking. */ + while (1) { + reply = redisCommand(context, "WATCH %s", client_type); + freeReplyObject(reply); + reply = redisCommand(context, "HLEN %s", client_type); + num_clients = reply->integer; + freeReplyObject(reply); + reply = redisCommand(context, "MULTI"); + freeReplyObject(reply); + reply = redisCommand(context, "HSET %s %lld %s:%d", client_type, + num_clients, client_addr, client_port); + freeReplyObject(reply); + reply = redisCommand(context, "EXEC"); + CHECK(reply); + if (reply->type != REDIS_REPLY_NIL) { + freeReplyObject(reply); + break; + } + freeReplyObject(reply); + } + + db->client_type = strdup(client_type); + db->client_id = num_clients; + db->service_cache = NULL; + db->sync_context = context; + utarray_new(db->callback_freelist, &ut_ptr_icd); + + /* Establish async connection */ + db->context = redisAsyncConnect(address, port); + CHECK_REDIS_CONNECT(redisAsyncContext, db->context, + "could not connect to redis %s:%d", address, port); + db->context->data = (void *) db; + /* Establish async connection for subscription */ + db->sub_context = redisAsyncConnect(address, port); + CHECK_REDIS_CONNECT(redisAsyncContext, db->sub_context, + "could not connect to redis %s:%d", address, port); + db->sub_context->data = (void *) db; + + return db; +} + +void db_disconnect(db_handle *db) { + redisFree(db->sync_context); + redisAsyncFree(db->context); + redisAsyncFree(db->sub_context); + service_cache_entry *e, *tmp; + HASH_ITER(hh, db->service_cache, e, tmp) { + free(e->addr); + HASH_DEL(db->service_cache, e); + free(e); + } + free(db->client_type); + void **p = NULL; + while ((p = (void **) utarray_next(db->callback_freelist, p))) { + free(*p); + } + utarray_free(db->callback_freelist); + free(db); +} + +void db_attach(db_handle *db, event_loop *loop) { + redisAeAttach(loop, db->context); + redisAeAttach(loop, db->sub_context); +} + +void object_table_add(db_handle *db, unique_id object_id) { + redisAsyncCommand(db->context, NULL, NULL, "SADD obj:%b %d", &object_id.id[0], + UNIQUE_ID_SIZE, db->client_id); + if (db->context->err) { + LOG_REDIS_ERR(db->context, "could not add object_table entry"); + } +} + +void object_table_get_entry(redisAsyncContext *c, void *r, void *privdata) { + db_handle *db = c->data; + lookup_callback_data *cb_data = privdata; + redisReply *reply = r; + if (reply == NULL) + return; + int *result = malloc(reply->elements * sizeof(int)); + int64_t manager_count = reply->elements; + if (reply->type == REDIS_REPLY_ARRAY) { + for (int j = 0; j < reply->elements; j++) { + CHECK(reply->element[j]->type == REDIS_REPLY_STRING); + result[j] = atoi(reply->element[j]->str); + service_cache_entry *entry; + HASH_FIND_INT(db->service_cache, &result[j], entry); + if (!entry) { + redisReply *reply = redisCommand(db->sync_context, "HGET %s %lld", + db->client_type, result[j]); + CHECK(reply->type == REDIS_REPLY_STRING); + entry = malloc(sizeof(service_cache_entry)); + entry->service_id = result[j]; + entry->addr = strdup(reply->str); + HASH_ADD_INT(db->service_cache, service_id, entry); + freeReplyObject(reply); + } + } + } else { + LOG_ERR("expected integer or string, received type %d", reply->type); + exit(-1); + } + const char **manager_vector = malloc(manager_count * sizeof(char *)); + for (int j = 0; j < manager_count; ++j) { + service_cache_entry *entry; + HASH_FIND_INT(db->service_cache, &result[j], entry); + manager_vector[j] = entry->addr; + } + cb_data->callback(cb_data->object_id, manager_count, manager_vector, + cb_data->context); + free(privdata); + free(result); +} + +void object_table_lookup(db_handle *db, + object_id object_id, + lookup_callback callback, + void *context) { + lookup_callback_data *cb_data = malloc(sizeof(lookup_callback_data)); + cb_data->callback = callback; + cb_data->object_id = object_id; + cb_data->context = context; + redisAsyncCommand(db->context, object_table_get_entry, cb_data, + "SMEMBERS obj:%b", &object_id.id[0], UNIQUE_ID_SIZE); + if (db->context->err) { + LOG_REDIS_ERR(db->context, "error in object_table lookup"); + } +} + +void task_log_add_task(db_handle *db, task_instance *task_instance) { + task_iid task_iid = *task_instance_id(task_instance); + redisAsyncCommand(db->context, NULL, NULL, "HMSET tasklog:%b 0 %b", + (char *) &task_iid.id[0], UNIQUE_ID_SIZE, + (char *) task_instance, task_instance_size(task_instance)); + if (db->context->err) { + LOG_REDIS_ERR(db->context, "error setting task in task_log_add_task"); + } + node_id node = *task_instance_node(task_instance); + int32_t state = *task_instance_state(task_instance); + redisAsyncCommand(db->context, NULL, NULL, "PUBLISH task_log:%b:%d %b", + (char *) &node.id[0], UNIQUE_ID_SIZE, state, + (char *) task_instance, task_instance_size(task_instance)); + if (db->context->err) { + LOG_REDIS_ERR(db->context, "error publishing task in task_log_add_task"); + } +} + +void task_log_redis_callback(redisAsyncContext *c, + void *reply, + void *privdata) { + redisReply *r = reply; + if (reply == NULL) + return; + CHECK(r->type == REDIS_REPLY_ARRAY); + /* First entry is message type, second is topic, third is payload. */ + CHECK(r->elements > 2); + /* If this condition is true, we got the initial message that acknowledged the + * subscription. */ + if (r->element[2]->str == NULL) { + return; + } + /* Otherwise, parse the task and call the callback. */ + CHECK(privdata); + task_log_callback_data *callback_data = privdata; + task_instance *instance = malloc(r->element[2]->len); + memcpy(instance, r->element[2]->str, r->element[2]->len); + callback_data->callback(instance, callback_data->userdata); + task_instance_free(instance); +} +void task_log_register_callback(db_handle *db, + task_log_callback callback, + node_id node, + int32_t state, + void *userdata) { + task_log_callback_data *callback_data = + malloc(sizeof(task_log_callback_data)); + utarray_push_back(db->callback_freelist, &callback_data); + callback_data->callback = callback; + callback_data->userdata = userdata; + if (memcmp(&node.id[0], &NIL_ID.id[0], UNIQUE_ID_SIZE) == 0) { + redisAsyncCommand(db->sub_context, task_log_redis_callback, callback_data, + "PSUBSCRIBE task_log:*:%d", state); + } else { + redisAsyncCommand(db->sub_context, task_log_redis_callback, callback_data, + "SUBSCRIBE task_log:%b:%d", (char *) &node.id[0], + UNIQUE_ID_SIZE, state); + } + if (db->sub_context->err) { + LOG_REDIS_ERR(db->sub_context, "error in task_log_register_callback"); + } +} + +int get_client_id(db_handle *db) { + if (db) { + return db->client_id; + } else { + return -1; + } +} diff --git a/src/common/state/redis.h b/src/common/state/redis.h new file mode 100644 index 000000000..cf368b992 --- /dev/null +++ b/src/common/state/redis.h @@ -0,0 +1,68 @@ +#ifndef REDIS_H +#define REDIS_H + +#include "db.h" +#include "object_table.h" +#include "task_log.h" + +#include "hiredis/hiredis.h" +#include "hiredis/async.h" +#include "uthash.h" +#include "utarray.h" + +typedef struct { + /* Unique ID for this service. */ + int service_id; + /* IP address and port of this service. */ + char *addr; + /* Handle for the uthash table. */ + UT_hash_handle hh; +} service_cache_entry; + +typedef struct { + /* The callback that will be called. */ + task_log_callback callback; + /* Userdata associated with the callback. */ + void *userdata; +} task_log_callback_data; + +struct db_handle_impl { + /* String that identifies this client type. */ + char *client_type; + /* Unique ID for this client within the type. */ + int64_t client_id; + /* Redis context for this global state store connection. */ + redisAsyncContext *context; + /* Redis context for "subscribe" communication. + * Yes, we need a separate one for that, see + * https://github.com/redis/hiredis/issues/55 */ + redisAsyncContext *sub_context; + /* The event loop this global state store connection is part of. */ + event_loop *loop; + /* Index of the database connection in the event loop */ + int64_t db_index; + /* Cache for the IP addresses of services. */ + service_cache_entry *service_cache; + /* Redis context for synchronous connections. + * Should only be used very rarely, it is not asynchronous. */ + redisContext *sync_context; + /* Data structure for callbacks that needs to be freed. */ + UT_array *callback_freelist; +}; + +typedef struct { + /* The callback that will be called. */ + lookup_callback callback; + /* Object ID that is looked up. */ + object_id object_id; + /* Data context for the callback. */ + void *context; +} lookup_callback_data; + +void object_table_get_entry(redisAsyncContext *c, void *r, void *privdata); + +void object_table_lookup_callback(redisAsyncContext *c, + void *r, + void *privdata); + +#endif diff --git a/src/common/state/task_log.h b/src/common/state/task_log.h new file mode 100644 index 000000000..acf5dbcd0 --- /dev/null +++ b/src/common/state/task_log.h @@ -0,0 +1,41 @@ +#ifndef TASK_LOG_H +#define TASK_LOG_H + +#include "db.h" +#include "task.h" + +/* The task log is a message bus that is used for all communication between + * local and global schedulers (and also persisted to the state database). + * Here are examples of events that are recorded by the task log: + * + * 1) local scheduler writes it when submits a task to the global scheduler; + * 2) global scheduler reads it to get the task submitted by local schedulers; + * 3) global scheduler writes it when assigning the task to a local scheduler; + * 4) local scheduler reads it to get its tasks assigned by global scheduler; + * 5) local scheduler writes it when a task finishes execution; + * 6) global scheduler reads it to get the tasks that have finished; */ + +/* Callback for subscribing to the task log. */ +typedef void (*task_log_callback)(task_instance *task_instance, void *userdata); + +/* Initially add a task instance to the task log. */ +void task_log_add_task(db_handle *db, task_instance *task_instance); + +/* Update task instance in the task log. */ +void task_log_update_task(db_handle *db, + task_iid task_iid, + int32_t state, + node_id node); + +/* Register callback for a certain event. The node specifies the node whose + * events we want to listen to. If you want to listen to all events for this + * node, use state_filter = + * TASK_WAITING | TASK_SCHEDULED | TASK_RUNNING | TASK_DONE. + * If you want to register to updates from all nodes, set node = NIL_ID. */ +void task_log_register_callback(db_handle *db, + task_log_callback callback, + node_id node, + int32_t state_filter, + void *userdata); + +#endif /* TASK_LOG_H */ diff --git a/src/common/state/task_table.h b/src/common/state/task_table.h new file mode 100644 index 000000000..3a1852522 --- /dev/null +++ b/src/common/state/task_table.h @@ -0,0 +1,20 @@ +#ifndef TASK_TABLE_H +#define TASK_TABLE_H + +#include "db.h" +#include "task.h" + +/* Add task to the task table, handle errors here. */ +status task_table_add_task(db_handle *db, task_spec *task); + +/* Callback for getting an entry from the task table. Task spec will be freed + * by the system after the callback */ +typedef void (*task_table_callback)(task_spec *task, void *context); + +/* Get specific task from the task table. */ +status task_table_get_task(db_handle *db, + task_id task_id, + task_table_callback callback, + void *context); + +#endif /* TASK_TABLE_H */ diff --git a/src/common/task.c b/src/common/task.c new file mode 100644 index 000000000..0086eb337 --- /dev/null +++ b/src/common/task.c @@ -0,0 +1,218 @@ +#include +#include +#include + +#include "utarray.h" + +#include "task.h" +#include "common.h" +#include "io.h" + +/* TASK SPECIFICATIONS */ + +/* Tasks are stored in a consecutive chunk of memory, the first + * sizeof(task_spec) bytes are arranged according to the struct + * task_spec. Then there is an array of task_args of length + * (num_args + num_returns), and then follows the data of + * pass-by-value arguments of size args_value_size. The offsets in the + * task_arg.val are with respect to the end of the augmented structure, + * i.e. with respect to the address &task_spec.args_and_returns[0] + + * (task_spec->num_args + task_spec->num_returns) * sizeof(task_arg). */ + +typedef struct { + /* Either ARG_BY_REF or ARG_BY_VAL. */ + int8_t type; + union { + object_id obj_id; + struct { + /* Offset where the data associated to this arg is located relative + * to &task_spec.args_and_returns[0]. */ + ptrdiff_t offset; + int64_t length; + } value; + }; +} task_arg; + +struct task_spec_impl { + /* Function ID of the task. */ + function_id function_id; + /* Total number of arguments. */ + int64_t num_args; + /* Index of the last argument that has been constructed. */ + int64_t arg_index; + /* Number of return values. */ + int64_t num_returns; + /* Number of bytes the pass-by-value arguments are occupying. */ + int64_t args_value_size; + /* The offset of the number of bytes of pass-by-value data that + * has been written so far, relative to &task_spec->args_and_returns[0] + + * (task_spec->num_args + task_spec->num_returns) * sizeof(task_arg) */ + int64_t args_value_offset; + /* Argument and return IDs as well as offsets for pass-by-value args. */ + task_arg args_and_returns[0]; +}; + +/* The size of a task specification is given by the following expression. */ +#define TASK_SPEC_SIZE(NUM_ARGS, NUM_RETURNS, ARGS_VALUE_SIZE) \ + (sizeof(task_spec) + ((NUM_ARGS) + (NUM_RETURNS)) * sizeof(task_arg) + \ + (ARGS_VALUE_SIZE)) + +task_spec *alloc_task_spec(function_id function_id, + int64_t num_args, + int64_t num_returns, + int64_t args_value_size) { + int64_t size = TASK_SPEC_SIZE(num_args, num_returns, args_value_size); + task_spec *task = malloc(size); + memset(task, 0, size); + task->function_id = function_id; + task->num_args = num_args; + task->arg_index = 0; + task->num_returns = num_returns; + task->args_value_size = args_value_size; + return task; +} + +int64_t task_size(task_spec *spec) { + return TASK_SPEC_SIZE(spec->num_args, spec->num_returns, + spec->args_value_size); +} + +unique_id *task_function(task_spec *spec) { + return &spec->function_id; +} + +int64_t task_num_args(task_spec *spec) { + return spec->num_args; +} + +int64_t task_num_returns(task_spec *spec) { + return spec->num_returns; +} + +int8_t task_arg_type(task_spec *spec, int64_t arg_index) { + CHECK(0 <= arg_index && arg_index < spec->num_args); + return spec->args_and_returns[arg_index].type; +} + +object_id *task_arg_id(task_spec *spec, int64_t arg_index) { + CHECK(0 <= arg_index && arg_index < spec->num_args); + task_arg *arg = &spec->args_and_returns[arg_index]; + CHECK(arg->type == ARG_BY_REF) + return &arg->obj_id; +} + +uint8_t *task_arg_val(task_spec *spec, int64_t arg_index) { + CHECK(0 <= arg_index && arg_index < spec->num_args); + task_arg *arg = &spec->args_and_returns[arg_index]; + CHECK(arg->type == ARG_BY_VAL); + uint8_t *data = (uint8_t *) &spec->args_and_returns[0]; + data += (spec->num_args + spec->num_returns) * sizeof(task_arg); + return data + arg->value.offset; +} + +int64_t task_arg_length(task_spec *spec, int64_t arg_index) { + CHECK(0 <= arg_index && arg_index < spec->num_args); + task_arg *arg = &spec->args_and_returns[arg_index]; + CHECK(arg->type == ARG_BY_VAL); + return arg->value.length; +} + +int64_t task_args_add_ref(task_spec *spec, object_id obj_id) { + task_arg *arg = &spec->args_and_returns[spec->arg_index]; + arg->type = ARG_BY_REF; + arg->obj_id = obj_id; + return spec->arg_index++; +} + +int64_t task_args_add_val(task_spec *spec, uint8_t *data, int64_t length) { + task_arg *arg = &spec->args_and_returns[spec->arg_index]; + arg->type = ARG_BY_VAL; + arg->value.offset = spec->args_value_offset; + arg->value.length = length; + uint8_t *addr = task_arg_val(spec, spec->arg_index); + CHECK(spec->args_value_offset + length <= spec->args_value_size); + CHECK(spec->arg_index != spec->num_args - 1 || + spec->args_value_offset + length == spec->args_value_size); + memcpy(addr, data, length); + spec->args_value_offset += length; + return spec->arg_index++; +} + +object_id *task_return(task_spec *spec, int64_t ret_index) { + CHECK(0 <= ret_index && ret_index < spec->num_returns); + task_arg *ret = &spec->args_and_returns[spec->num_args + ret_index]; + CHECK(ret->type == ARG_BY_REF); /* No memory corruption. */ + return &ret->obj_id; +} + +void free_task_spec(task_spec *spec) { + CHECK(spec->arg_index == spec->num_args); /* Task was fully constructed */ + free(spec); +} + +void print_task(task_spec *spec, UT_string *output) { + /* For converting an id to hex, which has double the number + * of bytes compared to the id (+ 1 byte for '\0'). */ + static char hex[2 * UNIQUE_ID_SIZE + 1]; + /* Print function id. */ + sha1_to_hex(&task_function(spec)->id[0], &hex[0]); + utstring_printf(output, "fun %s ", &hex[0]); + /* Print arguments. */ + for (int i = 0; i < task_num_args(spec); ++i) { + sha1_to_hex(&task_arg_id(spec, i)->id[0], &hex[0]); + utstring_printf(output, " id:%d %s", i, &hex[0]); + } + /* Print return ids. */ + for (int i = 0; i < task_num_returns(spec); ++i) { + object_id *object_id = task_return(spec, i); + sha1_to_hex(&object_id->id[0], &hex[0]); + utstring_printf(output, " ret:%d %s", i, &hex[0]); + } +} + +/* TASK INSTANCES */ + +struct task_instance_impl { + task_iid iid; + int32_t state; + node_id node; + task_spec spec; +}; + +task_instance *make_task_instance(task_iid task_iid, + task_spec *spec, + int32_t state, + node_id node) { + int64_t size = sizeof(task_instance) - sizeof(task_spec) + task_size(spec); + task_instance *result = malloc(size); + memset(result, 0, size); + result->iid = task_iid; + result->state = state; + result->node = node; + memcpy(&result->spec, spec, task_size(spec)); + return result; +} + +int64_t task_instance_size(task_instance *instance) { + return sizeof(task_instance) - sizeof(task_spec) + task_size(&instance->spec); +} + +task_iid *task_instance_id(task_instance *instance) { + return &instance->iid; +} + +int32_t *task_instance_state(task_instance *instance) { + return &instance->state; +} + +node_id *task_instance_node(task_instance *instance) { + return &instance->node; +} + +task_spec *task_instance_task_spec(task_instance *instance) { + return &instance->spec; +} + +void task_instance_free(task_instance *instance) { + free(instance); +} diff --git a/src/common/task.h b/src/common/task.h new file mode 100644 index 000000000..782bf685b --- /dev/null +++ b/src/common/task.h @@ -0,0 +1,132 @@ +#ifndef TASK_H +#define TASK_H + +/* This API specifies the task data structures. It is in C so we can + * easily construct tasks from other languages like Python. The datastructures + * are also defined in such a way that memory is contiguous and all pointers + * are relative, so that we can memcpy the datastructure and ship it over the + * network without serialization and deserialization. */ + +#include +#include +#include "common.h" +#include "utstring.h" + +typedef unique_id function_id; + +/* The task ID is a deterministic hash of the function ID that + * the task executes and the argument IDs or argument values */ +typedef unique_id task_id; + +/* The task instance ID is a globally unique ID generated which + * identifies this particular execution of the task */ +typedef unique_id task_iid; + +/* The node id is an identifier for the node the task is + * scheduled on */ +typedef unique_id node_id; + +/* + * TASK SPECIFICATIONS: Contain all the information neccessary + * to execute the task (function id, arguments, return object ids). + * + */ + +typedef struct task_spec_impl task_spec; + +/* If argument is passed by value or reference. */ +enum arg_type { ARG_BY_REF, ARG_BY_VAL }; + +/* Construct and modify task specifications. */ + +/* Allocating and initializing a task. */ +task_spec *alloc_task_spec(function_id function_id, + int64_t num_args, + int64_t num_returns, + int64_t args_value_size); + +/* Size of the task in bytes. */ +int64_t task_size(task_spec *spec); + +/* Return the function ID of the task. */ +unique_id *task_function(task_spec *spec); + +/* Getting the number of arguments and returns. */ +int64_t task_num_args(task_spec *spec); +int64_t task_num_returns(task_spec *spec); + +/* Getting task arguments. */ +int8_t task_arg_type(task_spec *spec, int64_t arg_index); +unique_id *task_arg_id(task_spec *spec, int64_t arg_index); +uint8_t *task_arg_val(task_spec *spec, int64_t arg_index); +int64_t task_arg_length(task_spec *spec, int64_t arg_index); + +/* Setting task arguments. Note that this API only allows you to set the + * arguments in their order of appearance. */ +int64_t task_args_add_ref(task_spec *spec, object_id obj_id); +int64_t task_args_add_val(task_spec *spec, uint8_t *data, int64_t length); + +/* Getting and setting return arguments. Tasks return by reference for now. */ +unique_id *task_return(task_spec *spec, int64_t ret_index); + +/* Freeing the task datastructure. */ +void free_task_spec(task_spec *spec); + +/* Write the task specification to a file or socket. */ +void write_task(int fd, task_spec *spec); + +/* Read the task specification from a file or socket. It is the user's + * responsibility to free the task after it has been used. */ +task_spec *read_task(int fd); + +/* Print task as a humanly readable string. */ +void print_task(task_spec *spec, UT_string *output); + +/* + * SCHEDULED TASK: Contains information about a scheduled task: + * the task iid, the task specification and the task status + * (WAITING, SCHEDULED, RUNNING, DONE) and which node the + * task is scheduled on. + * + */ + +/* The scheduling_state can be used as a flag when we are listening + * for an event, for example TASK_WAITING | TASK_SCHEDULED. */ +enum scheduling_state { + TASK_STATUS_WAITING = 1, + TASK_STATUS_SCHEDULED = 2, + TASK_STATUS_RUNNING = 4, + TASK_STATUS_DONE = 8 +}; + +/* A task instance is one execution of a task specification. + * It has a unique instance id, a state of execution (see scheduling_state) + * and a node it is scheduled on or running on. */ +typedef struct task_instance_impl task_instance; + +/* Allocate and initialize a new task instance. Must be freed with + * scheduled_task_free after use. */ +task_instance *make_task_instance(task_iid task_iid, + task_spec *task, + int32_t state, + node_id node); + +/* Size of task instance structure in bytes. */ +int64_t task_instance_size(task_instance *instance); + +/* Instance ID of the task instance. */ +task_iid *task_instance_id(task_instance *instance); + +/* The scheduling state of the task instance. */ +int32_t *task_instance_state(task_instance *instance); + +/* Node this task instance has been assigned to or is running on. */ +node_id *task_instance_node(task_instance *instance); + +/* Task specification of this task instance. */ +task_spec *task_instance_task_spec(task_instance *instance); + +/* Free this task instance datastructure. */ +void task_instance_free(task_instance *instance); + +#endif diff --git a/src/common/test/common_tests.c b/src/common/test/common_tests.c new file mode 100644 index 000000000..47b643039 --- /dev/null +++ b/src/common/test/common_tests.c @@ -0,0 +1,24 @@ +#include "greatest.h" + +#include "common.h" + +SUITE(common_tests); + +TEST sha1_test(void) { + static char hex[2 * UNIQUE_ID_SIZE + 1]; + unique_id uid = globally_unique_id(); + sha1_to_hex(&uid.id[0], &hex[0]); + PASS(); +} + +SUITE(common_tests) { + RUN_TEST(sha1_test); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(common_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/db_tests.c b/src/common/test/db_tests.c new file mode 100644 index 000000000..74dd9a850 --- /dev/null +++ b/src/common/test/db_tests.c @@ -0,0 +1,175 @@ +#include "greatest.h" + +#include +#include +#include + +#include "event_loop.h" +#include "test/example_task.h" +#include "state/db.h" +#include "state/object_table.h" +#include "state/task_log.h" +#include "state/redis.h" +#include "task.h" + +SUITE(db_tests); + +const char *manager_addr = "127.0.0.1"; +int manager_port1 = 12345; +int manager_port2 = 12346; +char received_addr1[16] = {0}; +char received_port1[6] = {0}; +char received_addr2[16] = {0}; +char received_port2[6] = {0}; + +/* Test if entries have been written to the database. */ +void test_callback(object_id object_id, + int manager_count, + const char *manager_vector[], + void *context) { + CHECK(manager_count == 2); + if (!manager_vector[0] || + sscanf(manager_vector[0], "%15[0-9.]:%5[0-9]", received_addr1, + received_port1) != 2) { + CHECK(0); + } + if (!manager_vector[1] || + sscanf(manager_vector[1], "%15[0-9.]:%5[0-9]", received_addr2, + received_port2) != 2) { + CHECK(0); + } + free(manager_vector); +} + +int timeout_handler(event_loop *loop, timer_id timer_id, void *context) { + event_loop_stop(loop); + return EVENT_LOOP_TIMER_DONE; +} + +TEST object_table_lookup_test(void) { + event_loop *loop = event_loop_create(); + db_handle *db1 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr, + manager_port1); + db_handle *db2 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr, + manager_port2); + db_attach(db1, loop); + db_attach(db2, loop); + unique_id id = globally_unique_id(); + object_table_add(db1, id); + object_table_add(db2, id); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + object_table_lookup(db1, id, test_callback, NULL); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + int port1 = atoi(received_port1); + int port2 = atoi(received_port2); + ASSERT_STR_EQ(&received_addr1[0], manager_addr); + ASSERT((port1 == manager_port1 && port2 == manager_port2) || + (port2 == manager_port1 && port1 == manager_port2)); + + db_disconnect(db1); + db_disconnect(db2); + + event_loop_destroy(loop); + PASS(); +} + +void task_log_test_callback(task_instance *instance, void *userdata) { + task_instance *other = userdata; + CHECK(*task_instance_state(instance) == TASK_STATUS_SCHEDULED); + CHECK(task_instance_size(instance) == task_instance_size(other)); + CHECK(memcmp(instance, other, task_instance_size(instance)) == 0); +} + +TEST task_log_test(void) { + event_loop *loop = event_loop_create(); + db_handle *db = db_connect("127.0.0.1", 6379, "local_scheduler", "", -1); + db_attach(db, loop); + node_id node = globally_unique_id(); + task_spec *task = example_task(); + task_instance *instance = make_task_instance(globally_unique_id(), task, + TASK_STATUS_SCHEDULED, node); + task_log_register_callback(db, task_log_test_callback, node, + TASK_STATUS_SCHEDULED, instance); + task_log_add_task(db, instance); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + task_instance_free(instance); + free_task_spec(task); + db_disconnect(db); + event_loop_destroy(loop); + PASS(); +} + +int num_test_callback_called = 0; + +void task_log_all_test_callback(task_instance *instance, void *userdata) { + num_test_callback_called += 1; +} + +TEST task_log_all_test(void) { + event_loop *loop = event_loop_create(); + db_handle *db = db_connect("127.0.0.1", 6379, "local_scheduler", "", -1); + db_attach(db, loop); + task_spec *task = example_task(); + /* Schedule two tasks on different nodes. */ + task_instance *instance1 = make_task_instance( + globally_unique_id(), task, TASK_STATUS_SCHEDULED, globally_unique_id()); + task_instance *instance2 = make_task_instance( + globally_unique_id(), task, TASK_STATUS_SCHEDULED, globally_unique_id()); + task_log_register_callback(db, task_log_all_test_callback, NIL_ID, + TASK_STATUS_SCHEDULED, NULL); + task_log_add_task(db, instance1); + task_log_add_task(db, instance2); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + task_instance_free(instance2); + task_instance_free(instance1); + free_task_spec(task); + db_disconnect(db); + event_loop_destroy(loop); + ASSERT(num_test_callback_called == 2); + PASS(); +} + +TEST unique_client_id_test(void) { + const int num_conns = 50; + + db_handle *db; + pid_t pid = fork(); + for (int i = 0; i < num_conns; ++i) { + db = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr, + manager_port1); + db_disconnect(db); + } + if (pid == 0) { + exit(0); + } else { + wait(NULL); + } + + db = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr, + manager_port1); + ASSERT_EQ(get_client_id(db), num_conns * 2); + db_disconnect(db); + PASS(); +} + +SUITE(db_tests) { + redisContext *context = redisConnect("127.0.0.1", 6379); + freeReplyObject(redisCommand(context, "FLUSHALL")); + RUN_REDIS_TEST(context, object_table_lookup_test); + RUN_REDIS_TEST(context, task_log_test); + RUN_REDIS_TEST(context, task_log_all_test); + RUN_REDIS_TEST(context, unique_client_id_test); + redisFree(context); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(db_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/example_task.h b/src/common/test/example_task.h new file mode 100644 index 000000000..0dddc4dc1 --- /dev/null +++ b/src/common/test/example_task.h @@ -0,0 +1,14 @@ +#ifndef EXAMPLE_TASK_H +#define EXAMPLE_TASK_H + +#include "task.h" + +task_spec *example_task(void) { + function_id func_id = globally_unique_id(); + task_spec *task = alloc_task_spec(func_id, 2, 1, 0); + task_args_add_ref(task, globally_unique_id()); + task_args_add_ref(task, globally_unique_id()); + return task; +} + +#endif diff --git a/src/common/test/io_tests.c b/src/common/test/io_tests.c new file mode 100644 index 000000000..56ebf0607 --- /dev/null +++ b/src/common/test/io_tests.c @@ -0,0 +1,106 @@ +#include "greatest.h" + +#include +#include +#include + +#include "io.h" +#include "utstring.h" + +SUITE(io_tests); + +TEST ipc_socket_test(void) { + const char *socket_pathname = "test-socket"; + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + + char *test_string = "hello world"; + char *test_bytes = "another string"; + pid_t pid = fork(); + if (pid == 0) { + close(socket_fd); + socket_fd = connect_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + write_log_message(socket_fd, test_string); + write_message(socket_fd, LOG_MESSAGE, strlen(test_bytes), + (uint8_t *) test_bytes); + close(socket_fd); + exit(0); + } else { + int client_fd = accept_client(socket_fd); + ASSERT(client_fd >= 0); + char *message = read_log_message(client_fd); + ASSERT(message != NULL); + ASSERT_STR_EQ(test_string, message); + free(message); + int64_t type; + int64_t len; + uint8_t *bytes; + read_message(client_fd, &type, &len, &bytes); + ASSERT(type == LOG_MESSAGE); + ASSERT(memcmp(test_bytes, bytes, len) == 0); + free(bytes); + close(client_fd); + close(socket_fd); + unlink(socket_pathname); + } + + PASS(); +} + +TEST long_ipc_socket_test(void) { + const char *socket_pathname = "long-test-socket"; + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + + UT_string *test_string; + utstring_new(test_string); + for (int i = 0; i < 10000; i++) { + utstring_printf(test_string, "hello world "); + } + char *test_bytes = "another string"; + pid_t pid = fork(); + if (pid == 0) { + close(socket_fd); + socket_fd = connect_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + write_log_message(socket_fd, utstring_body(test_string)); + write_message(socket_fd, LOG_MESSAGE, strlen(test_bytes), + (uint8_t *) test_bytes); + close(socket_fd); + exit(0); + } else { + int client_fd = accept_client(socket_fd); + ASSERT(client_fd >= 0); + char *message = read_log_message(client_fd); + ASSERT(message != NULL); + ASSERT_STR_EQ(utstring_body(test_string), message); + free(message); + int64_t type; + int64_t len; + uint8_t *bytes; + read_message(client_fd, &type, &len, &bytes); + ASSERT(type == LOG_MESSAGE); + ASSERT(memcmp(test_bytes, bytes, len) == 0); + free(bytes); + close(client_fd); + close(socket_fd); + unlink(socket_pathname); + } + + utstring_free(test_string); + PASS(); +} + +SUITE(io_tests) { + RUN_TEST(ipc_socket_test); + RUN_TEST(long_ipc_socket_test); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(io_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/redis_tests.c b/src/common/test/redis_tests.c new file mode 100644 index 000000000..0a50c7d1b --- /dev/null +++ b/src/common/test/redis_tests.c @@ -0,0 +1,225 @@ +#include "greatest.h" + +#include +#include + +#include "utarray.h" + +#include "event_loop.h" +#include "state/db.h" +#include "state/redis.h" +#include "io.h" +#include "logging.h" + +SUITE(redis_tests); + +const char *test_set_format = "SET %s %s"; +const char *test_get_format = "GET %s"; +const char *test_key = "foo"; +const char *test_value = "bar"; +UT_array *connections = NULL; + +int async_redis_socket_test_callback_called = 0; + +void async_redis_socket_test_callback(redisAsyncContext *ac, + void *r, + void *privdata) { + async_redis_socket_test_callback_called = 1; + redisContext *context = redisConnect("127.0.0.1", 6379); + redisReply *reply = redisCommand(context, test_get_format, test_key); + redisFree(context); + CHECK(reply != NULL); + if (strcmp(reply->str, test_value)) { + freeReplyObject(reply); + CHECK(0); + } + freeReplyObject(reply); +} + +TEST redis_socket_test(void) { + const char *socket_pathname = "redis-test-socket"; + redisContext *context = redisConnect("127.0.0.1", 6379); + ASSERT(context != NULL); + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + + int client_fd = connect_ipc_sock(socket_pathname); + ASSERT(client_fd >= 0); + write_formatted_log_message(client_fd, test_set_format, test_key, test_value); + + int server_fd = accept_client(socket_fd); + char *cmd = read_log_message(server_fd); + close(client_fd); + close(server_fd); + close(socket_fd); + unlink(socket_pathname); + + redisReply *reply; + reply = redisCommand(context, cmd, 0, 0); + freeReplyObject(reply); + reply = redisCommand(context, "GET %s", test_key); + ASSERT(reply != NULL); + ASSERT_STR_EQ(reply->str, test_value); + freeReplyObject(reply); + + free(cmd); + redisFree(context); + PASS(); +} + +void redis_read_callback(event_loop *loop, int fd, void *context, int events) { + db_handle *db = context; + char *cmd = read_log_message(fd); + redisAsyncCommand(db->context, async_redis_socket_test_callback, NULL, cmd, + db->client_id, 0); + free(cmd); +} + +void redis_accept_callback(event_loop *loop, + int socket_fd, + void *context, + int events) { + int accept_fd = accept_client(socket_fd); + CHECK(accept_fd >= 0); + utarray_push_back(connections, &accept_fd); + event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, redis_read_callback, + context); +} + +int timeout_handler(event_loop *loop, timer_id timer_id, void *context) { + event_loop_stop(loop); + return EVENT_LOOP_TIMER_DONE; +} + +TEST async_redis_socket_test(void) { + utarray_new(connections, &ut_int_icd); + event_loop *loop = event_loop_create(); + + /* Start IPC channel. */ + const char *socket_pathname = "async-redis-test-socket"; + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + utarray_push_back(connections, &socket_fd); + + /* Start connection to Redis. */ + db_handle *db = db_connect("127.0.0.1", 6379, "", "", 0); + db_attach(db, loop); + + /* Send a command to the Redis process. */ + int client_fd = connect_ipc_sock(socket_pathname); + ASSERT(client_fd >= 0); + utarray_push_back(connections, &client_fd); + write_formatted_log_message(client_fd, test_set_format, test_key, test_value); + + event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, redis_read_callback, + db); + event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, redis_accept_callback, + db); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + + CHECK(async_redis_socket_test_callback_called); + + db_disconnect(db); + event_loop_destroy(loop); + for (int *p = (int *) utarray_front(connections); p != NULL; + p = (int *) utarray_next(connections, p)) { + close(*p); + } + unlink(socket_pathname); + utarray_free(connections); + PASS(); +} + +int logging_test_callback_called = 0; + +void logging_test_callback(redisAsyncContext *ac, void *r, void *privdata) { + logging_test_callback_called = 1; + redisContext *context = redisConnect("127.0.0.1", 6379); + redisReply *reply = redisCommand(context, "KEYS %s", "log:*"); + redisFree(context); + CHECK(reply != NULL); + CHECK(reply->elements > 0); + freeReplyObject(reply); +} + +void logging_read_callback(event_loop *loop, + int fd, + void *context, + int events) { + db_handle *conn = context; + char *cmd = read_log_message(fd); + redisAsyncCommand(conn->context, logging_test_callback, NULL, cmd, + conn->client_id, 0); + free(cmd); +} + +void logging_accept_callback(event_loop *loop, + int socket_fd, + void *context, + int events) { + int accept_fd = accept_client(socket_fd); + CHECK(accept_fd >= 0); + utarray_push_back(connections, &accept_fd); + event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, logging_read_callback, + context); +} + +TEST logging_test(void) { + utarray_new(connections, &ut_int_icd); + event_loop *loop = event_loop_create(); + + /* Start IPC channel. */ + const char *socket_pathname = "logging-test-socket"; + int socket_fd = bind_ipc_sock(socket_pathname); + ASSERT(socket_fd >= 0); + utarray_push_back(connections, &socket_fd); + + /* Start connection to Redis. */ + db_handle *conn = db_connect("127.0.0.1", 6379, "", "", 0); + db_attach(conn, loop); + + /* Send a command to the Redis process. */ + int client_fd = connect_ipc_sock(socket_pathname); + ASSERT(client_fd >= 0); + utarray_push_back(connections, &client_fd); + ray_logger *logger = init_ray_logger("worker", RAY_INFO, 0, &client_fd); + ray_log(logger, RAY_INFO, "TEST", "Message"); + + event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, logging_accept_callback, + conn); + event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, logging_read_callback, + conn); + event_loop_add_timer(loop, 100, timeout_handler, NULL); + event_loop_run(loop); + + CHECK(logging_test_callback_called); + + free_ray_logger(logger); + db_disconnect(conn); + event_loop_destroy(loop); + for (int *p = (int *) utarray_front(connections); p != NULL; + p = (int *) utarray_next(connections, p)) { + close(*p); + } + unlink(socket_pathname); + utarray_free(connections); + PASS(); +} + +SUITE(redis_tests) { + redisContext *context = redisConnect("127.0.0.1", 6379); + freeReplyObject(redisCommand(context, "FLUSHALL")); + RUN_REDIS_TEST(context, redis_socket_test); + RUN_REDIS_TEST(context, async_redis_socket_test); + RUN_REDIS_TEST(context, logging_test); + redisFree(context); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(redis_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/task_tests.c b/src/common/test/task_tests.c new file mode 100644 index 000000000..77f43bbf0 --- /dev/null +++ b/src/common/test/task_tests.c @@ -0,0 +1,77 @@ +#include "greatest.h" + +#include +#include +#include + +#include "common.h" +#include "test/example_task.h" +#include "task.h" +#include "io.h" + +SUITE(task_tests); + +TEST task_test(void) { + function_id func_id = globally_unique_id(); + task_spec *task = alloc_task_spec(func_id, 4, 2, 10); + ASSERT(task_num_args(task) == 4); + ASSERT(task_num_returns(task) == 2); + + unique_id arg1 = globally_unique_id(); + ASSERT(task_args_add_ref(task, arg1) == 0); + ASSERT(task_args_add_val(task, (uint8_t *) "hello", 5) == 1); + unique_id arg2 = globally_unique_id(); + ASSERT(task_args_add_ref(task, arg2) == 2); + ASSERT(task_args_add_val(task, (uint8_t *) "world", 5) == 3); + + unique_id ret0 = globally_unique_id(); + unique_id ret1 = globally_unique_id(); + memcpy(task_return(task, 0), &ret0, sizeof(ret0)); + memcpy(task_return(task, 1), &ret1, sizeof(ret1)); + + ASSERT(memcmp(task_arg_id(task, 0), &arg1, sizeof(arg1)) == 0); + ASSERT(memcmp(task_arg_val(task, 1), (uint8_t *) "hello", + task_arg_length(task, 1)) == 0); + ASSERT(memcmp(task_arg_id(task, 2), &arg2, sizeof(arg2)) == 0); + ASSERT(memcmp(task_arg_val(task, 3), (uint8_t *) "world", + task_arg_length(task, 3)) == 0); + + ASSERT(memcmp(task_return(task, 0), &ret0, sizeof(unique_id)) == 0); + ASSERT(memcmp(task_return(task, 1), &ret1, sizeof(unique_id)) == 0); + + free_task_spec(task); + PASS(); +} + +TEST send_task(void) { + function_id func_id = globally_unique_id(); + task_spec *task = alloc_task_spec(func_id, 4, 2, 10); + *task_return(task, 1) = globally_unique_id(); + int fd[2]; + socketpair(AF_UNIX, SOCK_STREAM, 0, fd); + write_message(fd[0], SUBMIT_TASK, task_size(task), (uint8_t *) task); + int64_t type; + int64_t length; + uint8_t *message; + read_message(fd[1], &type, &length, &message); + task_spec *result = (task_spec *) message; + ASSERT(type == SUBMIT_TASK); + ASSERT(memcmp(task, result, task_size(task)) == 0); + ASSERT(memcmp(task, result, task_size(result)) == 0); + free(task); + free(result); + PASS(); +} + +SUITE(task_tests) { + RUN_TEST(task_test); + RUN_TEST(send_task); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); + RUN_SUITE(task_tests); + GREATEST_MAIN_END(); +} diff --git a/src/common/test/test.py b/src/common/test/test.py new file mode 100644 index 000000000..359e8c030 --- /dev/null +++ b/src/common/test/test.py @@ -0,0 +1,119 @@ +from __future__ import print_function + +import pickle +import unittest + +import common + +BASE_SIMPLE_OBJECTS = [ + 0, 1, 100000, 0L, 1L, 100000L, 1L << 100, 0.0, 0.5, 0.9, 100000.1, (), [], {}, + "", 990 * "h", u"", 990 * u"h" +] + +LIST_SIMPLE_OBJECTS = [[obj] for obj in BASE_SIMPLE_OBJECTS] +TUPLE_SIMPLE_OBJECTS = [(obj,) for obj in BASE_SIMPLE_OBJECTS] +DICT_SIMPLE_OBJECTS = [{(): obj} for obj in BASE_SIMPLE_OBJECTS] + +SIMPLE_OBJECTS = (BASE_SIMPLE_OBJECTS + + LIST_SIMPLE_OBJECTS + + TUPLE_SIMPLE_OBJECTS + + DICT_SIMPLE_OBJECTS) + +# Create some complex objects that cannot be serialized by value in tasks. + +l = [] +l.append(l) + +class Foo(object): + def __init__(self): + pass + +BASE_COMPLEX_OBJECTS = [999 * "h", 999 * u"h", l, Foo(), 10 * [10 * [10 * [1]]]] + +LIST_COMPLEX_OBJECTS = [[obj] for obj in BASE_COMPLEX_OBJECTS] +TUPLE_COMPLEX_OBJECTS = [(obj,) for obj in BASE_COMPLEX_OBJECTS] +DICT_COMPLEX_OBJECTS = [{(): obj} for obj in BASE_COMPLEX_OBJECTS] + +COMPLEX_OBJECTS = (BASE_COMPLEX_OBJECTS + + LIST_COMPLEX_OBJECTS + + TUPLE_COMPLEX_OBJECTS + + DICT_COMPLEX_OBJECTS) + +class TestSerialization(unittest.TestCase): + + def test_serialize_by_value(self): + + for val in SIMPLE_OBJECTS: + self.assertTrue(common.check_simple_value(val)) + for val in COMPLEX_OBJECTS: + self.assertFalse(common.check_simple_value(val)) + +class TestObjectID(unittest.TestCase): + + def test_create_object_id(self): + object_id = common.ObjectID(20 * "a") + + def test_cannot_pickle_object_ids(self): + object_ids = [common.ObjectID(20 * chr(i)) for i in range(256)] + def f(): + return object_ids + def g(val=object_ids): + return 1 + def h(): + x = object_ids[0] + return 1 + # Make sure that object IDs cannot be pickled (including functions that + # close over object IDs). + self.assertRaises(Exception, lambda : pickling.dumps(object_ids[0])) + self.assertRaises(Exception, lambda : pickling.dumps(object_ids)) + self.assertRaises(Exception, lambda : pickling.dumps(f)) + self.assertRaises(Exception, lambda : pickling.dumps(g)) + self.assertRaises(Exception, lambda : pickling.dumps(h)) + +class TestTask(unittest.TestCase): + + def test_create_task(self): + # TODO(rkn): The function ID should be a FunctionID object, not an ObjectID. + function_id = common.ObjectID(20 * "a") + object_ids = [common.ObjectID(20 * chr(i)) for i in range(256)] + args_list = [ + [], + 1 * [1], + 10 * [1], + 100 * [1], + 1000 * [1], + 1 * ["a"], + 10 * ["a"], + 100 * ["a"], + 1000 * ["a"], + [1, 1.3, 2L, 1L << 100, "hi", u"hi", [1, 2]], + object_ids[:1], + object_ids[:2], + object_ids[:3], + object_ids[:4], + object_ids[:5], + object_ids[:10], + object_ids[:100], + object_ids[:256], + [1, object_ids[0]], + [object_ids[0], "a"], + [1, object_ids[0], "a"], + [object_ids[0], 1, object_ids[1], "a"], + object_ids[:3] + [1, "hi", 2.3] + object_ids[:5], + object_ids + 100 * ["a"] + object_ids + ] + for args in args_list: + for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: + task = common.Task(function_id, args, num_return_vals) + self.assertEqual(function_id.id(), task.function_id().id()) + retrieved_args = task.arguments() + self.assertEqual(num_return_vals, len(task.returns())) + self.assertEqual(len(args), len(retrieved_args)) + for i in range(len(retrieved_args)): + if isinstance(retrieved_args[i], common.ObjectID): + self.assertEqual(retrieved_args[i].id(), args[i].id()) + else: + self.assertEqual(retrieved_args[i], args[i]) + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/src/common/thirdparty/ae/ae.c b/src/common/thirdparty/ae/ae.c new file mode 100644 index 000000000..e66808a81 --- /dev/null +++ b/src/common/thirdparty/ae/ae.c @@ -0,0 +1,465 @@ +/* A simple event-driven programming library. Originally I wrote this code + * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated + * it in form of a library for easy reuse. + * + * Copyright (c) 2006-2010, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ae.h" +#include "zmalloc.h" +#include "config.h" + +/* Include the best multiplexing layer supported by this system. + * The following should be ordered by performances, descending. */ +#ifdef HAVE_EVPORT +#include "ae_evport.c" +#else + #ifdef HAVE_EPOLL + #include "ae_epoll.c" + #else + #ifdef HAVE_KQUEUE + #include "ae_kqueue.c" + #else + #include "ae_select.c" + #endif + #endif +#endif + +aeEventLoop *aeCreateEventLoop(int setsize) { + aeEventLoop *eventLoop; + int i; + + if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err; + eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize); + eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize); + if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err; + eventLoop->setsize = setsize; + eventLoop->lastTime = time(NULL); + eventLoop->timeEventHead = NULL; + eventLoop->timeEventNextId = 0; + eventLoop->stop = 0; + eventLoop->maxfd = -1; + eventLoop->beforesleep = NULL; + if (aeApiCreate(eventLoop) == -1) goto err; + /* Events with mask == AE_NONE are not set. So let's initialize the + * vector with it. */ + for (i = 0; i < setsize; i++) + eventLoop->events[i].mask = AE_NONE; + return eventLoop; + +err: + if (eventLoop) { + zfree(eventLoop->events); + zfree(eventLoop->fired); + zfree(eventLoop); + } + return NULL; +} + +/* Return the current set size. */ +int aeGetSetSize(aeEventLoop *eventLoop) { + return eventLoop->setsize; +} + +/* Resize the maximum set size of the event loop. + * If the requested set size is smaller than the current set size, but + * there is already a file descriptor in use that is >= the requested + * set size minus one, AE_ERR is returned and the operation is not + * performed at all. + * + * Otherwise AE_OK is returned and the operation is successful. */ +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { + int i; + + if (setsize == eventLoop->setsize) return AE_OK; + if (eventLoop->maxfd >= setsize) return AE_ERR; + if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR; + + eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize); + eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize); + eventLoop->setsize = setsize; + + /* Make sure that if we created new slots, they are initialized with + * an AE_NONE mask. */ + for (i = eventLoop->maxfd+1; i < setsize; i++) + eventLoop->events[i].mask = AE_NONE; + return AE_OK; +} + +void aeDeleteEventLoop(aeEventLoop *eventLoop) { + aeApiFree(eventLoop); + zfree(eventLoop->events); + zfree(eventLoop->fired); + zfree(eventLoop); +} + +void aeStop(aeEventLoop *eventLoop) { + eventLoop->stop = 1; +} + +int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, + aeFileProc *proc, void *clientData) +{ + if (fd >= eventLoop->setsize) { + errno = ERANGE; + return AE_ERR; + } + aeFileEvent *fe = &eventLoop->events[fd]; + + if (aeApiAddEvent(eventLoop, fd, mask) == -1) + return AE_ERR; + fe->mask |= mask; + if (mask & AE_READABLE) fe->rfileProc = proc; + if (mask & AE_WRITABLE) fe->wfileProc = proc; + fe->clientData = clientData; + if (fd > eventLoop->maxfd) + eventLoop->maxfd = fd; + return AE_OK; +} + +void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) +{ + if (fd >= eventLoop->setsize) return; + aeFileEvent *fe = &eventLoop->events[fd]; + if (fe->mask == AE_NONE) return; + + aeApiDelEvent(eventLoop, fd, mask); + fe->mask = fe->mask & (~mask); + if (fd == eventLoop->maxfd && fe->mask == AE_NONE) { + /* Update the max fd */ + int j; + + for (j = eventLoop->maxfd-1; j >= 0; j--) + if (eventLoop->events[j].mask != AE_NONE) break; + eventLoop->maxfd = j; + } +} + +int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { + if (fd >= eventLoop->setsize) return 0; + aeFileEvent *fe = &eventLoop->events[fd]; + + return fe->mask; +} + +static void aeGetTime(long *seconds, long *milliseconds) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + *seconds = tv.tv_sec; + *milliseconds = tv.tv_usec/1000; +} + +static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) { + long cur_sec, cur_ms, when_sec, when_ms; + + aeGetTime(&cur_sec, &cur_ms); + when_sec = cur_sec + milliseconds/1000; + when_ms = cur_ms + milliseconds%1000; + if (when_ms >= 1000) { + when_sec ++; + when_ms -= 1000; + } + *sec = when_sec; + *ms = when_ms; +} + +long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, + aeTimeProc *proc, void *clientData, + aeEventFinalizerProc *finalizerProc) +{ + long long id = eventLoop->timeEventNextId++; + aeTimeEvent *te; + + te = zmalloc(sizeof(*te)); + if (te == NULL) return AE_ERR; + te->id = id; + aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms); + te->timeProc = proc; + te->finalizerProc = finalizerProc; + te->clientData = clientData; + te->next = eventLoop->timeEventHead; + eventLoop->timeEventHead = te; + return id; +} + +int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) +{ + aeTimeEvent *te = eventLoop->timeEventHead; + while(te) { + if (te->id == id) { + te->id = AE_DELETED_EVENT_ID; + return AE_OK; + } + te = te->next; + } + return AE_ERR; /* NO event with the specified ID found */ +} + +/* Search the first timer to fire. + * This operation is useful to know how many time the select can be + * put in sleep without to delay any event. + * If there are no timers NULL is returned. + * + * Note that's O(N) since time events are unsorted. + * Possible optimizations (not needed by Redis so far, but...): + * 1) Insert the event in order, so that the nearest is just the head. + * Much better but still insertion or deletion of timers is O(N). + * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)). + */ +static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop) +{ + aeTimeEvent *te = eventLoop->timeEventHead; + aeTimeEvent *nearest = NULL; + + while(te) { + if (!nearest || te->when_sec < nearest->when_sec || + (te->when_sec == nearest->when_sec && + te->when_ms < nearest->when_ms)) + nearest = te; + te = te->next; + } + return nearest; +} + +/* Process time events */ +static int processTimeEvents(aeEventLoop *eventLoop) { + int processed = 0; + aeTimeEvent *te, *prev; + long long maxId; + time_t now = time(NULL); + + /* If the system clock is moved to the future, and then set back to the + * right value, time events may be delayed in a random way. Often this + * means that scheduled operations will not be performed soon enough. + * + * Here we try to detect system clock skews, and force all the time + * events to be processed ASAP when this happens: the idea is that + * processing events earlier is less dangerous than delaying them + * indefinitely, and practice suggests it is. */ + if (now < eventLoop->lastTime) { + te = eventLoop->timeEventHead; + while(te) { + te->when_sec = 0; + te = te->next; + } + } + eventLoop->lastTime = now; + + prev = NULL; + te = eventLoop->timeEventHead; + maxId = eventLoop->timeEventNextId-1; + while(te) { + long now_sec, now_ms; + long long id; + + /* Remove events scheduled for deletion. */ + if (te->id == AE_DELETED_EVENT_ID) { + aeTimeEvent *next = te->next; + if (prev == NULL) + eventLoop->timeEventHead = te->next; + else + prev->next = te->next; + if (te->finalizerProc) + te->finalizerProc(eventLoop, te->clientData); + zfree(te); + te = next; + continue; + } + + /* Make sure we don't process time events created by time events in + * this iteration. Note that this check is currently useless: we always + * add new timers on the head, however if we change the implementation + * detail, this check may be useful again: we keep it here for future + * defense. */ + if (te->id > maxId) { + te = te->next; + continue; + } + aeGetTime(&now_sec, &now_ms); + if (now_sec > te->when_sec || + (now_sec == te->when_sec && now_ms >= te->when_ms)) + { + int retval; + + id = te->id; + retval = te->timeProc(eventLoop, id, te->clientData); + processed++; + if (retval != AE_NOMORE) { + aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms); + } else { + te->id = AE_DELETED_EVENT_ID; + } + } + prev = te; + te = te->next; + } + return processed; +} + +/* Process every pending time event, then every pending file event + * (that may be registered by time event callbacks just processed). + * Without special flags the function sleeps until some file event + * fires, or when the next time event occurs (if any). + * + * If flags is 0, the function does nothing and returns. + * if flags has AE_ALL_EVENTS set, all the kind of events are processed. + * if flags has AE_FILE_EVENTS set, file events are processed. + * if flags has AE_TIME_EVENTS set, time events are processed. + * if flags has AE_DONT_WAIT set the function returns ASAP until all + * the events that's possible to process without to wait are processed. + * + * The function returns the number of events processed. */ +int aeProcessEvents(aeEventLoop *eventLoop, int flags) +{ + int processed = 0, numevents; + + /* Nothing to do? return ASAP */ + if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0; + + /* Note that we want call select() even if there are no + * file events to process as long as we want to process time + * events, in order to sleep until the next time event is ready + * to fire. */ + if (eventLoop->maxfd != -1 || + ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) { + int j; + aeTimeEvent *shortest = NULL; + struct timeval tv, *tvp; + + if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT)) + shortest = aeSearchNearestTimer(eventLoop); + if (shortest) { + long now_sec, now_ms; + + aeGetTime(&now_sec, &now_ms); + tvp = &tv; + + /* How many milliseconds we need to wait for the next + * time event to fire? */ + long long ms = + (shortest->when_sec - now_sec)*1000 + + shortest->when_ms - now_ms; + + if (ms > 0) { + tvp->tv_sec = ms/1000; + tvp->tv_usec = (ms % 1000)*1000; + } else { + tvp->tv_sec = 0; + tvp->tv_usec = 0; + } + } else { + /* If we have to check for events but need to return + * ASAP because of AE_DONT_WAIT we need to set the timeout + * to zero */ + if (flags & AE_DONT_WAIT) { + tv.tv_sec = tv.tv_usec = 0; + tvp = &tv; + } else { + /* Otherwise we can block */ + tvp = NULL; /* wait forever */ + } + } + + numevents = aeApiPoll(eventLoop, tvp); + for (j = 0; j < numevents; j++) { + aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd]; + int mask = eventLoop->fired[j].mask; + int fd = eventLoop->fired[j].fd; + int rfired = 0; + + /* note the fe->mask & mask & ... code: maybe an already processed + * event removed an element that fired and we still didn't + * processed, so we check if the event is still valid. */ + if (fe->mask & mask & AE_READABLE) { + rfired = 1; + fe->rfileProc(eventLoop,fd,fe->clientData,mask); + } + if (fe->mask & mask & AE_WRITABLE) { + if (!rfired || fe->wfileProc != fe->rfileProc) + fe->wfileProc(eventLoop,fd,fe->clientData,mask); + } + processed++; + } + } + /* Check time events */ + if (flags & AE_TIME_EVENTS) + processed += processTimeEvents(eventLoop); + + return processed; /* return the number of processed file/time events */ +} + +/* Wait for milliseconds until the given file descriptor becomes + * writable/readable/exception */ +int aeWait(int fd, int mask, long long milliseconds) { + struct pollfd pfd; + int retmask = 0, retval; + + memset(&pfd, 0, sizeof(pfd)); + pfd.fd = fd; + if (mask & AE_READABLE) pfd.events |= POLLIN; + if (mask & AE_WRITABLE) pfd.events |= POLLOUT; + + if ((retval = poll(&pfd, 1, milliseconds))== 1) { + if (pfd.revents & POLLIN) retmask |= AE_READABLE; + if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE; + if (pfd.revents & POLLERR) retmask |= AE_WRITABLE; + if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE; + return retmask; + } else { + return retval; + } +} + +void aeMain(aeEventLoop *eventLoop) { + eventLoop->stop = 0; + while (!eventLoop->stop) { + if (eventLoop->beforesleep != NULL) + eventLoop->beforesleep(eventLoop); + aeProcessEvents(eventLoop, AE_ALL_EVENTS); + } +} + +char *aeGetApiName(void) { + return aeApiName(); +} + +void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) { + eventLoop->beforesleep = beforesleep; +} diff --git a/src/common/thirdparty/ae/ae.h b/src/common/thirdparty/ae/ae.h new file mode 100644 index 000000000..827c4c9e4 --- /dev/null +++ b/src/common/thirdparty/ae/ae.h @@ -0,0 +1,123 @@ +/* A simple event-driven programming library. Originally I wrote this code + * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated + * it in form of a library for easy reuse. + * + * Copyright (c) 2006-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __AE_H__ +#define __AE_H__ + +#include + +#define AE_OK 0 +#define AE_ERR -1 + +#define AE_NONE 0 +#define AE_READABLE 1 +#define AE_WRITABLE 2 + +#define AE_FILE_EVENTS 1 +#define AE_TIME_EVENTS 2 +#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS) +#define AE_DONT_WAIT 4 + +#define AE_NOMORE -1 +#define AE_DELETED_EVENT_ID -1 + +/* Macros */ +#define AE_NOTUSED(V) ((void) V) + +struct aeEventLoop; + +/* Types and data structures */ +typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask); +typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData); +typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData); +typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop); + +/* File event structure */ +typedef struct aeFileEvent { + int mask; /* one of AE_(READABLE|WRITABLE) */ + aeFileProc *rfileProc; + aeFileProc *wfileProc; + void *clientData; +} aeFileEvent; + +/* Time event structure */ +typedef struct aeTimeEvent { + long long id; /* time event identifier. */ + long when_sec; /* seconds */ + long when_ms; /* milliseconds */ + aeTimeProc *timeProc; + aeEventFinalizerProc *finalizerProc; + void *clientData; + struct aeTimeEvent *next; +} aeTimeEvent; + +/* A fired event */ +typedef struct aeFiredEvent { + int fd; + int mask; +} aeFiredEvent; + +/* State of an event based program */ +typedef struct aeEventLoop { + int maxfd; /* highest file descriptor currently registered */ + int setsize; /* max number of file descriptors tracked */ + long long timeEventNextId; + time_t lastTime; /* Used to detect system clock skew */ + aeFileEvent *events; /* Registered events */ + aeFiredEvent *fired; /* Fired events */ + aeTimeEvent *timeEventHead; + int stop; + void *apidata; /* This is used for polling API specific data */ + aeBeforeSleepProc *beforesleep; +} aeEventLoop; + +/* Prototypes */ +aeEventLoop *aeCreateEventLoop(int setsize); +void aeDeleteEventLoop(aeEventLoop *eventLoop); +void aeStop(aeEventLoop *eventLoop); +int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, + aeFileProc *proc, void *clientData); +void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask); +int aeGetFileEvents(aeEventLoop *eventLoop, int fd); +long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, + aeTimeProc *proc, void *clientData, + aeEventFinalizerProc *finalizerProc); +int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id); +int aeProcessEvents(aeEventLoop *eventLoop, int flags); +int aeWait(int fd, int mask, long long milliseconds); +void aeMain(aeEventLoop *eventLoop); +char *aeGetApiName(void); +void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep); +int aeGetSetSize(aeEventLoop *eventLoop); +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); + +#endif diff --git a/src/common/thirdparty/ae/ae_epoll.c b/src/common/thirdparty/ae/ae_epoll.c new file mode 100644 index 000000000..410aac70d --- /dev/null +++ b/src/common/thirdparty/ae/ae_epoll.c @@ -0,0 +1,135 @@ +/* Linux epoll(2) based ae.c module + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include + +typedef struct aeApiState { + int epfd; + struct epoll_event *events; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize); + if (!state->events) { + zfree(state); + return -1; + } + state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */ + if (state->epfd == -1) { + zfree(state->events); + zfree(state); + return -1; + } + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize); + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->epfd); + zfree(state->events); + zfree(state); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct epoll_event ee = {0}; /* avoid valgrind warning */ + /* If the fd was already monitored for some event, we need a MOD + * operation. Otherwise we need an ADD operation. */ + int op = eventLoop->events[fd].mask == AE_NONE ? + EPOLL_CTL_ADD : EPOLL_CTL_MOD; + + ee.events = 0; + mask |= eventLoop->events[fd].mask; /* Merge old events */ + if (mask & AE_READABLE) ee.events |= EPOLLIN; + if (mask & AE_WRITABLE) ee.events |= EPOLLOUT; + ee.data.fd = fd; + if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1; + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) { + aeApiState *state = eventLoop->apidata; + struct epoll_event ee = {0}; /* avoid valgrind warning */ + int mask = eventLoop->events[fd].mask & (~delmask); + + ee.events = 0; + if (mask & AE_READABLE) ee.events |= EPOLLIN; + if (mask & AE_WRITABLE) ee.events |= EPOLLOUT; + ee.data.fd = fd; + if (mask != AE_NONE) { + epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee); + } else { + /* Note, Kernel < 2.6.9 requires a non null event pointer even for + * EPOLL_CTL_DEL. */ + epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee); + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, numevents = 0; + + retval = epoll_wait(state->epfd,state->events,eventLoop->setsize, + tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1); + if (retval > 0) { + int j; + + numevents = retval; + for (j = 0; j < numevents; j++) { + int mask = 0; + struct epoll_event *e = state->events+j; + + if (e->events & EPOLLIN) mask |= AE_READABLE; + if (e->events & EPOLLOUT) mask |= AE_WRITABLE; + if (e->events & EPOLLERR) mask |= AE_WRITABLE; + if (e->events & EPOLLHUP) mask |= AE_WRITABLE; + eventLoop->fired[j].fd = e->data.fd; + eventLoop->fired[j].mask = mask; + } + } + return numevents; +} + +static char *aeApiName(void) { + return "epoll"; +} diff --git a/src/common/thirdparty/ae/ae_evport.c b/src/common/thirdparty/ae/ae_evport.c new file mode 100644 index 000000000..5c317becb --- /dev/null +++ b/src/common/thirdparty/ae/ae_evport.c @@ -0,0 +1,320 @@ +/* ae.c module for illumos event ports. + * + * Copyright (c) 2012, Joyent, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include +#include +#include + +#include +#include + +#include + +static int evport_debug = 0; + +/* + * This file implements the ae API using event ports, present on Solaris-based + * systems since Solaris 10. Using the event port interface, we associate file + * descriptors with the port. Each association also includes the set of poll(2) + * events that the consumer is interested in (e.g., POLLIN and POLLOUT). + * + * There's one tricky piece to this implementation: when we return events via + * aeApiPoll, the corresponding file descriptors become dissociated from the + * port. This is necessary because poll events are level-triggered, so if the + * fd didn't become dissociated, it would immediately fire another event since + * the underlying state hasn't changed yet. We must re-associate the file + * descriptor, but only after we know that our caller has actually read from it. + * The ae API does not tell us exactly when that happens, but we do know that + * it must happen by the time aeApiPoll is called again. Our solution is to + * keep track of the last fds returned by aeApiPoll and re-associate them next + * time aeApiPoll is invoked. + * + * To summarize, in this module, each fd association is EITHER (a) represented + * only via the in-kernel association OR (b) represented by pending_fds and + * pending_masks. (b) is only true for the last fds we returned from aeApiPoll, + * and only until we enter aeApiPoll again (at which point we restore the + * in-kernel association). + */ +#define MAX_EVENT_BATCHSZ 512 + +typedef struct aeApiState { + int portfd; /* event port */ + int npending; /* # of pending fds */ + int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */ + int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */ +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + int i; + aeApiState *state = zmalloc(sizeof(aeApiState)); + if (!state) return -1; + + state->portfd = port_create(); + if (state->portfd == -1) { + zfree(state); + return -1; + } + + state->npending = 0; + + for (i = 0; i < MAX_EVENT_BATCHSZ; i++) { + state->pending_fds[i] = -1; + state->pending_masks[i] = AE_NONE; + } + + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Nothing to resize here. */ + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->portfd); + zfree(state); +} + +static int aeApiLookupPending(aeApiState *state, int fd) { + int i; + + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == fd) + return (i); + } + + return (-1); +} + +/* + * Helper function to invoke port_associate for the given fd and mask. + */ +static int aeApiAssociate(const char *where, int portfd, int fd, int mask) { + int events = 0; + int rv, err; + + if (mask & AE_READABLE) + events |= POLLIN; + if (mask & AE_WRITABLE) + events |= POLLOUT; + + if (evport_debug) + fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events); + + rv = port_associate(portfd, PORT_SOURCE_FD, fd, events, + (void *)(uintptr_t)mask); + err = errno; + + if (evport_debug) + fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err)); + + if (rv == -1) { + fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err)); + + if (err == EAGAIN) + fprintf(stderr, "aeApiAssociate: event port limit exceeded."); + } + + return rv; +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask, pfd; + + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask); + + /* + * Since port_associate's "events" argument replaces any existing events, we + * must be sure to include whatever events are already associated when + * we call port_associate() again. + */ + fullmask = mask | eventLoop->events[fd].mask; + pfd = aeApiLookupPending(state, fd); + + if (pfd != -1) { + /* + * This fd was recently returned from aeApiPoll. It should be safe to + * assume that the consumer has processed that poll event, but we play + * it safer by simply updating pending_mask. The fd will be + * re-associated as usual when aeApiPoll is called again. + */ + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd); + state->pending_masks[pfd] |= fullmask; + return 0; + } + + return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask)); +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask, pfd; + + if (evport_debug) + fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask); + + pfd = aeApiLookupPending(state, fd); + + if (pfd != -1) { + if (evport_debug) + fprintf(stderr, "deleting event from pending fd %d\n", fd); + + /* + * This fd was just returned from aeApiPoll, so it's not currently + * associated with the port. All we need to do is update + * pending_mask appropriately. + */ + state->pending_masks[pfd] &= ~mask; + + if (state->pending_masks[pfd] == AE_NONE) + state->pending_fds[pfd] = -1; + + return; + } + + /* + * The fd is currently associated with the port. Like with the add case + * above, we must look at the full mask for the file descriptor before + * updating that association. We don't have a good way of knowing what the + * events are without looking into the eventLoop state directly. We rely on + * the fact that our caller has already updated the mask in the eventLoop. + */ + + fullmask = eventLoop->events[fd].mask; + if (fullmask == AE_NONE) { + /* + * We're removing *all* events, so use port_dissociate to remove the + * association completely. Failure here indicates a bug. + */ + if (evport_debug) + fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd); + + if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) { + perror("aeApiDelEvent: port_dissociate"); + abort(); /* will not return */ + } + } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd, + fullmask) != 0) { + /* + * ENOMEM is a potentially transient condition, but the kernel won't + * generally return it unless things are really bad. EAGAIN indicates + * we've reached an resource limit, for which it doesn't make sense to + * retry (counter-intuitively). All other errors indicate a bug. In any + * of these cases, the best we can do is to abort. + */ + abort(); /* will not return */ + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + struct timespec timeout, *tsp; + int mask, i; + uint_t nevents; + port_event_t event[MAX_EVENT_BATCHSZ]; + + /* + * If we've returned fd events before, we must re-associate them with the + * port now, before calling port_get(). See the block comment at the top of + * this file for an explanation of why. + */ + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == -1) + /* This fd has since been deleted. */ + continue; + + if (aeApiAssociate("aeApiPoll", state->portfd, + state->pending_fds[i], state->pending_masks[i]) != 0) { + /* See aeApiDelEvent for why this case is fatal. */ + abort(); + } + + state->pending_masks[i] = AE_NONE; + state->pending_fds[i] = -1; + } + + state->npending = 0; + + if (tvp != NULL) { + timeout.tv_sec = tvp->tv_sec; + timeout.tv_nsec = tvp->tv_usec * 1000; + tsp = &timeout; + } else { + tsp = NULL; + } + + /* + * port_getn can return with errno == ETIME having returned some events (!). + * So if we get ETIME, we check nevents, too. + */ + nevents = 1; + if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents, + tsp) == -1 && (errno != ETIME || nevents == 0)) { + if (errno == ETIME || errno == EINTR) + return 0; + + /* Any other error indicates a bug. */ + perror("aeApiPoll: port_get"); + abort(); + } + + state->npending = nevents; + + for (i = 0; i < nevents; i++) { + mask = 0; + if (event[i].portev_events & POLLIN) + mask |= AE_READABLE; + if (event[i].portev_events & POLLOUT) + mask |= AE_WRITABLE; + + eventLoop->fired[i].fd = event[i].portev_object; + eventLoop->fired[i].mask = mask; + + if (evport_debug) + fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n", + (int)event[i].portev_object, mask); + + state->pending_fds[i] = event[i].portev_object; + state->pending_masks[i] = (uintptr_t)event[i].portev_user; + } + + return nevents; +} + +static char *aeApiName(void) { + return "evport"; +} diff --git a/src/common/thirdparty/ae/ae_kqueue.c b/src/common/thirdparty/ae/ae_kqueue.c new file mode 100644 index 000000000..6796f4ceb --- /dev/null +++ b/src/common/thirdparty/ae/ae_kqueue.c @@ -0,0 +1,138 @@ +/* Kqueue(2)-based ae.c module + * + * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include +#include + +typedef struct aeApiState { + int kqfd; + struct kevent *events; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize); + if (!state->events) { + zfree(state); + return -1; + } + state->kqfd = kqueue(); + if (state->kqfd == -1) { + zfree(state->events); + zfree(state); + return -1; + } + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct kevent)*setsize); + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->kqfd); + zfree(state->events); + zfree(state); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct kevent ke; + + if (mask & AE_READABLE) { + EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL); + if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; + } + if (mask & AE_WRITABLE) { + EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL); + if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; + } + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct kevent ke; + + if (mask & AE_READABLE) { + EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); + kevent(state->kqfd, &ke, 1, NULL, 0, NULL); + } + if (mask & AE_WRITABLE) { + EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL); + kevent(state->kqfd, &ke, 1, NULL, 0, NULL); + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, numevents = 0; + + if (tvp != NULL) { + struct timespec timeout; + timeout.tv_sec = tvp->tv_sec; + timeout.tv_nsec = tvp->tv_usec * 1000; + retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize, + &timeout); + } else { + retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize, + NULL); + } + + if (retval > 0) { + int j; + + numevents = retval; + for(j = 0; j < numevents; j++) { + int mask = 0; + struct kevent *e = state->events+j; + + if (e->filter == EVFILT_READ) mask |= AE_READABLE; + if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE; + eventLoop->fired[j].fd = e->ident; + eventLoop->fired[j].mask = mask; + } + } + return numevents; +} + +static char *aeApiName(void) { + return "kqueue"; +} diff --git a/src/common/thirdparty/ae/ae_select.c b/src/common/thirdparty/ae/ae_select.c new file mode 100644 index 000000000..c039a8ea3 --- /dev/null +++ b/src/common/thirdparty/ae/ae_select.c @@ -0,0 +1,106 @@ +/* Select()-based ae.c module. + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include + +typedef struct aeApiState { + fd_set rfds, wfds; + /* We need to have a copy of the fd sets as it's not safe to reuse + * FD sets after select(). */ + fd_set _rfds, _wfds; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + FD_ZERO(&state->rfds); + FD_ZERO(&state->wfds); + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Just ensure we have enough room in the fd_set type. */ + if (setsize >= FD_SETSIZE) return -1; + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + zfree(eventLoop->apidata); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + + if (mask & AE_READABLE) FD_SET(fd,&state->rfds); + if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds); + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + + if (mask & AE_READABLE) FD_CLR(fd,&state->rfds); + if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds); +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, j, numevents = 0; + + memcpy(&state->_rfds,&state->rfds,sizeof(fd_set)); + memcpy(&state->_wfds,&state->wfds,sizeof(fd_set)); + + retval = select(eventLoop->maxfd+1, + &state->_rfds,&state->_wfds,NULL,tvp); + if (retval > 0) { + for (j = 0; j <= eventLoop->maxfd; j++) { + int mask = 0; + aeFileEvent *fe = &eventLoop->events[j]; + + if (fe->mask == AE_NONE) continue; + if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds)) + mask |= AE_READABLE; + if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds)) + mask |= AE_WRITABLE; + eventLoop->fired[numevents].fd = j; + eventLoop->fired[numevents].mask = mask; + numevents++; + } + } + return numevents; +} + +static char *aeApiName(void) { + return "select"; +} diff --git a/src/common/thirdparty/ae/config.h b/src/common/thirdparty/ae/config.h new file mode 100644 index 000000000..4f8e1ea1b --- /dev/null +++ b/src/common/thirdparty/ae/config.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __CONFIG_H +#define __CONFIG_H + +#ifdef __APPLE__ +#include +#endif + +/* Test for polling API */ +#ifdef __linux__ +#define HAVE_EPOLL 1 +#endif + +#if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__) +#define HAVE_KQUEUE 1 +#endif + +#ifdef __sun +#include +#ifdef _DTRACE_VERSION +#define HAVE_EVPORT 1 +#endif +#endif + + +#endif diff --git a/src/common/thirdparty/ae/zmalloc.h b/src/common/thirdparty/ae/zmalloc.h new file mode 100644 index 000000000..54c8a69cb --- /dev/null +++ b/src/common/thirdparty/ae/zmalloc.h @@ -0,0 +1,16 @@ +#ifndef _ZMALLOC_H +#define _ZMALLOC_H + +#ifndef zmalloc +#define zmalloc malloc +#endif + +#ifndef zfree +#define zfree free +#endif + +#ifndef zrealloc +#define zrealloc realloc +#endif + +#endif /* _ZMALLOC_H */ diff --git a/src/common/thirdparty/build-redis.sh b/src/common/thirdparty/build-redis.sh new file mode 100644 index 000000000..230e9ae29 --- /dev/null +++ b/src/common/thirdparty/build-redis.sh @@ -0,0 +1,6 @@ +if [ ! -f redis-3.2.3/src/redis-server ]; then + wget http://download.redis.io/releases/redis-3.2.3.tar.gz + tar xvfz redis-3.2.3.tar.gz + cd redis-3.2.3 + make +fi diff --git a/src/common/thirdparty/greatest.h b/src/common/thirdparty/greatest.h new file mode 100644 index 000000000..eb34ff426 --- /dev/null +++ b/src/common/thirdparty/greatest.h @@ -0,0 +1,1023 @@ +/* + * Copyright (c) 2011-2016 Scott Vokes + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef GREATEST_H +#define GREATEST_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* 1.2.1 */ +#define GREATEST_VERSION_MAJOR 1 +#define GREATEST_VERSION_MINOR 2 +#define GREATEST_VERSION_PATCH 1 + +/* A unit testing system for C, contained in 1 file. + * It doesn't use dynamic allocation or depend on anything + * beyond ANSI C89. + * + * An up-to-date version can be found at: + * https://github.com/silentbicycle/greatest/ + */ + + +/********************************************************************* + * Minimal test runner template + *********************************************************************/ +#if 0 +#include "greatest.h" +TEST foo_should_foo(void) { + PASS(); +} +static void setup_cb(void *data) { + printf("setup callback for each test case\n"); +} +static void teardown_cb(void *data) { + printf("teardown callback for each test case\n"); +} +SUITE(suite) { + /* Optional setup/teardown callbacks which will be run before/after + * every test case. If using a test suite, they will be cleared when + * the suite finishes. */ + SET_SETUP(setup_cb, voidp_to_callback_data); + SET_TEARDOWN(teardown_cb, voidp_to_callback_data); + RUN_TEST(foo_should_foo); +} +/* Add definitions that need to be in the test runner's main file. */ +GREATEST_MAIN_DEFS(); +/* Set up, run suite(s) of tests, report pass/fail/skip stats. */ +int run_tests(void) { + GREATEST_INIT(); /* init. greatest internals */ + /* List of suites to run (if any). */ + RUN_SUITE(suite); + /* Tests can also be run directly, without using test suites. */ + RUN_TEST(foo_should_foo); + GREATEST_PRINT_REPORT(); /* display results */ + return greatest_all_passed(); +} +/* main(), for a standalone command-line test runner. + * This replaces run_tests above, and adds command line option + * handling and exiting with a pass/fail status. */ +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); /* init & parse command-line args */ + RUN_SUITE(suite); + GREATEST_MAIN_END(); /* display results */ +} +#endif +/*********************************************************************/ + + +#include +#include +#include +#include + +/*********** + * Options * + ***********/ + +/* Default column width for non-verbose output. */ +#ifndef GREATEST_DEFAULT_WIDTH +#define GREATEST_DEFAULT_WIDTH 72 +#endif + +/* FILE *, for test logging. */ +#ifndef GREATEST_STDOUT +#define GREATEST_STDOUT stdout +#endif + +/* Remove GREATEST_ prefix from most commonly used symbols? */ +#ifndef GREATEST_USE_ABBREVS +#define GREATEST_USE_ABBREVS 1 +#endif + +/* Set to 0 to disable all use of setjmp/longjmp. */ +#ifndef GREATEST_USE_LONGJMP +#define GREATEST_USE_LONGJMP 1 +#endif + +#if GREATEST_USE_LONGJMP +#include +#endif + +/* Set to 0 to disable all use of time.h / clock(). */ +#ifndef GREATEST_USE_TIME +#define GREATEST_USE_TIME 1 +#endif + +#if GREATEST_USE_TIME +#include +#endif + +/* Floating point type, for ASSERT_IN_RANGE. */ +#ifndef GREATEST_FLOAT +#define GREATEST_FLOAT double +#define GREATEST_FLOAT_FMT "%g" +#endif + +/********* + * Types * + *********/ + +/* Info for the current running suite. */ +typedef struct greatest_suite_info { + unsigned int tests_run; + unsigned int passed; + unsigned int failed; + unsigned int skipped; + +#if GREATEST_USE_TIME + /* timers, pre/post running suite and individual tests */ + clock_t pre_suite; + clock_t post_suite; + clock_t pre_test; + clock_t post_test; +#endif +} greatest_suite_info; + +/* Type for a suite function. */ +typedef void (greatest_suite_cb)(void); + +/* Types for setup/teardown callbacks. If non-NULL, these will be run + * and passed the pointer to their additional data. */ +typedef void (greatest_setup_cb)(void *udata); +typedef void (greatest_teardown_cb)(void *udata); + +/* Type for an equality comparison between two pointers of the same type. + * Should return non-0 if equal, otherwise 0. + * UDATA is a closure value, passed through from ASSERT_EQUAL_T[m]. */ +typedef int greatest_equal_cb(const void *exp, const void *got, void *udata); + +/* Type for a callback that prints a value pointed to by T. + * Return value has the same meaning as printf's. + * UDATA is a closure value, passed through from ASSERT_EQUAL_T[m]. */ +typedef int greatest_printf_cb(const void *t, void *udata); + +/* Callbacks for an arbitrary type; needed for type-specific + * comparisons via GREATEST_ASSERT_EQUAL_T[m].*/ +typedef struct greatest_type_info { + greatest_equal_cb *equal; + greatest_printf_cb *print; +} greatest_type_info; + +typedef struct greatest_memory_cmp_env { + const unsigned char *exp; + const unsigned char *got; + size_t size; +} greatest_memory_cmp_env; + +/* Callbacks for string and raw memory types. */ +extern greatest_type_info greatest_type_info_string; +extern greatest_type_info greatest_type_info_memory; + +typedef enum { + GREATEST_FLAG_FIRST_FAIL = 0x01, + GREATEST_FLAG_LIST_ONLY = 0x02 +} greatest_flag_t; + +/* Struct containing all test runner state. */ +typedef struct greatest_run_info { + unsigned char flags; + unsigned char verbosity; + unsigned int tests_run; /* total test count */ + + /* overall pass/fail/skip counts */ + unsigned int passed; + unsigned int failed; + unsigned int skipped; + unsigned int assertions; + + /* currently running test suite */ + greatest_suite_info suite; + + /* info to print about the most recent failure */ + const char *fail_file; + unsigned int fail_line; + const char *msg; + + /* current setup/teardown hooks and userdata */ + greatest_setup_cb *setup; + void *setup_udata; + greatest_teardown_cb *teardown; + void *teardown_udata; + + /* formatting info for ".....s...F"-style output */ + unsigned int col; + unsigned int width; + + /* only run a specific suite or test */ + const char *suite_filter; + const char *test_filter; + +#if GREATEST_USE_TIME + /* overall timers */ + clock_t begin; + clock_t end; +#endif + +#if GREATEST_USE_LONGJMP + jmp_buf jump_dest; +#endif +} greatest_run_info; + +struct greatest_report_t { + /* overall pass/fail/skip counts */ + unsigned int passed; + unsigned int failed; + unsigned int skipped; + unsigned int assertions; +}; + +/* Global var for the current testing context. + * Initialized by GREATEST_MAIN_DEFS(). */ +extern greatest_run_info greatest_info; + +/* Type for ASSERT_ENUM_EQ's ENUM_STR argument. */ +typedef const char *greatest_enum_str_fun(int value); + +/********************** + * Exported functions * + **********************/ + +/* These are used internally by greatest. */ +void greatest_do_pass(const char *name); +void greatest_do_fail(const char *name); +void greatest_do_skip(const char *name); +int greatest_pre_test(const char *name); +void greatest_post_test(const char *name, int res); +void greatest_usage(const char *name); +int greatest_do_assert_equal_t(const void *exp, const void *got, + greatest_type_info *type_info, void *udata); + +/* These are part of the public greatest API. */ +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata); +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, void *udata); +int greatest_all_passed(void); +void greatest_set_test_filter(const char *name); +void greatest_set_suite_filter(const char *name); +void greatest_get_report(struct greatest_report_t *report); +unsigned int greatest_get_verbosity(void); +void greatest_set_verbosity(unsigned int verbosity); +void greatest_set_flag(greatest_flag_t flag); + + +/******************** +* Language Support * +********************/ + +/* If __VA_ARGS__ (C99) is supported, allow parametric testing +* without needing to manually manage the argument struct. */ +#if __STDC_VERSION__ >= 19901L || _MSC_VER >= 1800 +#define GREATEST_VA_ARGS +#endif + + +/********** + * Macros * + **********/ + +/* Define a suite. */ +#define GREATEST_SUITE(NAME) void NAME(void); void NAME(void) + +/* Declare a suite, provided by another compilation unit. */ +#define GREATEST_SUITE_EXTERN(NAME) void NAME(void) + +/* Start defining a test function. + * The arguments are not included, to allow parametric testing. */ +#define GREATEST_TEST static enum greatest_test_res + +/* PASS/FAIL/SKIP result from a test. Used internally. */ +typedef enum greatest_test_res { + GREATEST_TEST_RES_PASS = 0, + GREATEST_TEST_RES_FAIL = -1, + GREATEST_TEST_RES_SKIP = 1 +} greatest_test_res; + +/* Run a suite. */ +#define GREATEST_RUN_SUITE(S_NAME) greatest_run_suite(S_NAME, #S_NAME) + +/* Run a test in the current suite. */ +#define GREATEST_RUN_TEST(TEST) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + enum greatest_test_res res = GREATEST_SAVE_CONTEXT(); \ + if (res == GREATEST_TEST_RES_PASS) { \ + res = TEST(); \ + } \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +/* Ignore a test, don't warn about it being unused. */ +#define GREATEST_IGNORE_TEST(TEST) (void)TEST + +/* Run a test in the current suite with one void * argument, + * which can be a pointer to a struct with multiple arguments. */ +#define GREATEST_RUN_TEST1(TEST, ENV) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(ENV); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +#ifdef GREATEST_VA_ARGS +#define GREATEST_RUN_TESTp(TEST, ...) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(__VA_ARGS__); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) +#endif + + +/* Check if the test runner is in verbose mode. */ +#define GREATEST_IS_VERBOSE() ((greatest_info.verbosity) > 0) +#define GREATEST_LIST_ONLY() \ + (greatest_info.flags & GREATEST_FLAG_LIST_ONLY) +#define GREATEST_FIRST_FAIL() \ + (greatest_info.flags & GREATEST_FLAG_FIRST_FAIL) +#define GREATEST_FAILURE_ABORT() \ + (greatest_info.suite.failed > 0 && GREATEST_FIRST_FAIL()) + +/* Message-less forms of tests defined below. */ +#define GREATEST_PASS() GREATEST_PASSm(NULL) +#define GREATEST_FAIL() GREATEST_FAILm(NULL) +#define GREATEST_SKIP() GREATEST_SKIPm(NULL) +#define GREATEST_ASSERT(COND) \ + GREATEST_ASSERTm(#COND, COND) +#define GREATEST_ASSERT_OR_LONGJMP(COND) \ + GREATEST_ASSERT_OR_LONGJMPm(#COND, COND) +#define GREATEST_ASSERT_FALSE(COND) \ + GREATEST_ASSERT_FALSEm(#COND, COND) +#define GREATEST_ASSERT_EQ(EXP, GOT) \ + GREATEST_ASSERT_EQm(#EXP " != " #GOT, EXP, GOT) +#define GREATEST_ASSERT_EQ_FMT(EXP, GOT, FMT) \ + GREATEST_ASSERT_EQ_FMTm(#EXP " != " #GOT, EXP, GOT, FMT) +#define GREATEST_ASSERT_IN_RANGE(EXP, GOT, TOL) \ + GREATEST_ASSERT_IN_RANGEm(#EXP " != " #GOT " +/- " #TOL, EXP, GOT, TOL) +#define GREATEST_ASSERT_EQUAL_T(EXP, GOT, TYPE_INFO, UDATA) \ + GREATEST_ASSERT_EQUAL_Tm(#EXP " != " #GOT, EXP, GOT, TYPE_INFO, UDATA) +#define GREATEST_ASSERT_STR_EQ(EXP, GOT) \ + GREATEST_ASSERT_STR_EQm(#EXP " != " #GOT, EXP, GOT) +#define GREATEST_ASSERT_STRN_EQ(EXP, GOT, SIZE) \ + GREATEST_ASSERT_STRN_EQm(#EXP " != " #GOT, EXP, GOT, SIZE) +#define GREATEST_ASSERT_MEM_EQ(EXP, GOT, SIZE) \ + GREATEST_ASSERT_MEM_EQm(#EXP " != " #GOT, EXP, GOT, SIZE) +#define GREATEST_ASSERT_ENUM_EQ(EXP, GOT, ENUM_STR) \ + GREATEST_ASSERT_ENUM_EQm(#EXP " != " #GOT, EXP, GOT, ENUM_STR) + +/* The following forms take an additional message argument first, + * to be displayed by the test runner. */ + +/* Fail if a condition is not true, with message. */ +#define GREATEST_ASSERTm(MSG, COND) \ + do { \ + greatest_info.assertions++; \ + if (!(COND)) { GREATEST_FAILm(MSG); } \ + } while (0) + +/* Fail if a condition is not true, longjmping out of test. */ +#define GREATEST_ASSERT_OR_LONGJMPm(MSG, COND) \ + do { \ + greatest_info.assertions++; \ + if (!(COND)) { GREATEST_FAIL_WITH_LONGJMPm(MSG); } \ + } while (0) + +/* Fail if a condition is not false, with message. */ +#define GREATEST_ASSERT_FALSEm(MSG, COND) \ + do { \ + greatest_info.assertions++; \ + if ((COND)) { GREATEST_FAILm(MSG); } \ + } while (0) + +/* Fail if EXP != GOT (equality comparison by ==). */ +#define GREATEST_ASSERT_EQm(MSG, EXP, GOT) \ + do { \ + greatest_info.assertions++; \ + if ((EXP) != (GOT)) { GREATEST_FAILm(MSG); } \ + } while (0) + +/* Fail if EXP != GOT (equality comparison by ==). + * Warning: EXP and GOT will be evaluated more than once on failure. */ +#define GREATEST_ASSERT_EQ_FMTm(MSG, EXP, GOT, FMT) \ + do { \ + const char *greatest_FMT = ( FMT ); \ + greatest_info.assertions++; \ + if ((EXP) != (GOT)) { \ + fprintf(GREATEST_STDOUT, "\nExpected: "); \ + fprintf(GREATEST_STDOUT, greatest_FMT, EXP); \ + fprintf(GREATEST_STDOUT, "\n Got: "); \ + fprintf(GREATEST_STDOUT, greatest_FMT, GOT); \ + fprintf(GREATEST_STDOUT, "\n"); \ + GREATEST_FAILm(MSG); \ + } \ + } while (0) + +/* Fail if EXP is not equal to GOT, printing enum IDs. */ +#define GREATEST_ASSERT_ENUM_EQm(MSG, EXP, GOT, ENUM_STR) \ + do { \ + int greatest_EXP = (int)(EXP); \ + int greatest_GOT = (int)(GOT); \ + greatest_enum_str_fun *greatest_ENUM_STR = ENUM_STR; \ + if (greatest_EXP != greatest_GOT) { \ + fprintf(GREATEST_STDOUT, "\nExpected: %s", \ + greatest_ENUM_STR(greatest_EXP)); \ + fprintf(GREATEST_STDOUT, "\n Got: %s\n", \ + greatest_ENUM_STR(greatest_GOT)); \ + GREATEST_FAILm(MSG); \ + } \ + } while (0) \ + +/* Fail if GOT not in range of EXP +|- TOL. */ +#define GREATEST_ASSERT_IN_RANGEm(MSG, EXP, GOT, TOL) \ + do { \ + GREATEST_FLOAT greatest_EXP = (EXP); \ + GREATEST_FLOAT greatest_GOT = (GOT); \ + GREATEST_FLOAT greatest_TOL = (TOL); \ + greatest_info.assertions++; \ + if ((greatest_EXP > greatest_GOT && \ + greatest_EXP - greatest_GOT > greatest_TOL) || \ + (greatest_EXP < greatest_GOT && \ + greatest_GOT - greatest_EXP > greatest_TOL)) { \ + fprintf(GREATEST_STDOUT, \ + "\nExpected: " GREATEST_FLOAT_FMT \ + " +/- " GREATEST_FLOAT_FMT \ + "\n Got: " GREATEST_FLOAT_FMT \ + "\n", \ + greatest_EXP, greatest_TOL, greatest_GOT); \ + GREATEST_FAILm(MSG); \ + } \ + } while (0) + +/* Fail if EXP is not equal to GOT, according to strcmp. */ +#define GREATEST_ASSERT_STR_EQm(MSG, EXP, GOT) \ + do { \ + GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, \ + &greatest_type_info_string, NULL); \ + } while (0) \ + +/* Fail if EXP is not equal to GOT, according to strcmp. */ +#define GREATEST_ASSERT_STRN_EQm(MSG, EXP, GOT, SIZE) \ + do { \ + size_t size = SIZE; \ + GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, \ + &greatest_type_info_string, &size); \ + } while (0) \ + +/* Fail if EXP is not equal to GOT, according to memcmp. */ +#define GREATEST_ASSERT_MEM_EQm(MSG, EXP, GOT, SIZE) \ + do { \ + greatest_memory_cmp_env env; \ + env.exp = (const unsigned char *)EXP; \ + env.got = (const unsigned char *)GOT; \ + env.size = SIZE; \ + GREATEST_ASSERT_EQUAL_Tm(MSG, env.exp, env.got, \ + &greatest_type_info_memory, &env); \ + } while (0) \ + +/* Fail if EXP is not equal to GOT, according to a comparison + * callback in TYPE_INFO. If they are not equal, optionally use a + * print callback in TYPE_INFO to print them. */ +#define GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, TYPE_INFO, UDATA) \ + do { \ + greatest_type_info *type_info = (TYPE_INFO); \ + greatest_info.assertions++; \ + if (!greatest_do_assert_equal_t(EXP, GOT, \ + type_info, UDATA)) { \ + if (type_info == NULL || type_info->equal == NULL) { \ + GREATEST_FAILm("type_info->equal callback missing!"); \ + } else { \ + GREATEST_FAILm(MSG); \ + } \ + } \ + } while (0) \ + +/* Pass. */ +#define GREATEST_PASSm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return GREATEST_TEST_RES_PASS; \ + } while (0) + +/* Fail. */ +#define GREATEST_FAILm(MSG) \ + do { \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + greatest_info.msg = MSG; \ + return GREATEST_TEST_RES_FAIL; \ + } while (0) + +/* Optional GREATEST_FAILm variant that longjmps. */ +#if GREATEST_USE_LONGJMP +#define GREATEST_FAIL_WITH_LONGJMP() GREATEST_FAIL_WITH_LONGJMPm(NULL) +#define GREATEST_FAIL_WITH_LONGJMPm(MSG) \ + do { \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + greatest_info.msg = MSG; \ + longjmp(greatest_info.jump_dest, GREATEST_TEST_RES_FAIL); \ + } while (0) +#endif + +/* Skip the current test. */ +#define GREATEST_SKIPm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return GREATEST_TEST_RES_SKIP; \ + } while (0) + +/* Check the result of a subfunction using ASSERT, etc. */ +#define GREATEST_CHECK_CALL(RES) \ + do { \ + enum greatest_test_res greatest_RES = RES; \ + if (greatest_RES != GREATEST_TEST_RES_PASS) { \ + return greatest_RES; \ + } \ + } while (0) \ + +#if GREATEST_USE_TIME +#define GREATEST_SET_TIME(NAME) \ + NAME = clock(); \ + if (NAME == (clock_t) -1) { \ + fprintf(GREATEST_STDOUT, \ + "clock error: %s\n", #NAME); \ + exit(EXIT_FAILURE); \ + } + +#define GREATEST_CLOCK_DIFF(C1, C2) \ + fprintf(GREATEST_STDOUT, " (%lu ticks, %.3f sec)", \ + (long unsigned int) (C2) - (long unsigned int)(C1), \ + (double)((C2) - (C1)) / (1.0 * (double)CLOCKS_PER_SEC)) +#else +#define GREATEST_SET_TIME(UNUSED) +#define GREATEST_CLOCK_DIFF(UNUSED1, UNUSED2) +#endif + +#if GREATEST_USE_LONGJMP +#define GREATEST_SAVE_CONTEXT() \ + /* setjmp returns 0 (GREATEST_TEST_RES_PASS) on first call */ \ + /* so the test runs, then RES_FAIL from FAIL_WITH_LONGJMP. */ \ + ((enum greatest_test_res)(setjmp(greatest_info.jump_dest))) +#else +#define GREATEST_SAVE_CONTEXT() \ + /*a no-op, since setjmp/longjmp aren't being used */ \ + GREATEST_TEST_RES_PASS +#endif + +/* Include several function definitions in the main test file. */ +#define GREATEST_MAIN_DEFS() \ + \ +/* Is FILTER a subset of NAME? */ \ +static int greatest_name_match(const char *name, \ + const char *filter) { \ + size_t offset = 0; \ + size_t filter_len = strlen(filter); \ + while (name[offset] != '\0') { \ + if (name[offset] == filter[0]) { \ + if (0 == strncmp(&name[offset], filter, filter_len)) { \ + return 1; \ + } \ + } \ + offset++; \ + } \ + \ + return 0; \ +} \ + \ +int greatest_pre_test(const char *name) { \ + if (!GREATEST_LIST_ONLY() \ + && (!GREATEST_FIRST_FAIL() || greatest_info.suite.failed == 0) \ + && (greatest_info.test_filter == NULL || \ + greatest_name_match(name, greatest_info.test_filter))) { \ + GREATEST_SET_TIME(greatest_info.suite.pre_test); \ + if (greatest_info.setup) { \ + greatest_info.setup(greatest_info.setup_udata); \ + } \ + return 1; /* test should be run */ \ + } else { \ + return 0; /* skipped */ \ + } \ +} \ + \ +void greatest_post_test(const char *name, int res) { \ + GREATEST_SET_TIME(greatest_info.suite.post_test); \ + if (greatest_info.teardown) { \ + void *udata = greatest_info.teardown_udata; \ + greatest_info.teardown(udata); \ + } \ + \ + if (res <= GREATEST_TEST_RES_FAIL) { \ + greatest_do_fail(name); \ + } else if (res >= GREATEST_TEST_RES_SKIP) { \ + greatest_do_skip(name); \ + } else if (res == GREATEST_TEST_RES_PASS) { \ + greatest_do_pass(name); \ + } \ + greatest_info.suite.tests_run++; \ + greatest_info.col++; \ + if (GREATEST_IS_VERBOSE()) { \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_test, \ + greatest_info.suite.post_test); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } else if (greatest_info.col % greatest_info.width == 0) { \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + } \ + if (GREATEST_STDOUT == stdout) fflush(stdout); \ +} \ + \ +static void report_suite(void) { \ + if (greatest_info.suite.tests_run > 0) { \ + fprintf(GREATEST_STDOUT, \ + "\n%u test%s - %u passed, %u failed, %u skipped", \ + greatest_info.suite.tests_run, \ + greatest_info.suite.tests_run == 1 ? "" : "s", \ + greatest_info.suite.passed, \ + greatest_info.suite.failed, \ + greatest_info.suite.skipped); \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_suite, \ + greatest_info.suite.post_suite); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } \ +} \ + \ +static void update_counts_and_reset_suite(void) { \ + greatest_info.setup = NULL; \ + greatest_info.setup_udata = NULL; \ + greatest_info.teardown = NULL; \ + greatest_info.teardown_udata = NULL; \ + greatest_info.passed += greatest_info.suite.passed; \ + greatest_info.failed += greatest_info.suite.failed; \ + greatest_info.skipped += greatest_info.suite.skipped; \ + greatest_info.tests_run += greatest_info.suite.tests_run; \ + memset(&greatest_info.suite, 0, sizeof(greatest_info.suite)); \ + greatest_info.col = 0; \ +} \ + \ +static void greatest_run_suite(greatest_suite_cb *suite_cb, \ + const char *suite_name) { \ + if (greatest_info.suite_filter && \ + !greatest_name_match(suite_name, greatest_info.suite_filter)) { \ + return; \ + } \ + update_counts_and_reset_suite(); \ + if (GREATEST_FIRST_FAIL() && greatest_info.failed > 0) { return; } \ + fprintf(GREATEST_STDOUT, "\n* Suite %s:\n", suite_name); \ + GREATEST_SET_TIME(greatest_info.suite.pre_suite); \ + suite_cb(); \ + GREATEST_SET_TIME(greatest_info.suite.post_suite); \ + report_suite(); \ +} \ + \ +void greatest_do_pass(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "PASS %s: %s", \ + name, greatest_info.msg ? greatest_info.msg : ""); \ + } else { \ + fprintf(GREATEST_STDOUT, "."); \ + } \ + greatest_info.suite.passed++; \ +} \ + \ +void greatest_do_fail(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, \ + "FAIL %s: %s (%s:%u)", \ + name, greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } else { \ + fprintf(GREATEST_STDOUT, "F"); \ + greatest_info.col++; \ + /* add linebreak if in line of '.'s */ \ + if (greatest_info.col != 0) { \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + } \ + fprintf(GREATEST_STDOUT, "FAIL %s: %s (%s:%u)\n", \ + name, \ + greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } \ + greatest_info.suite.failed++; \ +} \ + \ +void greatest_do_skip(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "SKIP %s: %s", \ + name, \ + greatest_info.msg ? \ + greatest_info.msg : "" ); \ + } else { \ + fprintf(GREATEST_STDOUT, "s"); \ + } \ + greatest_info.suite.skipped++; \ +} \ + \ +int greatest_do_assert_equal_t(const void *exp, const void *got, \ + greatest_type_info *type_info, void *udata) { \ + int eq = 0; \ + if (type_info == NULL || type_info->equal == NULL) { \ + return 0; \ + } \ + eq = type_info->equal(exp, got, udata); \ + if (!eq) { \ + if (type_info->print != NULL) { \ + fprintf(GREATEST_STDOUT, "\nExpected: "); \ + (void)type_info->print(exp, udata); \ + fprintf(GREATEST_STDOUT, "\n Got: "); \ + (void)type_info->print(got, udata); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } else { \ + fprintf(GREATEST_STDOUT, \ + "GREATEST_ASSERT_EQUAL_T failure at %s:%u\n", \ + greatest_info.fail_file, \ + greatest_info.fail_line); \ + } \ + } \ + return eq; \ +} \ + \ +void greatest_usage(const char *name) { \ + fprintf(GREATEST_STDOUT, \ + "Usage: %s [-hlfv] [-s SUITE] [-t TEST]\n" \ + " -h, --help print this Help\n" \ + " -l List suites and their tests, then exit\n" \ + " -f Stop runner after first failure\n" \ + " -v Verbose output\n" \ + " -s SUITE only run suites containing string SUITE\n" \ + " -t TEST only run tests containing string TEST\n", \ + name); \ +} \ + \ +static void greatest_parse_args(int argc, char **argv) { \ + int i = 0; \ + for (i = 1; i < argc; i++) { \ + if (0 == strncmp("-t", argv[i], 2)) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.test_filter = argv[i+1]; \ + i++; \ + } else if (0 == strncmp("-s", argv[i], 2)) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.suite_filter = argv[i+1]; \ + i++; \ + } else if (0 == strncmp("-f", argv[i], 2)) { \ + greatest_info.flags |= GREATEST_FLAG_FIRST_FAIL; \ + } else if (0 == strncmp("-v", argv[i], 2)) { \ + greatest_info.verbosity++; \ + } else if (0 == strncmp("-l", argv[i], 2)) { \ + greatest_info.flags |= GREATEST_FLAG_LIST_ONLY; \ + } else if (0 == strncmp("-h", argv[i], 2) || \ + 0 == strncmp("--help", argv[i], 6)) { \ + greatest_usage(argv[0]); \ + exit(EXIT_SUCCESS); \ + } else if (0 == strncmp("--", argv[i], 2)) { \ + break; \ + } else { \ + fprintf(GREATEST_STDOUT, \ + "Unknown argument '%s'\n", argv[i]); \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + } \ +} \ + \ +int greatest_all_passed(void) { return (greatest_info.failed == 0); } \ + \ +void greatest_set_test_filter(const char *name) { \ + greatest_info.test_filter = name; \ +} \ + \ +void greatest_set_suite_filter(const char *name) { \ + greatest_info.suite_filter = name; \ +} \ + \ +void greatest_get_report(struct greatest_report_t *report) { \ + if (report) { \ + report->passed = greatest_info.passed; \ + report->failed = greatest_info.failed; \ + report->skipped = greatest_info.skipped; \ + report->assertions = greatest_info.assertions; \ + } \ +} \ + \ +unsigned int greatest_get_verbosity(void) { \ + return greatest_info.verbosity; \ +} \ + \ +void greatest_set_verbosity(unsigned int verbosity) { \ + greatest_info.verbosity = (unsigned char)verbosity; \ +} \ + \ +void greatest_set_flag(greatest_flag_t flag) { \ + greatest_info.flags |= flag; \ +} \ + \ +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata) { \ + greatest_info.setup = cb; \ + greatest_info.setup_udata = udata; \ +} \ + \ +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, \ + void *udata) { \ + greatest_info.teardown = cb; \ + greatest_info.teardown_udata = udata; \ +} \ + \ +static int greatest_string_equal_cb(const void *exp, const void *got, \ + void *udata) { \ + size_t *size = (size_t *)udata; \ + return (size != NULL \ + ? (0 == strncmp((const char *)exp, (const char *)got, *size)) \ + : (0 == strcmp((const char *)exp, (const char *)got))); \ +} \ + \ +static int greatest_string_printf_cb(const void *t, void *udata) { \ + (void)udata; /* note: does not check \0 termination. */ \ + return fprintf(GREATEST_STDOUT, "%s", (const char *)t); \ +} \ + \ +greatest_type_info greatest_type_info_string = { \ + greatest_string_equal_cb, \ + greatest_string_printf_cb, \ +}; \ + \ +static int greatest_memory_equal_cb(const void *exp, const void *got, \ + void *udata) { \ + greatest_memory_cmp_env *env = (greatest_memory_cmp_env *)udata; \ + return (0 == memcmp(exp, got, env->size)); \ +} \ + \ +static int greatest_memory_printf_cb(const void *t, void *udata) { \ + greatest_memory_cmp_env *env = (greatest_memory_cmp_env *)udata; \ + unsigned char *buf = (unsigned char *)t, diff_mark = ' '; \ + FILE *out = GREATEST_STDOUT; \ + size_t i, line_i, line_len = 0; \ + int len = 0; /* format hexdump with differences highlighted */ \ + for (i = 0; i < env->size; i+= line_len) { \ + diff_mark = ' '; \ + line_len = env->size - i; \ + if (line_len > 16) { line_len = 16; } \ + for (line_i = i; line_i < i + line_len; line_i++) { \ + if (env->exp[line_i] != env->got[line_i]) diff_mark = 'X'; \ + } \ + len += fprintf(out, "\n%04x %c ", (unsigned int)i, diff_mark); \ + for (line_i = i; line_i < i + line_len; line_i++) { \ + int m = env->exp[line_i] == env->got[line_i]; /* match? */ \ + len += fprintf(out, "%02x%c", buf[line_i], m ? ' ' : '<'); \ + } \ + for (line_i = 0; line_i < 16 - line_len; line_i++) { \ + len += fprintf(out, " "); \ + } \ + fprintf(out, " "); \ + for (line_i = i; line_i < i + line_len; line_i++) { \ + unsigned char c = buf[line_i]; \ + len += fprintf(out, "%c", isprint(c) ? c : '.'); \ + } \ + } \ + len += fprintf(out, "\n"); \ + return len; \ +} \ + \ +greatest_type_info greatest_type_info_memory = { \ + greatest_memory_equal_cb, \ + greatest_memory_printf_cb, \ +}; \ + \ +greatest_run_info greatest_info + +/* Init internals. */ +#define GREATEST_INIT() \ + do { \ + /* Suppress unused function warning if features aren't used */ \ + (void)greatest_run_suite; \ + (void)greatest_parse_args; \ + \ + memset(&greatest_info, 0, sizeof(greatest_info)); \ + greatest_info.width = GREATEST_DEFAULT_WIDTH; \ + GREATEST_SET_TIME(greatest_info.begin); \ + } while (0) \ + +/* Handle command-line arguments, etc. */ +#define GREATEST_MAIN_BEGIN() \ + do { \ + GREATEST_INIT(); \ + greatest_parse_args(argc, argv); \ + } while (0) + +/* Report passes, failures, skipped tests, the number of + * assertions, and the overall run time. */ +#define GREATEST_PRINT_REPORT() \ + do { \ + if (!GREATEST_LIST_ONLY()) { \ + update_counts_and_reset_suite(); \ + GREATEST_SET_TIME(greatest_info.end); \ + fprintf(GREATEST_STDOUT, \ + "\nTotal: %u test%s", \ + greatest_info.tests_run, \ + greatest_info.tests_run == 1 ? "" : "s"); \ + GREATEST_CLOCK_DIFF(greatest_info.begin, \ + greatest_info.end); \ + fprintf(GREATEST_STDOUT, ", %u assertion%s\n", \ + greatest_info.assertions, \ + greatest_info.assertions == 1 ? "" : "s"); \ + fprintf(GREATEST_STDOUT, \ + "Pass: %u, fail: %u, skip: %u.\n", \ + greatest_info.passed, \ + greatest_info.failed, greatest_info.skipped); \ + } \ + } while (0) + +/* Report results, exit with exit status based on results. */ +#define GREATEST_MAIN_END() \ + do { \ + GREATEST_PRINT_REPORT(); \ + return (greatest_all_passed() ? EXIT_SUCCESS : EXIT_FAILURE); \ + } while (0) + +/* Make abbreviations without the GREATEST_ prefix for the + * most commonly used symbols. */ +#if GREATEST_USE_ABBREVS +#define TEST GREATEST_TEST +#define SUITE GREATEST_SUITE +#define SUITE_EXTERN GREATEST_SUITE_EXTERN +#define RUN_TEST GREATEST_RUN_TEST +#define RUN_TEST1 GREATEST_RUN_TEST1 +#define RUN_SUITE GREATEST_RUN_SUITE +#define IGNORE_TEST GREATEST_IGNORE_TEST +#define ASSERT GREATEST_ASSERT +#define ASSERTm GREATEST_ASSERTm +#define ASSERT_FALSE GREATEST_ASSERT_FALSE +#define ASSERT_EQ GREATEST_ASSERT_EQ +#define ASSERT_EQ_FMT GREATEST_ASSERT_EQ_FMT +#define ASSERT_IN_RANGE GREATEST_ASSERT_IN_RANGE +#define ASSERT_EQUAL_T GREATEST_ASSERT_EQUAL_T +#define ASSERT_STR_EQ GREATEST_ASSERT_STR_EQ +#define ASSERT_STRN_EQ GREATEST_ASSERT_STRN_EQ +#define ASSERT_MEM_EQ GREATEST_ASSERT_MEM_EQ +#define ASSERT_ENUM_EQ GREATEST_ASSERT_ENUM_EQ +#define ASSERT_FALSEm GREATEST_ASSERT_FALSEm +#define ASSERT_EQm GREATEST_ASSERT_EQm +#define ASSERT_EQ_FMTm GREATEST_ASSERT_EQ_FMTm +#define ASSERT_IN_RANGEm GREATEST_ASSERT_IN_RANGEm +#define ASSERT_EQUAL_Tm GREATEST_ASSERT_EQUAL_Tm +#define ASSERT_STR_EQm GREATEST_ASSERT_STR_EQm +#define ASSERT_STRN_EQm GREATEST_ASSERT_STRN_EQm +#define ASSERT_MEM_EQm GREATEST_ASSERT_MEM_EQm +#define ASSERT_ENUM_EQm GREATEST_ASSERT_ENUM_EQm +#define PASS GREATEST_PASS +#define FAIL GREATEST_FAIL +#define SKIP GREATEST_SKIP +#define PASSm GREATEST_PASSm +#define FAILm GREATEST_FAILm +#define SKIPm GREATEST_SKIPm +#define SET_SETUP GREATEST_SET_SETUP_CB +#define SET_TEARDOWN GREATEST_SET_TEARDOWN_CB +#define CHECK_CALL GREATEST_CHECK_CALL + +#ifdef GREATEST_VA_ARGS +#define RUN_TESTp GREATEST_RUN_TESTp +#endif + +#if GREATEST_USE_LONGJMP +#define ASSERT_OR_LONGJMP GREATEST_ASSERT_OR_LONGJMP +#define ASSERT_OR_LONGJMPm GREATEST_ASSERT_OR_LONGJMPm +#define FAIL_WITH_LONGJMP GREATEST_FAIL_WITH_LONGJMP +#define FAIL_WITH_LONGJMPm GREATEST_FAIL_WITH_LONGJMPm +#endif + +#endif /* USE_ABBREVS */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/common/thirdparty/hiredis b/src/common/thirdparty/hiredis new file mode 160000 index 000000000..5f98e1d35 --- /dev/null +++ b/src/common/thirdparty/hiredis @@ -0,0 +1 @@ +Subproject commit 5f98e1d35dcf00a026793ada2662f6e1ba77eb17 diff --git a/src/common/thirdparty/utarray.h b/src/common/thirdparty/utarray.h new file mode 100644 index 000000000..979e99e98 --- /dev/null +++ b/src/common/thirdparty/utarray.h @@ -0,0 +1,238 @@ +/* +Copyright (c) 2008-2016, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* a dynamic array implementation using macros + */ +#ifndef UTARRAY_H +#define UTARRAY_H + +#define UTARRAY_VERSION 2.0.1 + +#ifdef __GNUC__ +#define _UNUSED_ __attribute__ ((__unused__)) +#else +#define _UNUSED_ +#endif + +#include /* size_t */ +#include /* memset, etc */ +#include /* exit */ + +#ifndef oom +#define oom() exit(-1) +#endif + +typedef void (ctor_f)(void *dst, const void *src); +typedef void (dtor_f)(void *elt); +typedef void (init_f)(void *elt); +typedef struct { + size_t sz; + init_f *init; + ctor_f *copy; + dtor_f *dtor; +} UT_icd; + +typedef struct { + unsigned i,n;/* i: index of next available slot, n: num slots */ + UT_icd icd; /* initializer, copy and destructor functions */ + char *d; /* n slots of size icd->sz*/ +} UT_array; + +#define utarray_init(a,_icd) do { \ + memset(a,0,sizeof(UT_array)); \ + (a)->icd = *(_icd); \ +} while(0) + +#define utarray_done(a) do { \ + if ((a)->n) { \ + if ((a)->icd.dtor) { \ + unsigned _ut_i; \ + for(_ut_i=0; _ut_i < (a)->i; _ut_i++) { \ + (a)->icd.dtor(utarray_eltptr(a,_ut_i)); \ + } \ + } \ + free((a)->d); \ + } \ + (a)->n=0; \ +} while(0) + +#define utarray_new(a,_icd) do { \ + (a) = (UT_array*)malloc(sizeof(UT_array)); \ + if ((a) == NULL) oom(); \ + utarray_init(a,_icd); \ +} while(0) + +#define utarray_free(a) do { \ + utarray_done(a); \ + free(a); \ +} while(0) + +#define utarray_reserve(a,by) do { \ + if (((a)->i+(by)) > (a)->n) { \ + char *utarray_tmp; \ + while (((a)->i+(by)) > (a)->n) { (a)->n = ((a)->n ? (2*(a)->n) : 8); } \ + utarray_tmp=(char*)realloc((a)->d, (a)->n*(a)->icd.sz); \ + if (utarray_tmp == NULL) oom(); \ + (a)->d=utarray_tmp; \ + } \ +} while(0) + +#define utarray_push_back(a,p) do { \ + utarray_reserve(a,1); \ + if ((a)->icd.copy) { (a)->icd.copy( _utarray_eltptr(a,(a)->i++), p); } \ + else { memcpy(_utarray_eltptr(a,(a)->i++), p, (a)->icd.sz); }; \ +} while(0) + +#define utarray_pop_back(a) do { \ + if ((a)->icd.dtor) { (a)->icd.dtor( _utarray_eltptr(a,--((a)->i))); } \ + else { (a)->i--; } \ +} while(0) + +#define utarray_extend_back(a) do { \ + utarray_reserve(a,1); \ + if ((a)->icd.init) { (a)->icd.init(_utarray_eltptr(a,(a)->i)); } \ + else { memset(_utarray_eltptr(a,(a)->i),0,(a)->icd.sz); } \ + (a)->i++; \ +} while(0) + +#define utarray_len(a) ((a)->i) + +#define utarray_eltptr(a,j) (((j) < (a)->i) ? _utarray_eltptr(a,j) : NULL) +#define _utarray_eltptr(a,j) ((a)->d + ((a)->icd.sz * (j))) + +#define utarray_insert(a,p,j) do { \ + if ((j) > (a)->i) utarray_resize(a,j); \ + utarray_reserve(a,1); \ + if ((j) < (a)->i) { \ + memmove( _utarray_eltptr(a,(j)+1), _utarray_eltptr(a,j), \ + ((a)->i - (j))*((a)->icd.sz)); \ + } \ + if ((a)->icd.copy) { (a)->icd.copy( _utarray_eltptr(a,j), p); } \ + else { memcpy(_utarray_eltptr(a,j), p, (a)->icd.sz); }; \ + (a)->i++; \ +} while(0) + +#define utarray_inserta(a,w,j) do { \ + if (utarray_len(w) == 0) break; \ + if ((j) > (a)->i) utarray_resize(a,j); \ + utarray_reserve(a,utarray_len(w)); \ + if ((j) < (a)->i) { \ + memmove(_utarray_eltptr(a,(j)+utarray_len(w)), \ + _utarray_eltptr(a,j), \ + ((a)->i - (j))*((a)->icd.sz)); \ + } \ + if ((a)->icd.copy) { \ + unsigned _ut_i; \ + for(_ut_i=0;_ut_i<(w)->i;_ut_i++) { \ + (a)->icd.copy(_utarray_eltptr(a, (j) + _ut_i), _utarray_eltptr(w, _ut_i)); \ + } \ + } else { \ + memcpy(_utarray_eltptr(a,j), _utarray_eltptr(w,0), \ + utarray_len(w)*((a)->icd.sz)); \ + } \ + (a)->i += utarray_len(w); \ +} while(0) + +#define utarray_resize(dst,num) do { \ + unsigned _ut_i; \ + if ((dst)->i > (unsigned)(num)) { \ + if ((dst)->icd.dtor) { \ + for (_ut_i = (num); _ut_i < (dst)->i; ++_ut_i) { \ + (dst)->icd.dtor(_utarray_eltptr(dst, _ut_i)); \ + } \ + } \ + } else if ((dst)->i < (unsigned)(num)) { \ + utarray_reserve(dst, (num) - (dst)->i); \ + if ((dst)->icd.init) { \ + for (_ut_i = (dst)->i; _ut_i < (unsigned)(num); ++_ut_i) { \ + (dst)->icd.init(_utarray_eltptr(dst, _ut_i)); \ + } \ + } else { \ + memset(_utarray_eltptr(dst, (dst)->i), 0, (dst)->icd.sz*((num) - (dst)->i)); \ + } \ + } \ + (dst)->i = (num); \ +} while(0) + +#define utarray_concat(dst,src) do { \ + utarray_inserta(dst, src, utarray_len(dst)); \ +} while(0) + +#define utarray_erase(a,pos,len) do { \ + if ((a)->icd.dtor) { \ + unsigned _ut_i; \ + for (_ut_i = 0; _ut_i < (len); _ut_i++) { \ + (a)->icd.dtor(utarray_eltptr(a, (pos) + _ut_i)); \ + } \ + } \ + if ((a)->i > ((pos) + (len))) { \ + memmove(_utarray_eltptr(a, pos), _utarray_eltptr(a, (pos) + (len)), \ + ((a)->i - ((pos) + (len))) * (a)->icd.sz); \ + } \ + (a)->i -= (len); \ +} while(0) + +#define utarray_renew(a,u) do { \ + if (a) utarray_clear(a); \ + else utarray_new(a, u); \ +} while(0) + +#define utarray_clear(a) do { \ + if ((a)->i > 0) { \ + if ((a)->icd.dtor) { \ + unsigned _ut_i; \ + for(_ut_i=0; _ut_i < (a)->i; _ut_i++) { \ + (a)->icd.dtor(_utarray_eltptr(a, _ut_i)); \ + } \ + } \ + (a)->i = 0; \ + } \ +} while(0) + +#define utarray_sort(a,cmp) do { \ + qsort((a)->d, (a)->i, (a)->icd.sz, cmp); \ +} while(0) + +#define utarray_find(a,v,cmp) bsearch((v),(a)->d,(a)->i,(a)->icd.sz,cmp) + +#define utarray_front(a) (((a)->i) ? (_utarray_eltptr(a,0)) : NULL) +#define utarray_next(a,e) (((e)==NULL) ? utarray_front(a) : ((((a)->i) > (utarray_eltidx(a,e)+1)) ? _utarray_eltptr(a,utarray_eltidx(a,e)+1) : NULL)) +#define utarray_prev(a,e) (((e)==NULL) ? utarray_back(a) : ((utarray_eltidx(a,e) > 0) ? _utarray_eltptr(a,utarray_eltidx(a,e)-1) : NULL)) +#define utarray_back(a) (((a)->i) ? (_utarray_eltptr(a,(a)->i-1)) : NULL) +#define utarray_eltidx(a,e) (((char*)(e) >= (a)->d) ? (((char*)(e) - (a)->d)/(a)->icd.sz) : -1) + +/* last we pre-define a few icd for common utarrays of ints and strings */ +static void utarray_str_cpy(void *dst, const void *src) { + char **_src = (char**)src, **_dst = (char**)dst; + *_dst = (*_src == NULL) ? NULL : strdup(*_src); +} +static void utarray_str_dtor(void *elt) { + char **eltc = (char**)elt; + if (*eltc != NULL) free(*eltc); +} +static const UT_icd ut_str_icd _UNUSED_ = {sizeof(char*),NULL,utarray_str_cpy,utarray_str_dtor}; +static const UT_icd ut_int_icd _UNUSED_ = {sizeof(int),NULL,NULL,NULL}; +static const UT_icd ut_ptr_icd _UNUSED_ = {sizeof(void*),NULL,NULL,NULL}; + + +#endif /* UTARRAY_H */ diff --git a/src/common/thirdparty/uthash.h b/src/common/thirdparty/uthash.h new file mode 100644 index 000000000..45d1f9fc1 --- /dev/null +++ b/src/common/thirdparty/uthash.h @@ -0,0 +1,1074 @@ +/* +Copyright (c) 2003-2016, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef UTHASH_H +#define UTHASH_H + +#define UTHASH_VERSION 2.0.1 + +#include /* memcmp,strlen */ +#include /* ptrdiff_t */ +#include /* exit() */ + +/* These macros use decltype or the earlier __typeof GNU extension. + As decltype is only available in newer compilers (VS2010 or gcc 4.3+ + when compiling c++ source) this code uses whatever method is needed + or, for VS2008 where neither is available, uses casting workarounds. */ +#if defined(_MSC_VER) /* MS compiler */ +#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ +#define DECLTYPE(x) (decltype(x)) +#else /* VS2008 or older (or VS2010 in C mode) */ +#define NO_DECLTYPE +#define DECLTYPE(x) +#endif +#elif defined(__BORLANDC__) || defined(__LCC__) || defined(__WATCOMC__) +#define NO_DECLTYPE +#define DECLTYPE(x) +#else /* GNU, Sun and other compilers */ +#define DECLTYPE(x) (__typeof(x)) +#endif + +#ifdef NO_DECLTYPE +#define DECLTYPE_ASSIGN(dst,src) \ +do { \ + char **_da_dst = (char**)(&(dst)); \ + *_da_dst = (char*)(src); \ +} while (0) +#else +#define DECLTYPE_ASSIGN(dst,src) \ +do { \ + (dst) = DECLTYPE(dst)(src); \ +} while (0) +#endif + +/* a number of the hash function use uint32_t which isn't defined on Pre VS2010 */ +#if defined(_WIN32) +#if defined(_MSC_VER) && _MSC_VER >= 1600 +#include +#elif defined(__WATCOMC__) || defined(__MINGW32__) || defined(__CYGWIN__) +#include +#else +typedef unsigned int uint32_t; +typedef unsigned char uint8_t; +#endif +#elif defined(__GNUC__) && !defined(__VXWORKS__) +#include +#else +typedef unsigned int uint32_t; +typedef unsigned char uint8_t; +#endif + +#ifndef uthash_fatal +#define uthash_fatal(msg) exit(-1) /* fatal error (out of memory,etc) */ +#endif +#ifndef uthash_malloc +#define uthash_malloc(sz) malloc(sz) /* malloc fcn */ +#endif +#ifndef uthash_free +#define uthash_free(ptr,sz) free(ptr) /* free fcn */ +#endif +#ifndef uthash_strlen +#define uthash_strlen(s) strlen(s) +#endif +#ifndef uthash_memcmp +#define uthash_memcmp(a,b,n) memcmp(a,b,n) +#endif + +#ifndef uthash_noexpand_fyi +#define uthash_noexpand_fyi(tbl) /* can be defined to log noexpand */ +#endif +#ifndef uthash_expand_fyi +#define uthash_expand_fyi(tbl) /* can be defined to log expands */ +#endif + +/* initial number of buckets */ +#define HASH_INITIAL_NUM_BUCKETS 32U /* initial number of buckets */ +#define HASH_INITIAL_NUM_BUCKETS_LOG2 5U /* lg2 of initial number of buckets */ +#define HASH_BKT_CAPACITY_THRESH 10U /* expand when bucket count reaches */ + +/* calculate the element whose hash handle address is hhp */ +#define ELMT_FROM_HH(tbl,hhp) ((void*)(((char*)(hhp)) - ((tbl)->hho))) +/* calculate the hash handle from element address elp */ +#define HH_FROM_ELMT(tbl,elp) ((UT_hash_handle *)(((char*)(elp)) + ((tbl)->hho))) + +#define HASH_VALUE(keyptr,keylen,hashv) \ +do { \ + HASH_FCN(keyptr, keylen, hashv); \ +} while (0) + +#define HASH_FIND_BYHASHVALUE(hh,head,keyptr,keylen,hashval,out) \ +do { \ + (out) = NULL; \ + if (head) { \ + unsigned _hf_bkt; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _hf_bkt); \ + if (HASH_BLOOM_TEST((head)->hh.tbl, hashval) != 0) { \ + HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ], keyptr, keylen, hashval, out); \ + } \ + } \ +} while (0) + +#define HASH_FIND(hh,head,keyptr,keylen,out) \ +do { \ + unsigned _hf_hashv; \ + HASH_VALUE(keyptr, keylen, _hf_hashv); \ + HASH_FIND_BYHASHVALUE(hh, head, keyptr, keylen, _hf_hashv, out); \ +} while (0) + +#ifdef HASH_BLOOM +#define HASH_BLOOM_BITLEN (1UL << HASH_BLOOM) +#define HASH_BLOOM_BYTELEN (HASH_BLOOM_BITLEN/8UL) + (((HASH_BLOOM_BITLEN%8UL)!=0UL) ? 1UL : 0UL) +#define HASH_BLOOM_MAKE(tbl) \ +do { \ + (tbl)->bloom_nbits = HASH_BLOOM; \ + (tbl)->bloom_bv = (uint8_t*)uthash_malloc(HASH_BLOOM_BYTELEN); \ + if (!((tbl)->bloom_bv)) { uthash_fatal( "out of memory"); } \ + memset((tbl)->bloom_bv, 0, HASH_BLOOM_BYTELEN); \ + (tbl)->bloom_sig = HASH_BLOOM_SIGNATURE; \ +} while (0) + +#define HASH_BLOOM_FREE(tbl) \ +do { \ + uthash_free((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \ +} while (0) + +#define HASH_BLOOM_BITSET(bv,idx) (bv[(idx)/8U] |= (1U << ((idx)%8U))) +#define HASH_BLOOM_BITTEST(bv,idx) (bv[(idx)/8U] & (1U << ((idx)%8U))) + +#define HASH_BLOOM_ADD(tbl,hashv) \ + HASH_BLOOM_BITSET((tbl)->bloom_bv, (hashv & (uint32_t)((1ULL << (tbl)->bloom_nbits) - 1U))) + +#define HASH_BLOOM_TEST(tbl,hashv) \ + HASH_BLOOM_BITTEST((tbl)->bloom_bv, (hashv & (uint32_t)((1ULL << (tbl)->bloom_nbits) - 1U))) + +#else +#define HASH_BLOOM_MAKE(tbl) +#define HASH_BLOOM_FREE(tbl) +#define HASH_BLOOM_ADD(tbl,hashv) +#define HASH_BLOOM_TEST(tbl,hashv) (1) +#define HASH_BLOOM_BYTELEN 0U +#endif + +#define HASH_MAKE_TABLE(hh,head) \ +do { \ + (head)->hh.tbl = (UT_hash_table*)uthash_malloc( \ + sizeof(UT_hash_table)); \ + if (!((head)->hh.tbl)) { uthash_fatal( "out of memory"); } \ + memset((head)->hh.tbl, 0, sizeof(UT_hash_table)); \ + (head)->hh.tbl->tail = &((head)->hh); \ + (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS; \ + (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2; \ + (head)->hh.tbl->hho = (char*)(&(head)->hh) - (char*)(head); \ + (head)->hh.tbl->buckets = (UT_hash_bucket*)uthash_malloc( \ + HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \ + if (! (head)->hh.tbl->buckets) { uthash_fatal( "out of memory"); } \ + memset((head)->hh.tbl->buckets, 0, \ + HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \ + HASH_BLOOM_MAKE((head)->hh.tbl); \ + (head)->hh.tbl->signature = HASH_SIGNATURE; \ +} while (0) + +#define HASH_REPLACE_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,replaced,cmpfcn) \ +do { \ + (replaced) = NULL; \ + HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \ + if (replaced) { \ + HASH_DELETE(hh, head, replaced); \ + } \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn); \ +} while (0) + +#define HASH_REPLACE_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add,replaced) \ +do { \ + (replaced) = NULL; \ + HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \ + if (replaced) { \ + HASH_DELETE(hh, head, replaced); \ + } \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add); \ +} while (0) + +#define HASH_REPLACE(hh,head,fieldname,keylen_in,add,replaced) \ +do { \ + unsigned _hr_hashv; \ + HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ + HASH_REPLACE_BYHASHVALUE(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced); \ +} while (0) + +#define HASH_REPLACE_INORDER(hh,head,fieldname,keylen_in,add,replaced,cmpfcn) \ +do { \ + unsigned _hr_hashv; \ + HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ + HASH_REPLACE_BYHASHVALUE_INORDER(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced, cmpfcn); \ +} while (0) + +#define HASH_APPEND_LIST(hh, head, add) \ +do { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail); \ + (head)->hh.tbl->tail->next = (add); \ + (head)->hh.tbl->tail = &((add)->hh); \ +} while (0) + +#define HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh,head,keyptr,keylen_in,hashval,add,cmpfcn) \ +do { \ + unsigned _ha_bkt; \ + (add)->hh.hashv = (hashval); \ + (add)->hh.key = (char*) (keyptr); \ + (add)->hh.keylen = (unsigned) (keylen_in); \ + if (!(head)) { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = NULL; \ + (head) = (add); \ + HASH_MAKE_TABLE(hh, head); \ + } else { \ + struct UT_hash_handle *_hs_iter = &(head)->hh; \ + (add)->hh.tbl = (head)->hh.tbl; \ + do { \ + if (cmpfcn(DECLTYPE(head) ELMT_FROM_HH((head)->hh.tbl, _hs_iter), add) > 0) \ + break; \ + } while ((_hs_iter = _hs_iter->next)); \ + if (_hs_iter) { \ + (add)->hh.next = _hs_iter; \ + if (((add)->hh.prev = _hs_iter->prev)) { \ + HH_FROM_ELMT((head)->hh.tbl, _hs_iter->prev)->next = (add); \ + } else { \ + (head) = (add); \ + } \ + _hs_iter->prev = (add); \ + } else { \ + HASH_APPEND_LIST(hh, head, add); \ + } \ + } \ + (head)->hh.tbl->num_items++; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ + HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], &(add)->hh); \ + HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ + HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ + HASH_FSCK(hh, head); \ +} while (0) + +#define HASH_ADD_KEYPTR_INORDER(hh,head,keyptr,keylen_in,add,cmpfcn) \ +do { \ + unsigned _hs_hashv; \ + HASH_VALUE(keyptr, keylen_in, _hs_hashv); \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, keyptr, keylen_in, _hs_hashv, add, cmpfcn); \ +} while (0) + +#define HASH_ADD_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,cmpfcn) \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn) + +#define HASH_ADD_INORDER(hh,head,fieldname,keylen_in,add,cmpfcn) \ + HASH_ADD_KEYPTR_INORDER(hh, head, &((add)->fieldname), keylen_in, add, cmpfcn) + +#define HASH_ADD_KEYPTR_BYHASHVALUE(hh,head,keyptr,keylen_in,hashval,add) \ +do { \ + unsigned _ha_bkt; \ + (add)->hh.hashv = (hashval); \ + (add)->hh.key = (char*) (keyptr); \ + (add)->hh.keylen = (unsigned) (keylen_in); \ + if (!(head)) { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = NULL; \ + (head) = (add); \ + HASH_MAKE_TABLE(hh, head); \ + } else { \ + (add)->hh.tbl = (head)->hh.tbl; \ + HASH_APPEND_LIST(hh, head, add); \ + } \ + (head)->hh.tbl->num_items++; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ + HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], &(add)->hh); \ + HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ + HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ + HASH_FSCK(hh, head); \ +} while (0) + +#define HASH_ADD_KEYPTR(hh,head,keyptr,keylen_in,add) \ +do { \ + unsigned _ha_hashv; \ + HASH_VALUE(keyptr, keylen_in, _ha_hashv); \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, keyptr, keylen_in, _ha_hashv, add); \ +} while (0) + +#define HASH_ADD_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add) \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add) + +#define HASH_ADD(hh,head,fieldname,keylen_in,add) \ + HASH_ADD_KEYPTR(hh, head, &((add)->fieldname), keylen_in, add) + +#define HASH_TO_BKT(hashv,num_bkts,bkt) \ +do { \ + bkt = ((hashv) & ((num_bkts) - 1U)); \ +} while (0) + +/* delete "delptr" from the hash table. + * "the usual" patch-up process for the app-order doubly-linked-list. + * The use of _hd_hh_del below deserves special explanation. + * These used to be expressed using (delptr) but that led to a bug + * if someone used the same symbol for the head and deletee, like + * HASH_DELETE(hh,users,users); + * We want that to work, but by changing the head (users) below + * we were forfeiting our ability to further refer to the deletee (users) + * in the patch-up process. Solution: use scratch space to + * copy the deletee pointer, then the latter references are via that + * scratch pointer rather than through the repointed (users) symbol. + */ +#define HASH_DELETE(hh,head,delptr) \ +do { \ + struct UT_hash_handle *_hd_hh_del; \ + if ( ((delptr)->hh.prev == NULL) && ((delptr)->hh.next == NULL) ) { \ + uthash_free((head)->hh.tbl->buckets, \ + (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket) ); \ + HASH_BLOOM_FREE((head)->hh.tbl); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + head = NULL; \ + } else { \ + unsigned _hd_bkt; \ + _hd_hh_del = &((delptr)->hh); \ + if ((delptr) == ELMT_FROM_HH((head)->hh.tbl,(head)->hh.tbl->tail)) { \ + (head)->hh.tbl->tail = \ + (UT_hash_handle*)((ptrdiff_t)((delptr)->hh.prev) + \ + (head)->hh.tbl->hho); \ + } \ + if ((delptr)->hh.prev != NULL) { \ + ((UT_hash_handle*)((ptrdiff_t)((delptr)->hh.prev) + \ + (head)->hh.tbl->hho))->next = (delptr)->hh.next; \ + } else { \ + DECLTYPE_ASSIGN(head,(delptr)->hh.next); \ + } \ + if (_hd_hh_del->next != NULL) { \ + ((UT_hash_handle*)((ptrdiff_t)_hd_hh_del->next + \ + (head)->hh.tbl->hho))->prev = \ + _hd_hh_del->prev; \ + } \ + HASH_TO_BKT( _hd_hh_del->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \ + HASH_DEL_IN_BKT(hh,(head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del); \ + (head)->hh.tbl->num_items--; \ + } \ + HASH_FSCK(hh,head); \ +} while (0) + + +/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */ +#define HASH_FIND_STR(head,findstr,out) \ + HASH_FIND(hh,head,findstr,(unsigned)uthash_strlen(findstr),out) +#define HASH_ADD_STR(head,strfield,add) \ + HASH_ADD(hh,head,strfield[0],(unsigned)uthash_strlen(add->strfield),add) +#define HASH_REPLACE_STR(head,strfield,add,replaced) \ + HASH_REPLACE(hh,head,strfield[0],(unsigned)uthash_strlen(add->strfield),add,replaced) +#define HASH_FIND_INT(head,findint,out) \ + HASH_FIND(hh,head,findint,sizeof(int),out) +#define HASH_ADD_INT(head,intfield,add) \ + HASH_ADD(hh,head,intfield,sizeof(int),add) +#define HASH_REPLACE_INT(head,intfield,add,replaced) \ + HASH_REPLACE(hh,head,intfield,sizeof(int),add,replaced) +#define HASH_FIND_PTR(head,findptr,out) \ + HASH_FIND(hh,head,findptr,sizeof(void *),out) +#define HASH_ADD_PTR(head,ptrfield,add) \ + HASH_ADD(hh,head,ptrfield,sizeof(void *),add) +#define HASH_REPLACE_PTR(head,ptrfield,add,replaced) \ + HASH_REPLACE(hh,head,ptrfield,sizeof(void *),add,replaced) +#define HASH_DEL(head,delptr) \ + HASH_DELETE(hh,head,delptr) + +/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined. + * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined. + */ +#ifdef HASH_DEBUG +#define HASH_OOPS(...) do { fprintf(stderr,__VA_ARGS__); exit(-1); } while (0) +#define HASH_FSCK(hh,head) \ +do { \ + struct UT_hash_handle *_thh; \ + if (head) { \ + unsigned _bkt_i; \ + unsigned _count; \ + char *_prev; \ + _count = 0; \ + for( _bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; _bkt_i++) { \ + unsigned _bkt_count = 0; \ + _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head; \ + _prev = NULL; \ + while (_thh) { \ + if (_prev != (char*)(_thh->hh_prev)) { \ + HASH_OOPS("invalid hh_prev %p, actual %p\n", \ + _thh->hh_prev, _prev ); \ + } \ + _bkt_count++; \ + _prev = (char*)(_thh); \ + _thh = _thh->hh_next; \ + } \ + _count += _bkt_count; \ + if ((head)->hh.tbl->buckets[_bkt_i].count != _bkt_count) { \ + HASH_OOPS("invalid bucket count %u, actual %u\n", \ + (head)->hh.tbl->buckets[_bkt_i].count, _bkt_count); \ + } \ + } \ + if (_count != (head)->hh.tbl->num_items) { \ + HASH_OOPS("invalid hh item count %u, actual %u\n", \ + (head)->hh.tbl->num_items, _count ); \ + } \ + /* traverse hh in app order; check next/prev integrity, count */ \ + _count = 0; \ + _prev = NULL; \ + _thh = &(head)->hh; \ + while (_thh) { \ + _count++; \ + if (_prev !=(char*)(_thh->prev)) { \ + HASH_OOPS("invalid prev %p, actual %p\n", \ + _thh->prev, _prev ); \ + } \ + _prev = (char*)ELMT_FROM_HH((head)->hh.tbl, _thh); \ + _thh = ( _thh->next ? (UT_hash_handle*)((char*)(_thh->next) + \ + (head)->hh.tbl->hho) : NULL ); \ + } \ + if (_count != (head)->hh.tbl->num_items) { \ + HASH_OOPS("invalid app item count %u, actual %u\n", \ + (head)->hh.tbl->num_items, _count ); \ + } \ + } \ +} while (0) +#else +#define HASH_FSCK(hh,head) +#endif + +/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to + * the descriptor to which this macro is defined for tuning the hash function. + * The app can #include to get the prototype for write(2). */ +#ifdef HASH_EMIT_KEYS +#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) \ +do { \ + unsigned _klen = fieldlen; \ + write(HASH_EMIT_KEYS, &_klen, sizeof(_klen)); \ + write(HASH_EMIT_KEYS, keyptr, (unsigned long)fieldlen); \ +} while (0) +#else +#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) +#endif + +/* default to Jenkin's hash unless overridden e.g. DHASH_FUNCTION=HASH_SAX */ +#ifdef HASH_FUNCTION +#define HASH_FCN HASH_FUNCTION +#else +#define HASH_FCN HASH_JEN +#endif + +/* The Bernstein hash function, used in Perl prior to v5.6. Note (x<<5+x)=x*33. */ +#define HASH_BER(key,keylen,hashv) \ +do { \ + unsigned _hb_keylen=(unsigned)keylen; \ + const unsigned char *_hb_key=(const unsigned char*)(key); \ + (hashv) = 0; \ + while (_hb_keylen-- != 0U) { \ + (hashv) = (((hashv) << 5) + (hashv)) + *_hb_key++; \ + } \ +} while (0) + + +/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at + * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */ +#define HASH_SAX(key,keylen,hashv) \ +do { \ + unsigned _sx_i; \ + const unsigned char *_hs_key=(const unsigned char*)(key); \ + hashv = 0; \ + for(_sx_i=0; _sx_i < keylen; _sx_i++) { \ + hashv ^= (hashv << 5) + (hashv >> 2) + _hs_key[_sx_i]; \ + } \ +} while (0) +/* FNV-1a variation */ +#define HASH_FNV(key,keylen,hashv) \ +do { \ + unsigned _fn_i; \ + const unsigned char *_hf_key=(const unsigned char*)(key); \ + hashv = 2166136261U; \ + for(_fn_i=0; _fn_i < keylen; _fn_i++) { \ + hashv = hashv ^ _hf_key[_fn_i]; \ + hashv = hashv * 16777619U; \ + } \ +} while (0) + +#define HASH_OAT(key,keylen,hashv) \ +do { \ + unsigned _ho_i; \ + const unsigned char *_ho_key=(const unsigned char*)(key); \ + hashv = 0; \ + for(_ho_i=0; _ho_i < keylen; _ho_i++) { \ + hashv += _ho_key[_ho_i]; \ + hashv += (hashv << 10); \ + hashv ^= (hashv >> 6); \ + } \ + hashv += (hashv << 3); \ + hashv ^= (hashv >> 11); \ + hashv += (hashv << 15); \ +} while (0) + +#define HASH_JEN_MIX(a,b,c) \ +do { \ + a -= b; a -= c; a ^= ( c >> 13 ); \ + b -= c; b -= a; b ^= ( a << 8 ); \ + c -= a; c -= b; c ^= ( b >> 13 ); \ + a -= b; a -= c; a ^= ( c >> 12 ); \ + b -= c; b -= a; b ^= ( a << 16 ); \ + c -= a; c -= b; c ^= ( b >> 5 ); \ + a -= b; a -= c; a ^= ( c >> 3 ); \ + b -= c; b -= a; b ^= ( a << 10 ); \ + c -= a; c -= b; c ^= ( b >> 15 ); \ +} while (0) + +#define HASH_JEN(key,keylen,hashv) \ +do { \ + unsigned _hj_i,_hj_j,_hj_k; \ + unsigned const char *_hj_key=(unsigned const char*)(key); \ + hashv = 0xfeedbeefu; \ + _hj_i = _hj_j = 0x9e3779b9u; \ + _hj_k = (unsigned)(keylen); \ + while (_hj_k >= 12U) { \ + _hj_i += (_hj_key[0] + ( (unsigned)_hj_key[1] << 8 ) \ + + ( (unsigned)_hj_key[2] << 16 ) \ + + ( (unsigned)_hj_key[3] << 24 ) ); \ + _hj_j += (_hj_key[4] + ( (unsigned)_hj_key[5] << 8 ) \ + + ( (unsigned)_hj_key[6] << 16 ) \ + + ( (unsigned)_hj_key[7] << 24 ) ); \ + hashv += (_hj_key[8] + ( (unsigned)_hj_key[9] << 8 ) \ + + ( (unsigned)_hj_key[10] << 16 ) \ + + ( (unsigned)_hj_key[11] << 24 ) ); \ + \ + HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ + \ + _hj_key += 12; \ + _hj_k -= 12U; \ + } \ + hashv += (unsigned)(keylen); \ + switch ( _hj_k ) { \ + case 11: hashv += ( (unsigned)_hj_key[10] << 24 ); /* FALLTHROUGH */ \ + case 10: hashv += ( (unsigned)_hj_key[9] << 16 ); /* FALLTHROUGH */ \ + case 9: hashv += ( (unsigned)_hj_key[8] << 8 ); /* FALLTHROUGH */ \ + case 8: _hj_j += ( (unsigned)_hj_key[7] << 24 ); /* FALLTHROUGH */ \ + case 7: _hj_j += ( (unsigned)_hj_key[6] << 16 ); /* FALLTHROUGH */ \ + case 6: _hj_j += ( (unsigned)_hj_key[5] << 8 ); /* FALLTHROUGH */ \ + case 5: _hj_j += _hj_key[4]; /* FALLTHROUGH */ \ + case 4: _hj_i += ( (unsigned)_hj_key[3] << 24 ); /* FALLTHROUGH */ \ + case 3: _hj_i += ( (unsigned)_hj_key[2] << 16 ); /* FALLTHROUGH */ \ + case 2: _hj_i += ( (unsigned)_hj_key[1] << 8 ); /* FALLTHROUGH */ \ + case 1: _hj_i += _hj_key[0]; \ + } \ + HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ +} while (0) + +/* The Paul Hsieh hash function */ +#undef get16bits +#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__) \ + || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__) +#define get16bits(d) (*((const uint16_t *) (d))) +#endif + +#if !defined (get16bits) +#define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8) \ + +(uint32_t)(((const uint8_t *)(d))[0]) ) +#endif +#define HASH_SFH(key,keylen,hashv) \ +do { \ + unsigned const char *_sfh_key=(unsigned const char*)(key); \ + uint32_t _sfh_tmp, _sfh_len = (uint32_t)keylen; \ + \ + unsigned _sfh_rem = _sfh_len & 3U; \ + _sfh_len >>= 2; \ + hashv = 0xcafebabeu; \ + \ + /* Main loop */ \ + for (;_sfh_len > 0U; _sfh_len--) { \ + hashv += get16bits (_sfh_key); \ + _sfh_tmp = ((uint32_t)(get16bits (_sfh_key+2)) << 11) ^ hashv; \ + hashv = (hashv << 16) ^ _sfh_tmp; \ + _sfh_key += 2U*sizeof (uint16_t); \ + hashv += hashv >> 11; \ + } \ + \ + /* Handle end cases */ \ + switch (_sfh_rem) { \ + case 3: hashv += get16bits (_sfh_key); \ + hashv ^= hashv << 16; \ + hashv ^= (uint32_t)(_sfh_key[sizeof (uint16_t)]) << 18; \ + hashv += hashv >> 11; \ + break; \ + case 2: hashv += get16bits (_sfh_key); \ + hashv ^= hashv << 11; \ + hashv += hashv >> 17; \ + break; \ + case 1: hashv += *_sfh_key; \ + hashv ^= hashv << 10; \ + hashv += hashv >> 1; \ + } \ + \ + /* Force "avalanching" of final 127 bits */ \ + hashv ^= hashv << 3; \ + hashv += hashv >> 5; \ + hashv ^= hashv << 4; \ + hashv += hashv >> 17; \ + hashv ^= hashv << 25; \ + hashv += hashv >> 6; \ +} while (0) + +#ifdef HASH_USING_NO_STRICT_ALIASING +/* The MurmurHash exploits some CPU's (x86,x86_64) tolerance for unaligned reads. + * For other types of CPU's (e.g. Sparc) an unaligned read causes a bus error. + * MurmurHash uses the faster approach only on CPU's where we know it's safe. + * + * Note the preprocessor built-in defines can be emitted using: + * + * gcc -m64 -dM -E - < /dev/null (on gcc) + * cc -## a.c (where a.c is a simple test file) (Sun Studio) + */ +#if (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86)) +#define MUR_GETBLOCK(p,i) p[i] +#else /* non intel */ +#define MUR_PLUS0_ALIGNED(p) (((unsigned long)p & 3UL) == 0UL) +#define MUR_PLUS1_ALIGNED(p) (((unsigned long)p & 3UL) == 1UL) +#define MUR_PLUS2_ALIGNED(p) (((unsigned long)p & 3UL) == 2UL) +#define MUR_PLUS3_ALIGNED(p) (((unsigned long)p & 3UL) == 3UL) +#define WP(p) ((uint32_t*)((unsigned long)(p) & ~3UL)) +#if (defined(__BIG_ENDIAN__) || defined(SPARC) || defined(__ppc__) || defined(__ppc64__)) +#define MUR_THREE_ONE(p) ((((*WP(p))&0x00ffffff) << 8) | (((*(WP(p)+1))&0xff000000) >> 24)) +#define MUR_TWO_TWO(p) ((((*WP(p))&0x0000ffff) <<16) | (((*(WP(p)+1))&0xffff0000) >> 16)) +#define MUR_ONE_THREE(p) ((((*WP(p))&0x000000ff) <<24) | (((*(WP(p)+1))&0xffffff00) >> 8)) +#else /* assume little endian non-intel */ +#define MUR_THREE_ONE(p) ((((*WP(p))&0xffffff00) >> 8) | (((*(WP(p)+1))&0x000000ff) << 24)) +#define MUR_TWO_TWO(p) ((((*WP(p))&0xffff0000) >>16) | (((*(WP(p)+1))&0x0000ffff) << 16)) +#define MUR_ONE_THREE(p) ((((*WP(p))&0xff000000) >>24) | (((*(WP(p)+1))&0x00ffffff) << 8)) +#endif +#define MUR_GETBLOCK(p,i) (MUR_PLUS0_ALIGNED(p) ? ((p)[i]) : \ + (MUR_PLUS1_ALIGNED(p) ? MUR_THREE_ONE(p) : \ + (MUR_PLUS2_ALIGNED(p) ? MUR_TWO_TWO(p) : \ + MUR_ONE_THREE(p)))) +#endif +#define MUR_ROTL32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +#define MUR_FMIX(_h) \ +do { \ + _h ^= _h >> 16; \ + _h *= 0x85ebca6bu; \ + _h ^= _h >> 13; \ + _h *= 0xc2b2ae35u; \ + _h ^= _h >> 16; \ +} while (0) + +#define HASH_MUR(key,keylen,hashv) \ +do { \ + const uint8_t *_mur_data = (const uint8_t*)(key); \ + const int _mur_nblocks = (int)(keylen) / 4; \ + uint32_t _mur_h1 = 0xf88D5353u; \ + uint32_t _mur_c1 = 0xcc9e2d51u; \ + uint32_t _mur_c2 = 0x1b873593u; \ + uint32_t _mur_k1 = 0; \ + const uint8_t *_mur_tail; \ + const uint32_t *_mur_blocks = (const uint32_t*)(_mur_data+(_mur_nblocks*4)); \ + int _mur_i; \ + for(_mur_i = -_mur_nblocks; _mur_i!=0; _mur_i++) { \ + _mur_k1 = MUR_GETBLOCK(_mur_blocks,_mur_i); \ + _mur_k1 *= _mur_c1; \ + _mur_k1 = MUR_ROTL32(_mur_k1,15); \ + _mur_k1 *= _mur_c2; \ + \ + _mur_h1 ^= _mur_k1; \ + _mur_h1 = MUR_ROTL32(_mur_h1,13); \ + _mur_h1 = (_mur_h1*5U) + 0xe6546b64u; \ + } \ + _mur_tail = (const uint8_t*)(_mur_data + (_mur_nblocks*4)); \ + _mur_k1=0; \ + switch((keylen) & 3U) { \ + case 3: _mur_k1 ^= (uint32_t)_mur_tail[2] << 16; /* FALLTHROUGH */ \ + case 2: _mur_k1 ^= (uint32_t)_mur_tail[1] << 8; /* FALLTHROUGH */ \ + case 1: _mur_k1 ^= (uint32_t)_mur_tail[0]; \ + _mur_k1 *= _mur_c1; \ + _mur_k1 = MUR_ROTL32(_mur_k1,15); \ + _mur_k1 *= _mur_c2; \ + _mur_h1 ^= _mur_k1; \ + } \ + _mur_h1 ^= (uint32_t)(keylen); \ + MUR_FMIX(_mur_h1); \ + hashv = _mur_h1; \ +} while (0) +#endif /* HASH_USING_NO_STRICT_ALIASING */ + +/* iterate over items in a known bucket to find desired item */ +#define HASH_FIND_IN_BKT(tbl,hh,head,keyptr,keylen_in,hashval,out) \ +do { \ + if ((head).hh_head != NULL) { \ + DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (head).hh_head)); \ + } else { \ + (out) = NULL; \ + } \ + while ((out) != NULL) { \ + if ((out)->hh.hashv == (hashval) && (out)->hh.keylen == (keylen_in)) { \ + if (uthash_memcmp((out)->hh.key, keyptr, keylen_in) == 0) { \ + break; \ + } \ + } \ + if ((out)->hh.hh_next != NULL) { \ + DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (out)->hh.hh_next)); \ + } else { \ + (out) = NULL; \ + } \ + } \ +} while (0) + +/* add an item to a bucket */ +#define HASH_ADD_TO_BKT(head,addhh) \ +do { \ + head.count++; \ + (addhh)->hh_next = head.hh_head; \ + (addhh)->hh_prev = NULL; \ + if (head.hh_head != NULL) { (head).hh_head->hh_prev = (addhh); } \ + (head).hh_head=addhh; \ + if ((head.count >= ((head.expand_mult+1U) * HASH_BKT_CAPACITY_THRESH)) \ + && ((addhh)->tbl->noexpand != 1U)) { \ + HASH_EXPAND_BUCKETS((addhh)->tbl); \ + } \ +} while (0) + +/* remove an item from a given bucket */ +#define HASH_DEL_IN_BKT(hh,head,hh_del) \ + (head).count--; \ + if ((head).hh_head == hh_del) { \ + (head).hh_head = hh_del->hh_next; \ + } \ + if (hh_del->hh_prev) { \ + hh_del->hh_prev->hh_next = hh_del->hh_next; \ + } \ + if (hh_del->hh_next) { \ + hh_del->hh_next->hh_prev = hh_del->hh_prev; \ + } + +/* Bucket expansion has the effect of doubling the number of buckets + * and redistributing the items into the new buckets. Ideally the + * items will distribute more or less evenly into the new buckets + * (the extent to which this is true is a measure of the quality of + * the hash function as it applies to the key domain). + * + * With the items distributed into more buckets, the chain length + * (item count) in each bucket is reduced. Thus by expanding buckets + * the hash keeps a bound on the chain length. This bounded chain + * length is the essence of how a hash provides constant time lookup. + * + * The calculation of tbl->ideal_chain_maxlen below deserves some + * explanation. First, keep in mind that we're calculating the ideal + * maximum chain length based on the *new* (doubled) bucket count. + * In fractions this is just n/b (n=number of items,b=new num buckets). + * Since the ideal chain length is an integer, we want to calculate + * ceil(n/b). We don't depend on floating point arithmetic in this + * hash, so to calculate ceil(n/b) with integers we could write + * + * ceil(n/b) = (n/b) + ((n%b)?1:0) + * + * and in fact a previous version of this hash did just that. + * But now we have improved things a bit by recognizing that b is + * always a power of two. We keep its base 2 log handy (call it lb), + * so now we can write this with a bit shift and logical AND: + * + * ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0) + * + */ +#define HASH_EXPAND_BUCKETS(tbl) \ +do { \ + unsigned _he_bkt; \ + unsigned _he_bkt_i; \ + struct UT_hash_handle *_he_thh, *_he_hh_nxt; \ + UT_hash_bucket *_he_new_buckets, *_he_newbkt; \ + _he_new_buckets = (UT_hash_bucket*)uthash_malloc( \ + 2UL * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \ + if (!_he_new_buckets) { uthash_fatal( "out of memory"); } \ + memset(_he_new_buckets, 0, \ + 2UL * tbl->num_buckets * sizeof(struct UT_hash_bucket)); \ + tbl->ideal_chain_maxlen = \ + (tbl->num_items >> (tbl->log2_num_buckets+1U)) + \ + (((tbl->num_items & ((tbl->num_buckets*2U)-1U)) != 0U) ? 1U : 0U); \ + tbl->nonideal_items = 0; \ + for(_he_bkt_i = 0; _he_bkt_i < tbl->num_buckets; _he_bkt_i++) \ + { \ + _he_thh = tbl->buckets[ _he_bkt_i ].hh_head; \ + while (_he_thh != NULL) { \ + _he_hh_nxt = _he_thh->hh_next; \ + HASH_TO_BKT( _he_thh->hashv, tbl->num_buckets*2U, _he_bkt); \ + _he_newbkt = &(_he_new_buckets[ _he_bkt ]); \ + if (++(_he_newbkt->count) > tbl->ideal_chain_maxlen) { \ + tbl->nonideal_items++; \ + _he_newbkt->expand_mult = _he_newbkt->count / \ + tbl->ideal_chain_maxlen; \ + } \ + _he_thh->hh_prev = NULL; \ + _he_thh->hh_next = _he_newbkt->hh_head; \ + if (_he_newbkt->hh_head != NULL) { _he_newbkt->hh_head->hh_prev = \ + _he_thh; } \ + _he_newbkt->hh_head = _he_thh; \ + _he_thh = _he_hh_nxt; \ + } \ + } \ + uthash_free( tbl->buckets, tbl->num_buckets*sizeof(struct UT_hash_bucket) ); \ + tbl->num_buckets *= 2U; \ + tbl->log2_num_buckets++; \ + tbl->buckets = _he_new_buckets; \ + tbl->ineff_expands = (tbl->nonideal_items > (tbl->num_items >> 1)) ? \ + (tbl->ineff_expands+1U) : 0U; \ + if (tbl->ineff_expands > 1U) { \ + tbl->noexpand=1; \ + uthash_noexpand_fyi(tbl); \ + } \ + uthash_expand_fyi(tbl); \ +} while (0) + + +/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */ +/* Note that HASH_SORT assumes the hash handle name to be hh. + * HASH_SRT was added to allow the hash handle name to be passed in. */ +#define HASH_SORT(head,cmpfcn) HASH_SRT(hh,head,cmpfcn) +#define HASH_SRT(hh,head,cmpfcn) \ +do { \ + unsigned _hs_i; \ + unsigned _hs_looping,_hs_nmerges,_hs_insize,_hs_psize,_hs_qsize; \ + struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail; \ + if (head != NULL) { \ + _hs_insize = 1; \ + _hs_looping = 1; \ + _hs_list = &((head)->hh); \ + while (_hs_looping != 0U) { \ + _hs_p = _hs_list; \ + _hs_list = NULL; \ + _hs_tail = NULL; \ + _hs_nmerges = 0; \ + while (_hs_p != NULL) { \ + _hs_nmerges++; \ + _hs_q = _hs_p; \ + _hs_psize = 0; \ + for ( _hs_i = 0; _hs_i < _hs_insize; _hs_i++ ) { \ + _hs_psize++; \ + _hs_q = (UT_hash_handle*)((_hs_q->next != NULL) ? \ + ((void*)((char*)(_hs_q->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + if (! (_hs_q) ) { break; } \ + } \ + _hs_qsize = _hs_insize; \ + while ((_hs_psize > 0U) || ((_hs_qsize > 0U) && (_hs_q != NULL))) {\ + if (_hs_psize == 0U) { \ + _hs_e = _hs_q; \ + _hs_q = (UT_hash_handle*)((_hs_q->next != NULL) ? \ + ((void*)((char*)(_hs_q->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + _hs_qsize--; \ + } else if ( (_hs_qsize == 0U) || (_hs_q == NULL) ) { \ + _hs_e = _hs_p; \ + if (_hs_p != NULL){ \ + _hs_p = (UT_hash_handle*)((_hs_p->next != NULL) ? \ + ((void*)((char*)(_hs_p->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + } \ + _hs_psize--; \ + } else if (( \ + cmpfcn(DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_p)), \ + DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_q))) \ + ) <= 0) { \ + _hs_e = _hs_p; \ + if (_hs_p != NULL){ \ + _hs_p = (UT_hash_handle*)((_hs_p->next != NULL) ? \ + ((void*)((char*)(_hs_p->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + } \ + _hs_psize--; \ + } else { \ + _hs_e = _hs_q; \ + _hs_q = (UT_hash_handle*)((_hs_q->next != NULL) ? \ + ((void*)((char*)(_hs_q->next) + \ + (head)->hh.tbl->hho)) : NULL); \ + _hs_qsize--; \ + } \ + if ( _hs_tail != NULL ) { \ + _hs_tail->next = ((_hs_e != NULL) ? \ + ELMT_FROM_HH((head)->hh.tbl,_hs_e) : NULL); \ + } else { \ + _hs_list = _hs_e; \ + } \ + if (_hs_e != NULL) { \ + _hs_e->prev = ((_hs_tail != NULL) ? \ + ELMT_FROM_HH((head)->hh.tbl,_hs_tail) : NULL); \ + } \ + _hs_tail = _hs_e; \ + } \ + _hs_p = _hs_q; \ + } \ + if (_hs_tail != NULL){ \ + _hs_tail->next = NULL; \ + } \ + if ( _hs_nmerges <= 1U ) { \ + _hs_looping=0; \ + (head)->hh.tbl->tail = _hs_tail; \ + DECLTYPE_ASSIGN(head,ELMT_FROM_HH((head)->hh.tbl, _hs_list)); \ + } \ + _hs_insize *= 2U; \ + } \ + HASH_FSCK(hh,head); \ + } \ +} while (0) + +/* This function selects items from one hash into another hash. + * The end result is that the selected items have dual presence + * in both hashes. There is no copy of the items made; rather + * they are added into the new hash through a secondary hash + * hash handle that must be present in the structure. */ +#define HASH_SELECT(hh_dst, dst, hh_src, src, cond) \ +do { \ + unsigned _src_bkt, _dst_bkt; \ + void *_last_elt=NULL, *_elt; \ + UT_hash_handle *_src_hh, *_dst_hh, *_last_elt_hh=NULL; \ + ptrdiff_t _dst_hho = ((char*)(&(dst)->hh_dst) - (char*)(dst)); \ + if (src != NULL) { \ + for(_src_bkt=0; _src_bkt < (src)->hh_src.tbl->num_buckets; _src_bkt++) { \ + for(_src_hh = (src)->hh_src.tbl->buckets[_src_bkt].hh_head; \ + _src_hh != NULL; \ + _src_hh = _src_hh->hh_next) { \ + _elt = ELMT_FROM_HH((src)->hh_src.tbl, _src_hh); \ + if (cond(_elt)) { \ + _dst_hh = (UT_hash_handle*)(((char*)_elt) + _dst_hho); \ + _dst_hh->key = _src_hh->key; \ + _dst_hh->keylen = _src_hh->keylen; \ + _dst_hh->hashv = _src_hh->hashv; \ + _dst_hh->prev = _last_elt; \ + _dst_hh->next = NULL; \ + if (_last_elt_hh != NULL) { _last_elt_hh->next = _elt; } \ + if (dst == NULL) { \ + DECLTYPE_ASSIGN(dst,_elt); \ + HASH_MAKE_TABLE(hh_dst,dst); \ + } else { \ + _dst_hh->tbl = (dst)->hh_dst.tbl; \ + } \ + HASH_TO_BKT(_dst_hh->hashv, _dst_hh->tbl->num_buckets, _dst_bkt); \ + HASH_ADD_TO_BKT(_dst_hh->tbl->buckets[_dst_bkt],_dst_hh); \ + (dst)->hh_dst.tbl->num_items++; \ + _last_elt = _elt; \ + _last_elt_hh = _dst_hh; \ + } \ + } \ + } \ + } \ + HASH_FSCK(hh_dst,dst); \ +} while (0) + +#define HASH_CLEAR(hh,head) \ +do { \ + if (head != NULL) { \ + uthash_free((head)->hh.tbl->buckets, \ + (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket)); \ + HASH_BLOOM_FREE((head)->hh.tbl); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + (head)=NULL; \ + } \ +} while (0) + +#define HASH_OVERHEAD(hh,head) \ + ((head != NULL) ? ( \ + (size_t)(((head)->hh.tbl->num_items * sizeof(UT_hash_handle)) + \ + ((head)->hh.tbl->num_buckets * sizeof(UT_hash_bucket)) + \ + sizeof(UT_hash_table) + \ + (HASH_BLOOM_BYTELEN))) : 0U) + +#ifdef NO_DECLTYPE +#define HASH_ITER(hh,head,el,tmp) \ +for(((el)=(head)), ((*(char**)(&(tmp)))=(char*)((head!=NULL)?(head)->hh.next:NULL)); \ + (el) != NULL; ((el)=(tmp)), ((*(char**)(&(tmp)))=(char*)((tmp!=NULL)?(tmp)->hh.next:NULL))) +#else +#define HASH_ITER(hh,head,el,tmp) \ +for(((el)=(head)), ((tmp)=DECLTYPE(el)((head!=NULL)?(head)->hh.next:NULL)); \ + (el) != NULL; ((el)=(tmp)), ((tmp)=DECLTYPE(el)((tmp!=NULL)?(tmp)->hh.next:NULL))) +#endif + +/* obtain a count of items in the hash */ +#define HASH_COUNT(head) HASH_CNT(hh,head) +#define HASH_CNT(hh,head) ((head != NULL)?((head)->hh.tbl->num_items):0U) + +typedef struct UT_hash_bucket { + struct UT_hash_handle *hh_head; + unsigned count; + + /* expand_mult is normally set to 0. In this situation, the max chain length + * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If + * the bucket's chain exceeds this length, bucket expansion is triggered). + * However, setting expand_mult to a non-zero value delays bucket expansion + * (that would be triggered by additions to this particular bucket) + * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH. + * (The multiplier is simply expand_mult+1). The whole idea of this + * multiplier is to reduce bucket expansions, since they are expensive, in + * situations where we know that a particular bucket tends to be overused. + * It is better to let its chain length grow to a longer yet-still-bounded + * value, than to do an O(n) bucket expansion too often. + */ + unsigned expand_mult; + +} UT_hash_bucket; + +/* random signature used only to find hash tables in external analysis */ +#define HASH_SIGNATURE 0xa0111fe1u +#define HASH_BLOOM_SIGNATURE 0xb12220f2u + +typedef struct UT_hash_table { + UT_hash_bucket *buckets; + unsigned num_buckets, log2_num_buckets; + unsigned num_items; + struct UT_hash_handle *tail; /* tail hh in app order, for fast append */ + ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */ + + /* in an ideal situation (all buckets used equally), no bucket would have + * more than ceil(#items/#buckets) items. that's the ideal chain length. */ + unsigned ideal_chain_maxlen; + + /* nonideal_items is the number of items in the hash whose chain position + * exceeds the ideal chain maxlen. these items pay the penalty for an uneven + * hash distribution; reaching them in a chain traversal takes >ideal steps */ + unsigned nonideal_items; + + /* ineffective expands occur when a bucket doubling was performed, but + * afterward, more than half the items in the hash had nonideal chain + * positions. If this happens on two consecutive expansions we inhibit any + * further expansion, as it's not helping; this happens when the hash + * function isn't a good fit for the key domain. When expansion is inhibited + * the hash will still work, albeit no longer in constant time. */ + unsigned ineff_expands, noexpand; + + uint32_t signature; /* used only to find hash tables in external analysis */ +#ifdef HASH_BLOOM + uint32_t bloom_sig; /* used only to test bloom exists in external analysis */ + uint8_t *bloom_bv; + uint8_t bloom_nbits; +#endif + +} UT_hash_table; + +typedef struct UT_hash_handle { + struct UT_hash_table *tbl; + void *prev; /* prev element in app order */ + void *next; /* next element in app order */ + struct UT_hash_handle *hh_prev; /* previous hh in bucket order */ + struct UT_hash_handle *hh_next; /* next hh in bucket order */ + void *key; /* ptr to enclosing struct's key */ + unsigned keylen; /* enclosing struct's key len */ + unsigned hashv; /* result of hash-fcn(key) */ +} UT_hash_handle; + +#endif /* UTHASH_H */ diff --git a/src/common/thirdparty/utlist.h b/src/common/thirdparty/utlist.h new file mode 100644 index 000000000..9b5534ffb --- /dev/null +++ b/src/common/thirdparty/utlist.h @@ -0,0 +1,895 @@ +/* +Copyright (c) 2007-2016, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef UTLIST_H +#define UTLIST_H + +#define UTLIST_VERSION 2.0.1 + +#include + +/* + * This file contains macros to manipulate singly and doubly-linked lists. + * + * 1. LL_ macros: singly-linked lists. + * 2. DL_ macros: doubly-linked lists. + * 3. CDL_ macros: circular doubly-linked lists. + * + * To use singly-linked lists, your structure must have a "next" pointer. + * To use doubly-linked lists, your structure must "prev" and "next" pointers. + * Either way, the pointer to the head of the list must be initialized to NULL. + * + * ----------------.EXAMPLE ------------------------- + * struct item { + * int id; + * struct item *prev, *next; + * } + * + * struct item *list = NULL: + * + * int main() { + * struct item *item; + * ... allocate and populate item ... + * DL_APPEND(list, item); + * } + * -------------------------------------------------- + * + * For doubly-linked lists, the append and delete macros are O(1) + * For singly-linked lists, append and delete are O(n) but prepend is O(1) + * The sort macro is O(n log(n)) for all types of single/double/circular lists. + */ + +/* These macros use decltype or the earlier __typeof GNU extension. + As decltype is only available in newer compilers (VS2010 or gcc 4.3+ + when compiling c++ code), this code uses whatever method is needed + or, for VS2008 where neither is available, uses casting workarounds. */ +#ifdef _MSC_VER /* MS compiler */ +#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ +#define LDECLTYPE(x) decltype(x) +#else /* VS2008 or older (or VS2010 in C mode) */ +#define NO_DECLTYPE +#endif +#elif defined(__ICCARM__) +#define NO_DECLTYPE +#else /* GNU, Sun and other compilers */ +#define LDECLTYPE(x) __typeof(x) +#endif + +/* for VS2008 we use some workarounds to get around the lack of decltype, + * namely, we always reassign our tmp variable to the list head if we need + * to dereference its prev/next pointers, and save/restore the real head.*/ +#ifdef NO_DECLTYPE +#define IF_NO_DECLTYPE(x) x +#define LDECLTYPE(x) char* +#define _SV(elt,list) _tmp = (char*)(list); {char **_alias = (char**)&(list); *_alias = (elt); } +#define _NEXT(elt,list,next) ((char*)((list)->next)) +#define _NEXTASGN(elt,list,to,next) { char **_alias = (char**)&((list)->next); *_alias=(char*)(to); } +/* #define _PREV(elt,list,prev) ((char*)((list)->prev)) */ +#define _PREVASGN(elt,list,to,prev) { char **_alias = (char**)&((list)->prev); *_alias=(char*)(to); } +#define _RS(list) { char **_alias = (char**)&(list); *_alias=_tmp; } +#define _CASTASGN(a,b) { char **_alias = (char**)&(a); *_alias=(char*)(b); } +#else +#define IF_NO_DECLTYPE(x) +#define _SV(elt,list) +#define _NEXT(elt,list,next) ((elt)->next) +#define _NEXTASGN(elt,list,to,next) ((elt)->next)=(to) +/* #define _PREV(elt,list,prev) ((elt)->prev) */ +#define _PREVASGN(elt,list,to,prev) ((elt)->prev)=(to) +#define _RS(list) +#define _CASTASGN(a,b) (a)=(b) +#endif + +/****************************************************************************** + * The sort macro is an adaptation of Simon Tatham's O(n log(n)) mergesort * + * Unwieldy variable names used here to avoid shadowing passed-in variables. * + *****************************************************************************/ +#define LL_SORT(list, cmp) \ + LL_SORT2(list, cmp, next) + +#define LL_SORT2(list, cmp, next) \ +do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + _CASTASGN(_ls_p,list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + _SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \ + if (!_ls_q) break; \ + } \ + _ls_qsize = _ls_insize; \ + while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + } else if (_ls_qsize == 0 || !_ls_q) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + } else if (cmp(_ls_p,_ls_q) <= 0) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + } else { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + } \ + if (_ls_tail) { \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \ + } else { \ + _CASTASGN(list,_ls_e); \ + } \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + if (_ls_tail) { \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \ + } \ + if (_ls_nmerges <= 1) { \ + _ls_looping=0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ +} while (0) + + +#define DL_SORT(list, cmp) \ + DL_SORT2(list, cmp, prev, next) + +#define DL_SORT2(list, cmp, prev, next) \ +do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + _CASTASGN(_ls_p,list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + _SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \ + if (!_ls_q) break; \ + } \ + _ls_qsize = _ls_insize; \ + while ((_ls_psize > 0) || ((_ls_qsize > 0) && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + } else if ((_ls_qsize == 0) || (!_ls_q)) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + } else if (cmp(_ls_p,_ls_q) <= 0) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + } else { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + } \ + if (_ls_tail) { \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \ + } else { \ + _CASTASGN(list,_ls_e); \ + } \ + _SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + _CASTASGN((list)->prev, _ls_tail); \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \ + if (_ls_nmerges <= 1) { \ + _ls_looping=0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ +} while (0) + +#define CDL_SORT(list, cmp) \ + CDL_SORT2(list, cmp, prev, next) + +#define CDL_SORT2(list, cmp, prev, next) \ +do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + LDECLTYPE(list) _ls_oldhead; \ + LDECLTYPE(list) _tmp; \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + _CASTASGN(_ls_p,list); \ + _CASTASGN(_ls_oldhead,list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + _SV(_ls_q,list); \ + if (_NEXT(_ls_q,list,next) == _ls_oldhead) { \ + _ls_q = NULL; \ + } else { \ + _ls_q = _NEXT(_ls_q,list,next); \ + } \ + _RS(list); \ + if (!_ls_q) break; \ + } \ + _ls_qsize = _ls_insize; \ + while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \ + } else if (_ls_qsize == 0 || !_ls_q) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \ + } else if (cmp(_ls_p,_ls_q) <= 0) { \ + _ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \ + _NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \ + if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \ + } else { \ + _ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \ + _NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \ + if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \ + } \ + if (_ls_tail) { \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \ + } else { \ + _CASTASGN(list,_ls_e); \ + } \ + _SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + _CASTASGN((list)->prev,_ls_tail); \ + _CASTASGN(_tmp,list); \ + _SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_tmp,next); _RS(list); \ + if (_ls_nmerges <= 1) { \ + _ls_looping=0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ +} while (0) + +/****************************************************************************** + * singly linked list macros (non-circular) * + *****************************************************************************/ +#define LL_PREPEND(head,add) \ + LL_PREPEND2(head,add,next) + +#define LL_PREPEND2(head,add,next) \ +do { \ + (add)->next = (head); \ + (head) = (add); \ +} while (0) + +#define LL_CONCAT(head1,head2) \ + LL_CONCAT2(head1,head2,next) + +#define LL_CONCAT2(head1,head2,next) \ +do { \ + LDECLTYPE(head1) _tmp; \ + if (head1) { \ + _tmp = (head1); \ + while (_tmp->next) { _tmp = _tmp->next; } \ + _tmp->next=(head2); \ + } else { \ + (head1)=(head2); \ + } \ +} while (0) + +#define LL_APPEND(head,add) \ + LL_APPEND2(head,add,next) + +#define LL_APPEND2(head,add,next) \ +do { \ + LDECLTYPE(head) _tmp; \ + (add)->next=NULL; \ + if (head) { \ + _tmp = (head); \ + while (_tmp->next) { _tmp = _tmp->next; } \ + _tmp->next=(add); \ + } else { \ + (head)=(add); \ + } \ +} while (0) + +#define LL_DELETE(head,del) \ + LL_DELETE2(head,del,next) + +#define LL_DELETE2(head,del,next) \ +do { \ + LDECLTYPE(head) _tmp; \ + if ((head) == (del)) { \ + (head)=(head)->next; \ + } else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (del))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (del)->next; \ + } \ + } \ +} while (0) + +#define LL_COUNT(head,el,counter) \ + LL_COUNT2(head,el,counter,next) \ + +#define LL_COUNT2(head,el,counter,next) \ +do { \ + (counter) = 0; \ + LL_FOREACH2(head,el,next) { ++(counter); } \ +} while (0) + +#define LL_FOREACH(head,el) \ + LL_FOREACH2(head,el,next) + +#define LL_FOREACH2(head,el,next) \ + for ((el) = (head); el; (el) = (el)->next) + +#define LL_FOREACH_SAFE(head,el,tmp) \ + LL_FOREACH_SAFE2(head,el,tmp,next) + +#define LL_FOREACH_SAFE2(head,el,tmp,next) \ + for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) + +#define LL_SEARCH_SCALAR(head,out,field,val) \ + LL_SEARCH_SCALAR2(head,out,field,val,next) + +#define LL_SEARCH_SCALAR2(head,out,field,val,next) \ +do { \ + LL_FOREACH2(head,out,next) { \ + if ((out)->field == (val)) break; \ + } \ +} while (0) + +#define LL_SEARCH(head,out,elt,cmp) \ + LL_SEARCH2(head,out,elt,cmp,next) + +#define LL_SEARCH2(head,out,elt,cmp,next) \ +do { \ + LL_FOREACH2(head,out,next) { \ + if ((cmp(out,elt))==0) break; \ + } \ +} while (0) + +#define LL_REPLACE_ELEM2(head, el, add, next) \ +do { \ + LDECLTYPE(head) _tmp; \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (el))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (add); \ + } \ + } \ +} while (0) + +#define LL_REPLACE_ELEM(head, el, add) \ + LL_REPLACE_ELEM2(head, el, add, next) + +#define LL_PREPEND_ELEM2(head, el, add, next) \ +do { \ + if (el) { \ + LDECLTYPE(head) _tmp; \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (el))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (add); \ + } \ + } \ + } else { \ + LL_APPEND2(head, add, next); \ + } \ +} while (0) \ + +#define LL_PREPEND_ELEM(head, el, add) \ + LL_PREPEND_ELEM2(head, el, add, next) + +#define LL_APPEND_ELEM2(head, el, add, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (el)->next = (add); \ + } else { \ + LL_PREPEND2(head, add, next); \ + } \ +} while (0) \ + +#define LL_APPEND_ELEM(head, el, add) \ + LL_APPEND_ELEM2(head, el, add, next) + +#ifdef NO_DECLTYPE +/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ + +#undef LL_CONCAT2 +#define LL_CONCAT2(head1,head2,next) \ +do { \ + char *_tmp; \ + if (head1) { \ + _tmp = (char*)(head1); \ + while ((head1)->next) { (head1) = (head1)->next; } \ + (head1)->next = (head2); \ + _RS(head1); \ + } else { \ + (head1)=(head2); \ + } \ +} while (0) + +#undef LL_APPEND2 +#define LL_APPEND2(head,add,next) \ +do { \ + if (head) { \ + (add)->next = head; /* use add->next as a temp variable */ \ + while ((add)->next->next) { (add)->next = (add)->next->next; } \ + (add)->next->next=(add); \ + } else { \ + (head)=(add); \ + } \ + (add)->next=NULL; \ +} while (0) + +#undef LL_DELETE2 +#define LL_DELETE2(head,del,next) \ +do { \ + if ((head) == (del)) { \ + (head)=(head)->next; \ + } else { \ + char *_tmp = (char*)(head); \ + while ((head)->next && ((head)->next != (del))) { \ + (head) = (head)->next; \ + } \ + if ((head)->next) { \ + (head)->next = ((del)->next); \ + } \ + _RS(head); \ + } \ +} while (0) + +#undef LL_REPLACE_ELEM2 +#define LL_REPLACE_ELEM2(head, el, add, next) \ +do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + (add)->next = head; \ + while ((add)->next->next && ((add)->next->next != (el))) { \ + (add)->next = (add)->next->next; \ + } \ + if ((add)->next->next) { \ + (add)->next->next = (add); \ + } \ + } \ + (add)->next = (el)->next; \ +} while (0) + +#undef LL_PREPEND_ELEM2 +#define LL_PREPEND_ELEM2(head, el, add, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + (add)->next = (head); \ + while ((add)->next->next && ((add)->next->next != (el))) { \ + (add)->next = (add)->next->next; \ + } \ + if ((add)->next->next) { \ + (add)->next->next = (add); \ + } \ + } \ + (add)->next = (el); \ + } else { \ + LL_APPEND2(head, add, next); \ + } \ +} while (0) \ + +#endif /* NO_DECLTYPE */ + +/****************************************************************************** + * doubly linked list macros (non-circular) * + *****************************************************************************/ +#define DL_PREPEND(head,add) \ + DL_PREPEND2(head,add,prev,next) + +#define DL_PREPEND2(head,add,prev,next) \ +do { \ + (add)->next = (head); \ + if (head) { \ + (add)->prev = (head)->prev; \ + (head)->prev = (add); \ + } else { \ + (add)->prev = (add); \ + } \ + (head) = (add); \ +} while (0) + +#define DL_APPEND(head,add) \ + DL_APPEND2(head,add,prev,next) + +#define DL_APPEND2(head,add,prev,next) \ +do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (head)->prev->next = (add); \ + (head)->prev = (add); \ + (add)->next = NULL; \ + } else { \ + (head)=(add); \ + (head)->prev = (head); \ + (head)->next = NULL; \ + } \ +} while (0) + +#define DL_CONCAT(head1,head2) \ + DL_CONCAT2(head1,head2,prev,next) + +#define DL_CONCAT2(head1,head2,prev,next) \ +do { \ + LDECLTYPE(head1) _tmp; \ + if (head2) { \ + if (head1) { \ + _CASTASGN(_tmp, (head2)->prev); \ + (head2)->prev = (head1)->prev; \ + (head1)->prev->next = (head2); \ + _CASTASGN((head1)->prev, _tmp); \ + } else { \ + (head1)=(head2); \ + } \ + } \ +} while (0) + +#define DL_DELETE(head,del) \ + DL_DELETE2(head,del,prev,next) + +#define DL_DELETE2(head,del,prev,next) \ +do { \ + assert((del)->prev != NULL); \ + if ((del)->prev == (del)) { \ + (head)=NULL; \ + } else if ((del)==(head)) { \ + (del)->next->prev = (del)->prev; \ + (head) = (del)->next; \ + } else { \ + (del)->prev->next = (del)->next; \ + if ((del)->next) { \ + (del)->next->prev = (del)->prev; \ + } else { \ + (head)->prev = (del)->prev; \ + } \ + } \ +} while (0) + +#define DL_COUNT(head,el,counter) \ + DL_COUNT2(head,el,counter,next) \ + +#define DL_COUNT2(head,el,counter,next) \ +do { \ + (counter) = 0; \ + DL_FOREACH2(head,el,next) { ++(counter); } \ +} while (0) + +#define DL_FOREACH(head,el) \ + DL_FOREACH2(head,el,next) + +#define DL_FOREACH2(head,el,next) \ + for ((el) = (head); el; (el) = (el)->next) + +/* this version is safe for deleting the elements during iteration */ +#define DL_FOREACH_SAFE(head,el,tmp) \ + DL_FOREACH_SAFE2(head,el,tmp,next) + +#define DL_FOREACH_SAFE2(head,el,tmp,next) \ + for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) + +/* these are identical to their singly-linked list counterparts */ +#define DL_SEARCH_SCALAR LL_SEARCH_SCALAR +#define DL_SEARCH LL_SEARCH +#define DL_SEARCH_SCALAR2 LL_SEARCH_SCALAR2 +#define DL_SEARCH2 LL_SEARCH2 + +#define DL_REPLACE_ELEM2(head, el, add, prev, next) \ +do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + (add)->next = (el)->next; \ + if ((el)->next == NULL) { \ + (add)->prev = (add); \ + } else { \ + (add)->prev = (el)->prev; \ + (add)->next->prev = (add); \ + } \ + } else { \ + (add)->next = (el)->next; \ + (add)->prev = (el)->prev; \ + (add)->prev->next = (add); \ + if ((el)->next == NULL) { \ + (head)->prev = (add); \ + } else { \ + (add)->next->prev = (add); \ + } \ + } \ +} while (0) + +#define DL_REPLACE_ELEM(head, el, add) \ + DL_REPLACE_ELEM2(head, el, add, prev, next) + +#define DL_PREPEND_ELEM2(head, el, add, prev, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + (add)->prev = (el)->prev; \ + (el)->prev = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } else { \ + (add)->prev->next = (add); \ + } \ + } else { \ + DL_APPEND2(head, add, prev, next); \ + } \ +} while (0) \ + +#define DL_PREPEND_ELEM(head, el, add) \ + DL_PREPEND_ELEM2(head, el, add, prev, next) + +#define DL_APPEND_ELEM2(head, el, add, prev, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (add)->prev = (el); \ + (el)->next = (add); \ + if ((add)->next) { \ + (add)->next->prev = (add); \ + } else { \ + (head)->prev = (add); \ + } \ + } else { \ + DL_PREPEND2(head, add, prev, next); \ + } \ +} while (0) \ + +#define DL_APPEND_ELEM(head, el, add) \ + DL_APPEND_ELEM2(head, el, add, prev, next) + +/****************************************************************************** + * circular doubly linked list macros * + *****************************************************************************/ +#define CDL_APPEND(head,add) \ + CDL_APPEND2(head,add,prev,next) + +#define CDL_APPEND2(head,add,prev,next) \ +do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (add)->prev->next = (add); \ + } else { \ + (add)->prev = (add); \ + (add)->next = (add); \ + (head) = (add); \ + } \ +} while (0) + +#define CDL_PREPEND(head,add) \ + CDL_PREPEND2(head,add,prev,next) + +#define CDL_PREPEND2(head,add,prev,next) \ +do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (add)->prev->next = (add); \ + } else { \ + (add)->prev = (add); \ + (add)->next = (add); \ + } \ + (head) = (add); \ +} while (0) + +#define CDL_DELETE(head,del) \ + CDL_DELETE2(head,del,prev,next) + +#define CDL_DELETE2(head,del,prev,next) \ +do { \ + if (((head)==(del)) && ((head)->next == (head))) { \ + (head) = NULL; \ + } else { \ + (del)->next->prev = (del)->prev; \ + (del)->prev->next = (del)->next; \ + if ((del) == (head)) (head)=(del)->next; \ + } \ +} while (0) + +#define CDL_COUNT(head,el,counter) \ + CDL_COUNT2(head,el,counter,next) \ + +#define CDL_COUNT2(head, el, counter,next) \ +do { \ + (counter) = 0; \ + CDL_FOREACH2(head,el,next) { ++(counter); } \ +} while (0) + +#define CDL_FOREACH(head,el) \ + CDL_FOREACH2(head,el,next) + +#define CDL_FOREACH2(head,el,next) \ + for ((el)=(head);el;(el)=(((el)->next==(head)) ? NULL : (el)->next)) + +#define CDL_FOREACH_SAFE(head,el,tmp1,tmp2) \ + CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next) + +#define CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next) \ + for ((el) = (head), (tmp1) = (head) ? (head)->prev : NULL; \ + (el) && ((tmp2) = (el)->next, 1); \ + (el) = ((el) == (tmp1) ? NULL : (tmp2))) + +#define CDL_SEARCH_SCALAR(head,out,field,val) \ + CDL_SEARCH_SCALAR2(head,out,field,val,next) + +#define CDL_SEARCH_SCALAR2(head,out,field,val,next) \ +do { \ + CDL_FOREACH2(head,out,next) { \ + if ((out)->field == (val)) break; \ + } \ +} while (0) + +#define CDL_SEARCH(head,out,elt,cmp) \ + CDL_SEARCH2(head,out,elt,cmp,next) + +#define CDL_SEARCH2(head,out,elt,cmp,next) \ +do { \ + CDL_FOREACH2(head,out,next) { \ + if ((cmp(out,elt))==0) break; \ + } \ +} while (0) + +#define CDL_REPLACE_ELEM2(head, el, add, prev, next) \ +do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((el)->next == (el)) { \ + (add)->next = (add); \ + (add)->prev = (add); \ + (head) = (add); \ + } else { \ + (add)->next = (el)->next; \ + (add)->prev = (el)->prev; \ + (add)->next->prev = (add); \ + (add)->prev->next = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + } \ +} while (0) + +#define CDL_REPLACE_ELEM(head, el, add) \ + CDL_REPLACE_ELEM2(head, el, add, prev, next) + +#define CDL_PREPEND_ELEM2(head, el, add, prev, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + (add)->prev = (el)->prev; \ + (el)->prev = (add); \ + (add)->prev->next = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + } else { \ + CDL_APPEND2(head, add, prev, next); \ + } \ +} while (0) + +#define CDL_PREPEND_ELEM(head, el, add) \ + CDL_PREPEND_ELEM2(head, el, add, prev, next) + +#define CDL_APPEND_ELEM2(head, el, add, prev, next) \ +do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (add)->prev = (el); \ + (el)->next = (add); \ + (add)->next->prev = (add); \ + } else { \ + CDL_PREPEND2(head, add, prev, next); \ + } \ +} while (0) + +#define CDL_APPEND_ELEM(head, el, add) \ + CDL_APPEND_ELEM2(head, el, add, prev, next) + +#endif /* UTLIST_H */ diff --git a/src/common/thirdparty/utstring.h b/src/common/thirdparty/utstring.h new file mode 100644 index 000000000..debe5f3df --- /dev/null +++ b/src/common/thirdparty/utstring.h @@ -0,0 +1,398 @@ +/* +Copyright (c) 2008-2016, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* a dynamic string implementation using macros + */ +#ifndef UTSTRING_H +#define UTSTRING_H + +#define UTSTRING_VERSION 2.0.1 + +#ifdef __GNUC__ +#define _UNUSED_ __attribute__ ((__unused__)) +#else +#define _UNUSED_ +#endif + +#include +#include +#include +#include + +#ifndef oom +#define oom() exit(-1) +#endif + +typedef struct { + char *d; + size_t n; /* allocd size */ + size_t i; /* index of first unused byte */ +} UT_string; + +#define utstring_reserve(s,amt) \ +do { \ + if (((s)->n - (s)->i) < (size_t)(amt)) { \ + char *utstring_tmp = (char*)realloc( \ + (s)->d, (s)->n + (amt)); \ + if (utstring_tmp == NULL) oom(); \ + (s)->d = utstring_tmp; \ + (s)->n += (amt); \ + } \ +} while(0) + +#define utstring_init(s) \ +do { \ + (s)->n = 0; (s)->i = 0; (s)->d = NULL; \ + utstring_reserve(s,100); \ + (s)->d[0] = '\0'; \ +} while(0) + +#define utstring_done(s) \ +do { \ + if ((s)->d != NULL) free((s)->d); \ + (s)->n = 0; \ +} while(0) + +#define utstring_free(s) \ +do { \ + utstring_done(s); \ + free(s); \ +} while(0) + +#define utstring_new(s) \ +do { \ + s = (UT_string*)calloc(sizeof(UT_string),1); \ + if (!s) oom(); \ + utstring_init(s); \ +} while(0) + +#define utstring_renew(s) \ +do { \ + if (s) { \ + utstring_clear(s); \ + } else { \ + utstring_new(s); \ + } \ +} while(0) + +#define utstring_clear(s) \ +do { \ + (s)->i = 0; \ + (s)->d[0] = '\0'; \ +} while(0) + +#define utstring_bincpy(s,b,l) \ +do { \ + utstring_reserve((s),(l)+1); \ + if (l) memcpy(&(s)->d[(s)->i], b, l); \ + (s)->i += (l); \ + (s)->d[(s)->i]='\0'; \ +} while(0) + +#define utstring_concat(dst,src) \ +do { \ + utstring_reserve((dst),((src)->i)+1); \ + if ((src)->i) memcpy(&(dst)->d[(dst)->i], (src)->d, (src)->i); \ + (dst)->i += (src)->i; \ + (dst)->d[(dst)->i]='\0'; \ +} while(0) + +#define utstring_len(s) ((unsigned)((s)->i)) + +#define utstring_body(s) ((s)->d) + +_UNUSED_ static void utstring_printf_va(UT_string *s, const char *fmt, va_list ap) { + int n; + va_list cp; + for (;;) { +#ifdef _WIN32 + cp = ap; +#else + va_copy(cp, ap); +#endif + n = vsnprintf (&s->d[s->i], s->n-s->i, fmt, cp); + va_end(cp); + + if ((n > -1) && ((size_t) n < (s->n-s->i))) { + s->i += n; + return; + } + + /* Else try again with more space. */ + if (n > -1) utstring_reserve(s,n+1); /* exact */ + else utstring_reserve(s,(s->n)*2); /* 2x */ + } +} +#ifdef __GNUC__ +/* support printf format checking (2=the format string, 3=start of varargs) */ +static void utstring_printf(UT_string *s, const char *fmt, ...) + __attribute__ (( format( printf, 2, 3) )); +#endif +_UNUSED_ static void utstring_printf(UT_string *s, const char *fmt, ...) { + va_list ap; + va_start(ap,fmt); + utstring_printf_va(s,fmt,ap); + va_end(ap); +} + +/******************************************************************************* + * begin substring search functions * + ******************************************************************************/ +/* Build KMP table from left to right. */ +_UNUSED_ static void _utstring_BuildTable( + const char *P_Needle, + size_t P_NeedleLen, + long *P_KMP_Table) +{ + long i, j; + + i = 0; + j = i - 1; + P_KMP_Table[i] = j; + while (i < (long) P_NeedleLen) + { + while ( (j > -1) && (P_Needle[i] != P_Needle[j]) ) + { + j = P_KMP_Table[j]; + } + i++; + j++; + if (i < (long) P_NeedleLen) + { + if (P_Needle[i] == P_Needle[j]) + { + P_KMP_Table[i] = P_KMP_Table[j]; + } + else + { + P_KMP_Table[i] = j; + } + } + else + { + P_KMP_Table[i] = j; + } + } + + return; +} + + +/* Build KMP table from right to left. */ +_UNUSED_ static void _utstring_BuildTableR( + const char *P_Needle, + size_t P_NeedleLen, + long *P_KMP_Table) +{ + long i, j; + + i = P_NeedleLen - 1; + j = i + 1; + P_KMP_Table[i + 1] = j; + while (i >= 0) + { + while ( (j < (long) P_NeedleLen) && (P_Needle[i] != P_Needle[j]) ) + { + j = P_KMP_Table[j + 1]; + } + i--; + j--; + if (i >= 0) + { + if (P_Needle[i] == P_Needle[j]) + { + P_KMP_Table[i + 1] = P_KMP_Table[j + 1]; + } + else + { + P_KMP_Table[i + 1] = j; + } + } + else + { + P_KMP_Table[i + 1] = j; + } + } + + return; +} + + +/* Search data from left to right. ( Multiple search mode. ) */ +_UNUSED_ static long _utstring_find( + const char *P_Haystack, + size_t P_HaystackLen, + const char *P_Needle, + size_t P_NeedleLen, + long *P_KMP_Table) +{ + long i, j; + long V_FindPosition = -1; + + /* Search from left to right. */ + i = j = 0; + while ( (j < (int)P_HaystackLen) && (((P_HaystackLen - j) + i) >= P_NeedleLen) ) + { + while ( (i > -1) && (P_Needle[i] != P_Haystack[j]) ) + { + i = P_KMP_Table[i]; + } + i++; + j++; + if (i >= (int)P_NeedleLen) + { + /* Found. */ + V_FindPosition = j - i; + break; + } + } + + return V_FindPosition; +} + + +/* Search data from right to left. ( Multiple search mode. ) */ +_UNUSED_ static long _utstring_findR( + const char *P_Haystack, + size_t P_HaystackLen, + const char *P_Needle, + size_t P_NeedleLen, + long *P_KMP_Table) +{ + long i, j; + long V_FindPosition = -1; + + /* Search from right to left. */ + j = (P_HaystackLen - 1); + i = (P_NeedleLen - 1); + while ( (j >= 0) && (j >= i) ) + { + while ( (i < (int)P_NeedleLen) && (P_Needle[i] != P_Haystack[j]) ) + { + i = P_KMP_Table[i + 1]; + } + i--; + j--; + if (i < 0) + { + /* Found. */ + V_FindPosition = j + 1; + break; + } + } + + return V_FindPosition; +} + + +/* Search data from left to right. ( One time search mode. ) */ +_UNUSED_ static long utstring_find( + UT_string *s, + long P_StartPosition, /* Start from 0. -1 means last position. */ + const char *P_Needle, + size_t P_NeedleLen) +{ + long V_StartPosition; + long V_HaystackLen; + long *V_KMP_Table; + long V_FindPosition = -1; + + if (P_StartPosition < 0) + { + V_StartPosition = s->i + P_StartPosition; + } + else + { + V_StartPosition = P_StartPosition; + } + V_HaystackLen = s->i - V_StartPosition; + if ( (V_HaystackLen >= (long) P_NeedleLen) && (P_NeedleLen > 0) ) + { + V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1)); + if (V_KMP_Table != NULL) + { + _utstring_BuildTable(P_Needle, P_NeedleLen, V_KMP_Table); + + V_FindPosition = _utstring_find(s->d + V_StartPosition, + V_HaystackLen, + P_Needle, + P_NeedleLen, + V_KMP_Table); + if (V_FindPosition >= 0) + { + V_FindPosition += V_StartPosition; + } + + free(V_KMP_Table); + } + } + + return V_FindPosition; +} + + +/* Search data from right to left. ( One time search mode. ) */ +_UNUSED_ static long utstring_findR( + UT_string *s, + long P_StartPosition, /* Start from 0. -1 means last position. */ + const char *P_Needle, + size_t P_NeedleLen) +{ + long V_StartPosition; + long V_HaystackLen; + long *V_KMP_Table; + long V_FindPosition = -1; + + if (P_StartPosition < 0) + { + V_StartPosition = s->i + P_StartPosition; + } + else + { + V_StartPosition = P_StartPosition; + } + V_HaystackLen = V_StartPosition + 1; + if ( (V_HaystackLen >= (long) P_NeedleLen) && (P_NeedleLen > 0) ) + { + V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1)); + if (V_KMP_Table != NULL) + { + _utstring_BuildTableR(P_Needle, P_NeedleLen, V_KMP_Table); + + V_FindPosition = _utstring_findR(s->d, + V_HaystackLen, + P_Needle, + P_NeedleLen, + V_KMP_Table); + + free(V_KMP_Table); + } + } + + return V_FindPosition; +} +/******************************************************************************* + * end substring search functions * + ******************************************************************************/ + +#endif /* UTSTRING_H */