mirror of
https://github.com/wassname/ray.git
synced 2026-06-28 17:34:51 +08:00
Merge remote-tracking branch 'r1remote/moveout' into switch
This commit is contained in:
Robert Nishihara
@@ -0,0 +1,46 @@
|
||||
CC = gcc
|
||||
CFLAGS = -g -Wall --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -fPIC -I. -Ithirdparty -Ithirdparty/ae -Wno-typedef-redefinition -Werror
|
||||
BUILD = build
|
||||
|
||||
all: hiredis $(BUILD)/libcommon.a
|
||||
|
||||
$(BUILD)/libcommon.a: event_loop.o common.o task.o io.o state/redis.o thirdparty/ae/ae.o
|
||||
ar rcs $@ $^
|
||||
|
||||
$(BUILD)/common_tests: test/common_tests.c $(BUILD)/libcommon.a
|
||||
$(CC) -o $@ test/common_tests.c $(BUILD)/libcommon.a $(CFLAGS)
|
||||
|
||||
$(BUILD)/db_tests: hiredis test/db_tests.c $(BUILD)/libcommon.a
|
||||
$(CC) -o $@ test/db_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS)
|
||||
|
||||
$(BUILD)/io_tests: test/io_tests.c $(BUILD)/libcommon.a
|
||||
$(CC) -o $@ $^ $(CFLAGS)
|
||||
|
||||
$(BUILD)/task_tests: test/task_tests.c $(BUILD)/libcommon.a
|
||||
$(CC) -o $@ $^ $(CFLAGS)
|
||||
|
||||
$(BUILD)/redis_tests: hiredis test/redis_tests.c $(BUILD)/libcommon.a logging.h
|
||||
$(CC) -o $@ test/redis_tests.c logging.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS)
|
||||
|
||||
clean:
|
||||
rm -f *.o state/*.o test/*.o thirdparty/ae/*.o
|
||||
rm -rf $(BUILD)/*
|
||||
|
||||
redis:
|
||||
cd thirdparty ; bash ./build-redis.sh
|
||||
|
||||
hiredis:
|
||||
git submodule update --init --recursive -- "thirdparty/hiredis" ; cd thirdparty/hiredis ; make
|
||||
|
||||
test: hiredis redis $(BUILD)/common_tests $(BUILD)/db_tests $(BUILD)/io_tests $(BUILD)/task_tests $(BUILD)/redis_tests FORCE
|
||||
./thirdparty/redis-3.2.3/src/redis-server &
|
||||
sleep 1s ; ./build/common_tests ; ./build/db_tests ; ./build/io_tests ; ./build/task_tests ; ./build/redis_tests
|
||||
|
||||
valgrind: test
|
||||
valgrind --leak-check=full --error-exitcode=1 ./build/common_tests
|
||||
valgrind --leak-check=full --error-exitcode=1 ./build/db_tests
|
||||
valgrind --leak-check=full --error-exitcode=1 ./build/io_tests
|
||||
valgrind --leak-check=full --error-exitcode=1 ./build/task_tests
|
||||
valgrind --leak-check=full --error-exitcode=1 ./build/redis_tests
|
||||
|
||||
FORCE:
|
||||
@@ -0,0 +1,36 @@
|
||||
#include "common.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
const unique_id NIL_ID = {{255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255, 255, 255}};
|
||||
|
||||
unique_id globally_unique_id(void) {
|
||||
/* Use /dev/urandom for "real" randomness. */
|
||||
int fd;
|
||||
if ((fd = open("/dev/urandom", O_RDONLY)) == -1) {
|
||||
LOG_ERR("Could not generate random number");
|
||||
}
|
||||
unique_id result;
|
||||
read(fd, &result.id[0], UNIQUE_ID_SIZE);
|
||||
close(fd);
|
||||
return result;
|
||||
}
|
||||
|
||||
char *sha1_to_hex(const unsigned char *sha1, char *buffer) {
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
char *buf = buffer;
|
||||
|
||||
for (int i = 0; i < UNIQUE_ID_SIZE; i++) {
|
||||
unsigned int val = *sha1++;
|
||||
*buf++ = hex[val >> 4];
|
||||
*buf++ = hex[val & 0xf];
|
||||
}
|
||||
*buf = '\0';
|
||||
|
||||
return buffer;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
#ifndef COMMON_H
|
||||
#define COMMON_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
|
||||
#ifndef RAY_COMMON_DEBUG
|
||||
#define LOG_DEBUG(M, ...)
|
||||
#else
|
||||
#define LOG_DEBUG(M, ...) \
|
||||
fprintf(stderr, "[DEBUG] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__)
|
||||
#endif
|
||||
|
||||
#define LOG_ERR(M, ...) \
|
||||
fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, \
|
||||
errno == 0 ? "None" : strerror(errno), ##__VA_ARGS__)
|
||||
|
||||
#define LOG_INFO(M, ...) \
|
||||
fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__)
|
||||
|
||||
#define CHECK(COND) \
|
||||
do { \
|
||||
if (!(COND)) { \
|
||||
LOG_ERR("Check failure: %s", #COND); \
|
||||
exit(-1); \
|
||||
} \
|
||||
} while (0);
|
||||
|
||||
#define CHECKM(COND, M, ...) \
|
||||
do { \
|
||||
if (!(COND)) { \
|
||||
LOG_ERR("Check failure: %s \n" M, #COND, ##__VA_ARGS__); \
|
||||
exit(-1); \
|
||||
} \
|
||||
} while (0);
|
||||
|
||||
#define UNIQUE_ID_SIZE 20
|
||||
|
||||
/* Cleanup method for running tests with the greatest library.
|
||||
* Runs the test, then clears the Redis database. */
|
||||
#define RUN_REDIS_TEST(context, test) \
|
||||
RUN_TEST(test); \
|
||||
freeReplyObject(redisCommand(context, "FLUSHALL"));
|
||||
|
||||
typedef struct { unsigned char id[UNIQUE_ID_SIZE]; } unique_id;
|
||||
|
||||
extern const unique_id NIL_ID;
|
||||
|
||||
/* Generate a globally unique ID. */
|
||||
unique_id globally_unique_id(void);
|
||||
|
||||
/* Convert a 20 byte sha1 hash to a hexdecimal string. This function assumes
|
||||
* that buffer points to an already allocated char array of size 2 *
|
||||
* UNIQUE_ID_SIZE + 1 */
|
||||
char *sha1_to_hex(const unsigned char *sha1, char *buffer);
|
||||
|
||||
typedef unique_id object_id;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,33 @@
|
||||
# Task specifications, task instances and task logs
|
||||
|
||||
A *task specification* contains all information that is needed for computing
|
||||
the results of a task:
|
||||
|
||||
- The function ID of the function that executes the task
|
||||
- The arguments (either object IDs for pass by reference
|
||||
or values for pass by value)
|
||||
- The IDs of the result objects
|
||||
|
||||
From these, a task ID can be computed which is also stored in the task
|
||||
specification.
|
||||
|
||||
A *task instance* represents one execution of a task specification.
|
||||
It consists of:
|
||||
|
||||
- A scheduling state (WAITING, SCHEDULED, RUNNING, DONE)
|
||||
- The target node where the task is scheduled or executed
|
||||
- A unique task instance ID that identifies the particular execution
|
||||
of the task.
|
||||
|
||||
The task data structures are defined in `common/task.h`.
|
||||
|
||||
The *task log* is a mapping from the task instance ID to a sequence of
|
||||
updates to the status of the task instance. It is updated by various parts
|
||||
of the system:
|
||||
|
||||
1. The local scheduler writes it with status WAITING when submits a task to the global scheduler
|
||||
2. The global scheduler appends an update WAITING -> SCHEDULED together with the node ID when assigning the task to a local scheduler
|
||||
3. The local scheduler appends an update SCHEDULED -> RUNNING when it assigns a task to a worker
|
||||
4. The local scheduler appends an update RUNNING -> DONE when the task finishes execution
|
||||
|
||||
The task log is defined in `common/state/task_log.h`.
|
||||
@@ -0,0 +1,62 @@
|
||||
#include "event_loop.h"
|
||||
|
||||
#include "common.h"
|
||||
#include <errno.h>
|
||||
|
||||
#define INITIAL_EVENT_LOOP_SIZE 1024
|
||||
|
||||
event_loop *event_loop_create() {
|
||||
return aeCreateEventLoop(INITIAL_EVENT_LOOP_SIZE);
|
||||
}
|
||||
|
||||
void event_loop_destroy(event_loop *loop) {
|
||||
/* Clean up timer events. This is to make valgrind happy. */
|
||||
aeTimeEvent *te = loop->timeEventHead;
|
||||
while (te) {
|
||||
aeTimeEvent *next = te->next;
|
||||
free(te);
|
||||
te = next;
|
||||
}
|
||||
aeDeleteEventLoop(loop);
|
||||
}
|
||||
|
||||
void event_loop_add_file(event_loop *loop,
|
||||
int fd,
|
||||
int events,
|
||||
event_loop_file_handler handler,
|
||||
void *context) {
|
||||
/* Try to add the file descriptor. */
|
||||
int err = aeCreateFileEvent(loop, fd, events, handler, context);
|
||||
/* If it cannot be added, increase the size of the event loop. */
|
||||
if (err == AE_ERR && errno == ERANGE) {
|
||||
err = aeResizeSetSize(loop, 3 * aeGetSetSize(loop) / 2);
|
||||
CHECK(err == AE_OK);
|
||||
err = aeCreateFileEvent(loop, fd, events, handler, context);
|
||||
}
|
||||
/* In any case, test if there were errors. */
|
||||
CHECK(err == AE_OK);
|
||||
}
|
||||
|
||||
void event_loop_remove_file(event_loop *loop, int fd) {
|
||||
aeDeleteFileEvent(loop, fd, EVENT_LOOP_READ | EVENT_LOOP_WRITE);
|
||||
}
|
||||
|
||||
int64_t event_loop_add_timer(event_loop *loop,
|
||||
int64_t milliseconds,
|
||||
event_loop_timer_handler handler,
|
||||
void *context) {
|
||||
return aeCreateTimeEvent(loop, milliseconds, handler, context, NULL);
|
||||
}
|
||||
|
||||
void event_loop_remove_timer(event_loop *loop, timer_id timer_id) {
|
||||
int err = aeDeleteTimeEvent(loop, timer_id);
|
||||
CHECK(err == AE_OK); /* timer id found? */
|
||||
}
|
||||
|
||||
void event_loop_run(event_loop *loop) {
|
||||
aeMain(loop);
|
||||
}
|
||||
|
||||
void event_loop_stop(event_loop *loop) {
|
||||
aeStop(loop);
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
#ifndef EVENT_LOOP_H
|
||||
#define EVENT_LOOP_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include "ae/ae.h"
|
||||
|
||||
typedef long long timer_id;
|
||||
|
||||
typedef aeEventLoop event_loop;
|
||||
|
||||
/* File descriptor is readable. */
|
||||
#define EVENT_LOOP_READ AE_READABLE
|
||||
|
||||
/* File descriptor is writable. */
|
||||
#define EVENT_LOOP_WRITE AE_WRITABLE
|
||||
|
||||
/* Constant specifying that the timer is done and it will be removed. */
|
||||
#define EVENT_LOOP_TIMER_DONE AE_NOMORE
|
||||
|
||||
/* Signature of the handler that will be called when there is a new event
|
||||
* on the file descriptor that this handler has been registered for. The
|
||||
* context is the one that was passed into add_file by the user. The
|
||||
* events parameter indicates which event is available on the file,
|
||||
* it can be EVENT_LOOP_READ or EVENT_LOOP_WRITE. */
|
||||
typedef void (*event_loop_file_handler)(event_loop *loop,
|
||||
int fd,
|
||||
void *context,
|
||||
int events);
|
||||
|
||||
/* This handler will be called when a timer times out. The id of the timer
|
||||
* as well as the context that was specified when registering this handler
|
||||
* are passed as arguments. The return is the number of milliseconds the
|
||||
* timer shall be reset to or EVENT_LOOP_TIMER_DONE if the timer shall
|
||||
* not be triggered again. */
|
||||
typedef int (*event_loop_timer_handler)(event_loop *loop,
|
||||
timer_id timer_id,
|
||||
void *context);
|
||||
|
||||
/* Create and return a new event loop. */
|
||||
event_loop *event_loop_create();
|
||||
|
||||
/* Deallocate space associated with the event loop that was created
|
||||
* with the "create" function. */
|
||||
void event_loop_destroy(event_loop *loop);
|
||||
|
||||
/* Register a handler that will be called any time a new event happens on
|
||||
* a file descriptor. Can specify a context that will be passed as an
|
||||
* argument to the handler. Currently there can only be one handler per file.
|
||||
* The events parameter specifies which events we listen to: EVENT_LOOP_READ
|
||||
* or EVENT_LOOP_WRITE. */
|
||||
void event_loop_add_file(event_loop *loop,
|
||||
int fd,
|
||||
int events,
|
||||
event_loop_file_handler handler,
|
||||
void *context);
|
||||
|
||||
/* Remove a registered file event handler from the event loop. */
|
||||
void event_loop_remove_file(event_loop *loop, int fd);
|
||||
|
||||
/* Register a handler that will be called after a time slice of
|
||||
* "milliseconds" milliseconds. Can specify a context that will be passed
|
||||
* as an argument to the handler. Return the id of the time event. */
|
||||
int64_t event_loop_add_timer(event_loop *loop,
|
||||
int64_t milliseconds,
|
||||
event_loop_timer_handler handler,
|
||||
void *context);
|
||||
|
||||
/* Remove a registered time event handler from the event loop. */
|
||||
void event_loop_remove_timer(event_loop *loop, timer_id timer_id);
|
||||
|
||||
/* Run the event loop. */
|
||||
void event_loop_run(event_loop *loop);
|
||||
|
||||
/* Stop the event loop. */
|
||||
void event_loop_stop(event_loop *loop);
|
||||
|
||||
#endif
|
||||
+333
@@ -0,0 +1,333 @@
|
||||
#include "io.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <inttypes.h>
|
||||
#include <stdarg.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <netinet/in.h>
|
||||
#include <utstring.h>
|
||||
|
||||
#include "common.h"
|
||||
|
||||
/**
|
||||
* Binds to an Internet socket at the given port. Removes any existing file at
|
||||
* the pathname. Returns a non-blocking file descriptor for the socket, or -1
|
||||
* if an error occurred.
|
||||
*
|
||||
* @note Since the returned file descriptor is non-blocking, it is not
|
||||
* recommended to use the Linux read and write calls directly, since these
|
||||
* might read or write a partial message. Instead, use the provided
|
||||
* write_message and read_message methods.
|
||||
*
|
||||
* @param port The port to bind to.
|
||||
* @return A non-blocking file descriptor for the socket, or -1 if an error
|
||||
* occurs.
|
||||
*/
|
||||
int bind_inet_sock(const int port) {
|
||||
struct sockaddr_in name;
|
||||
int socket_fd = socket(PF_INET, SOCK_STREAM, 0);
|
||||
if (socket_fd < 0) {
|
||||
LOG_ERR("socket() failed for port %d.", port);
|
||||
return -1;
|
||||
}
|
||||
name.sin_family = AF_INET;
|
||||
name.sin_port = htons(port);
|
||||
name.sin_addr.s_addr = htonl(INADDR_ANY);
|
||||
int on = 1;
|
||||
/* TODO(pcm): http://stackoverflow.com/q/1150635 */
|
||||
if (ioctl(socket_fd, FIONBIO, (char *) &on) < 0) {
|
||||
LOG_ERR("ioctl failed");
|
||||
close(socket_fd);
|
||||
return -1;
|
||||
}
|
||||
if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0) {
|
||||
LOG_ERR("setsockopt failed for port %d", port);
|
||||
close(socket_fd);
|
||||
return -1;
|
||||
}
|
||||
if (bind(socket_fd, (struct sockaddr *) &name, sizeof(name)) < 0) {
|
||||
LOG_ERR("Bind failed for port %d", port);
|
||||
close(socket_fd);
|
||||
return -1;
|
||||
}
|
||||
if (listen(socket_fd, 5) == -1) {
|
||||
LOG_ERR("Could not listen to socket %d", port);
|
||||
close(socket_fd);
|
||||
return -1;
|
||||
}
|
||||
return socket_fd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Binds to a Unix domain streaming socket at the given
|
||||
* pathname. Removes any existing file at the pathname.
|
||||
*
|
||||
* @param socket_pathname The pathname for the socket.
|
||||
* @return A blocking file descriptor for the socket, or -1 if an error
|
||||
* occurs.
|
||||
*/
|
||||
int bind_ipc_sock(const char *socket_pathname) {
|
||||
struct sockaddr_un socket_address;
|
||||
int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
|
||||
if (socket_fd < 0) {
|
||||
LOG_ERR("socket() failed for pathname %s.", socket_pathname);
|
||||
return -1;
|
||||
}
|
||||
/* Tell the system to allow the port to be reused. */
|
||||
int on = 1;
|
||||
if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on,
|
||||
sizeof(on)) < 0) {
|
||||
LOG_ERR("setsockopt failed for pathname %s", socket_pathname);
|
||||
close(socket_fd);
|
||||
return -1;
|
||||
}
|
||||
|
||||
unlink(socket_pathname);
|
||||
memset(&socket_address, 0, sizeof(struct sockaddr_un));
|
||||
socket_address.sun_family = AF_UNIX;
|
||||
if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) {
|
||||
LOG_ERR("Socket pathname is too long.");
|
||||
close(socket_fd);
|
||||
return -1;
|
||||
}
|
||||
strncpy(socket_address.sun_path, socket_pathname,
|
||||
strlen(socket_pathname) + 1);
|
||||
|
||||
if (bind(socket_fd, (struct sockaddr *) &socket_address,
|
||||
sizeof(struct sockaddr_un)) != 0) {
|
||||
LOG_ERR("Bind failed for pathname %s.", socket_pathname);
|
||||
close(socket_fd);
|
||||
return -1;
|
||||
}
|
||||
if (listen(socket_fd, 5) == -1) {
|
||||
LOG_ERR("Could not listen to socket %s", socket_pathname);
|
||||
close(socket_fd);
|
||||
return -1;
|
||||
}
|
||||
return socket_fd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connects to a Unix domain streaming socket at the given
|
||||
* pathname. Returns a file descriptor for the socket, or -1 if
|
||||
* an error occurred.
|
||||
*/
|
||||
int connect_ipc_sock(const char *socket_pathname) {
|
||||
struct sockaddr_un socket_address;
|
||||
int socket_fd;
|
||||
|
||||
socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
|
||||
if (socket_fd < 0) {
|
||||
LOG_ERR("socket() failed for pathname %s.", socket_pathname);
|
||||
return -1;
|
||||
}
|
||||
|
||||
memset(&socket_address, 0, sizeof(struct sockaddr_un));
|
||||
socket_address.sun_family = AF_UNIX;
|
||||
if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) {
|
||||
LOG_ERR("Socket pathname is too long.");
|
||||
return -1;
|
||||
}
|
||||
strncpy(socket_address.sun_path, socket_pathname,
|
||||
strlen(socket_pathname) + 1);
|
||||
|
||||
if (connect(socket_fd, (struct sockaddr *) &socket_address,
|
||||
sizeof(struct sockaddr_un)) != 0) {
|
||||
LOG_ERR("Connection to socket failed for pathname %s.", socket_pathname);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return socket_fd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Accept a new client connection on the given socket
|
||||
* descriptor. Returns a descriptor for the new socket.
|
||||
*/
|
||||
int accept_client(int socket_fd) {
|
||||
int client_fd = accept(socket_fd, NULL, NULL);
|
||||
if (client_fd < 0) {
|
||||
LOG_ERR("Error reading from socket.");
|
||||
return -1;
|
||||
}
|
||||
return client_fd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a sequence of bytes into a file descriptor. This will block until one
|
||||
* of the following happens: (1) there is an error (2) end of file, or (3) all
|
||||
* length bytes have been written.
|
||||
*
|
||||
* @param fd The file descriptor to write to. It can be non-blocking.
|
||||
* @param cursor The cursor pointing to the beginning of the bytes to send.
|
||||
* @param length The size of the bytes sequence to write.
|
||||
* @return int Whether there was an error while writing. 0 corresponds to
|
||||
* success and -1 corresponds to an error (errno will be set).
|
||||
*/
|
||||
int write_bytes(int fd, uint8_t *cursor, size_t length) {
|
||||
ssize_t nbytes = 0;
|
||||
while (length > 0) {
|
||||
/* While we haven't written the whole message, write to the file
|
||||
* descriptor, advance the cursor, and decrease the amount left to write. */
|
||||
nbytes = write(fd, cursor, length);
|
||||
if (nbytes < 0) {
|
||||
if (errno == EAGAIN || errno == EWOULDBLOCK) {
|
||||
continue;
|
||||
}
|
||||
/* TODO(swang): Return the error instead of exiting. */
|
||||
/* Force an exit if there was any other type of error. */
|
||||
CHECK(nbytes < 0);
|
||||
}
|
||||
if (nbytes == 0) {
|
||||
return -1;
|
||||
}
|
||||
cursor += nbytes;
|
||||
length -= nbytes;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a sequence of bytes on a file descriptor. The bytes should then be read
|
||||
* by read_message.
|
||||
*
|
||||
* @param fd The file descriptor to write to. It can be non-blocking.
|
||||
* @param type The type of the message to send.
|
||||
* @param length The size in bytes of the bytes parameter.
|
||||
* @param bytes The address of the message to send.
|
||||
* @return int Whether there was an error while writing. 0 corresponds to
|
||||
* success and -1 corresponds to an error (errno will be set).
|
||||
*/
|
||||
int write_message(int fd, int64_t type, int64_t length, uint8_t *bytes) {
|
||||
int closed;
|
||||
closed = write_bytes(fd, (uint8_t *) &type, sizeof(type));
|
||||
if (closed) {
|
||||
return closed;
|
||||
}
|
||||
closed = write_bytes(fd, (uint8_t *) &length, sizeof(length));
|
||||
if (closed) {
|
||||
return closed;
|
||||
}
|
||||
closed = write_bytes(fd, bytes, length * sizeof(char));
|
||||
if (closed) {
|
||||
return closed;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a sequence of bytes from a file descriptor into a buffer. This will
|
||||
* block until one of the following happens: (1) there is an error (2) end of
|
||||
* file, or (3) all length bytes have been written.
|
||||
*
|
||||
* @note The buffer pointed to by cursor must already have length number of
|
||||
* bytes allocated before calling this method.
|
||||
*
|
||||
* @param fd The file descriptor to read from. It can be non-blocking.
|
||||
* @param cursor The cursor pointing to the beginning of the buffer.
|
||||
* @param length The size of the byte sequence to read.
|
||||
* @return int Whether there was an error while writing. 0 corresponds to
|
||||
* success and -1 corresponds to an error (errno will be set).
|
||||
*/
|
||||
int read_bytes(int fd, uint8_t *cursor, size_t length) {
|
||||
ssize_t nbytes = 0;
|
||||
while (length > 0) {
|
||||
/* While we haven't read the whole message, read from the file descriptor,
|
||||
* advance the cursor, and decrease the amount left to read. */
|
||||
nbytes = read(fd, cursor, length);
|
||||
if (nbytes < 0) {
|
||||
if (errno == EAGAIN || errno == EWOULDBLOCK) {
|
||||
continue;
|
||||
}
|
||||
/* Force an exit if there was any other type of error. */
|
||||
CHECK(nbytes < 0);
|
||||
}
|
||||
if (nbytes == 0) {
|
||||
return -1;
|
||||
}
|
||||
cursor += nbytes;
|
||||
length -= nbytes;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a sequence of bytes written by write_message from a file descriptor.
|
||||
* This allocates space for the message.
|
||||
*
|
||||
* @note The caller must free the memory.
|
||||
*
|
||||
* @param fd The file descriptor to read from. It can be non-blocking.
|
||||
* @param type The type of the message that is read will be written at this
|
||||
address. If there was an error while reading, this will be
|
||||
DISCONNECT_CLIENT.
|
||||
* @param length The size in bytes of the message that is read will be written
|
||||
at this address. This size does not include the bytes used to encode
|
||||
the type and length. If there was an error while reading, this will
|
||||
be 0.
|
||||
* @param bytes The address at which to write the pointer to the bytes that are
|
||||
read and allocated by this function. If there was an error while
|
||||
reading, this will be NULL.
|
||||
|
||||
* @return Void.
|
||||
*/
|
||||
void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes) {
|
||||
int closed = read_bytes(fd, (uint8_t *) type, sizeof(int64_t));
|
||||
if (closed) {
|
||||
goto disconnected;
|
||||
}
|
||||
closed = read_bytes(fd, (uint8_t *) length, sizeof(int64_t));
|
||||
if (closed) {
|
||||
goto disconnected;
|
||||
}
|
||||
*bytes = malloc(*length * sizeof(uint8_t));
|
||||
closed = read_bytes(fd, *bytes, *length);
|
||||
if (closed) {
|
||||
free(*bytes);
|
||||
goto disconnected;
|
||||
}
|
||||
return;
|
||||
|
||||
disconnected:
|
||||
/* Handle the case in which the socket is closed. */
|
||||
*type = DISCONNECT_CLIENT;
|
||||
*length = 0;
|
||||
*bytes = NULL;
|
||||
return;
|
||||
}
|
||||
|
||||
/* Write a null-terminated string to a file descriptor. */
|
||||
void write_log_message(int fd, char *message) {
|
||||
/* Account for the \0 at the end of the string. */
|
||||
write_message(fd, LOG_MESSAGE, strlen(message) + 1, (uint8_t *) message);
|
||||
}
|
||||
|
||||
/* Reads a null-terminated string from the file descriptor that has been
|
||||
* written by write_log_message. Allocates and returns a pointer to the string.
|
||||
* NOTE: Caller must free the memory! */
|
||||
char *read_log_message(int fd) {
|
||||
uint8_t *bytes;
|
||||
int64_t type;
|
||||
int64_t length;
|
||||
read_message(fd, &type, &length, &bytes);
|
||||
CHECK(type == LOG_MESSAGE);
|
||||
return (char *) bytes;
|
||||
}
|
||||
|
||||
void write_formatted_log_message(int socket_fd, const char *format, ...) {
|
||||
UT_string *cmd;
|
||||
va_list ap;
|
||||
|
||||
utstring_new(cmd);
|
||||
va_start(ap, format);
|
||||
utstring_printf_va(cmd, format, ap);
|
||||
va_end(ap);
|
||||
|
||||
write_log_message(socket_fd, utstring_body(cmd));
|
||||
utstring_free(cmd);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
#ifndef IO_H
|
||||
#define IO_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
enum common_message_type {
|
||||
/** Disconnect a client. */
|
||||
DISCONNECT_CLIENT,
|
||||
/** Log a message from a client. */
|
||||
LOG_MESSAGE,
|
||||
/** Submit a task to the local scheduler. */
|
||||
SUBMIT_TASK,
|
||||
};
|
||||
|
||||
/* Helper functions for socket communication. */
|
||||
|
||||
int bind_inet_sock(const int port);
|
||||
int bind_ipc_sock(const char *socket_pathname);
|
||||
int connect_ipc_sock(const char *socket_pathname);
|
||||
|
||||
int accept_client(int socket_fd);
|
||||
|
||||
/* Reading and writing data */
|
||||
|
||||
int write_message(int fd, int64_t type, int64_t length, uint8_t *bytes);
|
||||
void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes);
|
||||
|
||||
void write_log_message(int fd, char *message);
|
||||
void write_formatted_log_message(int fd, const char *format, ...);
|
||||
char *read_log_message(int fd);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,344 @@
|
||||
#include <Python.h>
|
||||
#include "node.h"
|
||||
|
||||
#include "common_extension.h"
|
||||
#include "task.h"
|
||||
#include "utarray.h"
|
||||
#include "utstring.h"
|
||||
|
||||
PyObject *CommonError;
|
||||
|
||||
#define MARSHAL_VERSION 2
|
||||
|
||||
/* Define the PyObjectID class. */
|
||||
|
||||
int PyObjectToUniqueID(PyObject *object, object_id *objectid) {
|
||||
if (PyObject_IsInstance(object, (PyObject *) &PyObjectIDType)) {
|
||||
*objectid = ((PyObjectID *) object)->object_id;
|
||||
return 1;
|
||||
} else {
|
||||
PyErr_SetString(PyExc_TypeError, "must be an ObjectID");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int PyObjectID_init(PyObjectID *self, PyObject *args, PyObject *kwds) {
|
||||
const char *data;
|
||||
int size;
|
||||
if (!PyArg_ParseTuple(args, "s#", &data, &size)) {
|
||||
return -1;
|
||||
}
|
||||
if (size != UNIQUE_ID_SIZE) {
|
||||
PyErr_SetString(CommonError,
|
||||
"ObjectID: object id string needs to have length 20");
|
||||
return -1;
|
||||
}
|
||||
memcpy(&self->object_id.id[0], data, UNIQUE_ID_SIZE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Create a PyObjectID from C. */
|
||||
PyObject *PyObjectID_make(object_id object_id) {
|
||||
PyObjectID *result = PyObject_New(PyObjectID, &PyObjectIDType);
|
||||
result = (PyObjectID *) PyObject_Init((PyObject *) result, &PyObjectIDType);
|
||||
result->object_id = object_id;
|
||||
return (PyObject *) result;
|
||||
}
|
||||
|
||||
static PyObject *PyObjectID_id(PyObject *self) {
|
||||
PyObjectID *s = (PyObjectID *) self;
|
||||
return PyString_FromStringAndSize((char *) &s->object_id.id[0],
|
||||
UNIQUE_ID_SIZE);
|
||||
}
|
||||
|
||||
static PyObject *PyObjectID___reduce__(PyObjectID *self) {
|
||||
PyErr_SetString(CommonError, "ObjectID objects cannot be serialized.");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static PyMethodDef PyObjectID_methods[] = {
|
||||
{"id", (PyCFunction) PyObjectID_id, METH_NOARGS,
|
||||
"Return the hash associated with this ObjectID"},
|
||||
{"__reduce__", (PyCFunction) PyObjectID___reduce__, METH_NOARGS,
|
||||
"Say how to pickle this ObjectID. This raises an exception to prevent"
|
||||
"object IDs from being serialized."},
|
||||
{NULL} /* Sentinel */
|
||||
};
|
||||
|
||||
static PyMemberDef PyObjectID_members[] = {
|
||||
{NULL} /* Sentinel */
|
||||
};
|
||||
|
||||
PyTypeObject PyObjectIDType = {
|
||||
PyObject_HEAD_INIT(NULL) 0, /* ob_size */
|
||||
"common.ObjectID", /* tp_name */
|
||||
sizeof(PyObjectID), /* tp_basicsize */
|
||||
0, /* tp_itemsize */
|
||||
0, /* tp_dealloc */
|
||||
0, /* tp_print */
|
||||
0, /* tp_getattr */
|
||||
0, /* tp_setattr */
|
||||
0, /* tp_compare */
|
||||
0, /* tp_repr */
|
||||
0, /* tp_as_number */
|
||||
0, /* tp_as_sequence */
|
||||
0, /* tp_as_mapping */
|
||||
0, /* tp_hash */
|
||||
0, /* tp_call */
|
||||
0, /* tp_str */
|
||||
0, /* tp_getattro */
|
||||
0, /* tp_setattro */
|
||||
0, /* tp_as_buffer */
|
||||
Py_TPFLAGS_DEFAULT, /* tp_flags */
|
||||
"ObjectID object", /* tp_doc */
|
||||
0, /* tp_traverse */
|
||||
0, /* tp_clear */
|
||||
0, /* tp_richcompare */
|
||||
0, /* tp_weaklistoffset */
|
||||
0, /* tp_iter */
|
||||
0, /* tp_iternext */
|
||||
PyObjectID_methods, /* tp_methods */
|
||||
PyObjectID_members, /* tp_members */
|
||||
0, /* tp_getset */
|
||||
0, /* tp_base */
|
||||
0, /* tp_dict */
|
||||
0, /* tp_descr_get */
|
||||
0, /* tp_descr_set */
|
||||
0, /* tp_dictoffset */
|
||||
(initproc) PyObjectID_init, /* tp_init */
|
||||
0, /* tp_alloc */
|
||||
PyType_GenericNew, /* tp_new */
|
||||
};
|
||||
|
||||
/* Define the PyTask class. */
|
||||
|
||||
static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
|
||||
function_id function_id;
|
||||
/* Arguments of the task (can be PyObjectIDs or Python values). */
|
||||
PyObject *arguments;
|
||||
/* Array of pointers to string representations of pass-by-value args. */
|
||||
UT_array *val_repr_ptrs;
|
||||
utarray_new(val_repr_ptrs, &ut_ptr_icd);
|
||||
int num_returns;
|
||||
if (!PyArg_ParseTuple(args, "O&Oi", &PyObjectToUniqueID, &function_id,
|
||||
&arguments, &num_returns)) {
|
||||
return -1;
|
||||
}
|
||||
size_t size = PyList_Size(arguments);
|
||||
/* Determine the size of pass by value data in bytes. */
|
||||
size_t value_data_bytes = 0;
|
||||
for (size_t i = 0; i < size; ++i) {
|
||||
PyObject *arg = PyList_GetItem(arguments, i);
|
||||
if (!PyObject_IsInstance(arg, (PyObject *) &PyObjectIDType)) {
|
||||
PyObject *data = PyMarshal_WriteObjectToString(arg, MARSHAL_VERSION);
|
||||
value_data_bytes += PyString_Size(data);
|
||||
utarray_push_back(val_repr_ptrs, &data);
|
||||
}
|
||||
}
|
||||
/* Construct the task specification. */
|
||||
int val_repr_index = 0;
|
||||
self->spec =
|
||||
alloc_task_spec(function_id, size, num_returns, value_data_bytes);
|
||||
/* Add the task arguments. */
|
||||
for (size_t i = 0; i < size; ++i) {
|
||||
PyObject *arg = PyList_GetItem(arguments, i);
|
||||
if (PyObject_IsInstance(arg, (PyObject *) &PyObjectIDType)) {
|
||||
task_args_add_ref(self->spec, ((PyObjectID *) arg)->object_id);
|
||||
} else {
|
||||
PyObject *data =
|
||||
*((PyObject **) utarray_eltptr(val_repr_ptrs, val_repr_index));
|
||||
task_args_add_val(self->spec, (uint8_t *) PyString_AS_STRING(data),
|
||||
PyString_GET_SIZE(data));
|
||||
Py_DECREF(data);
|
||||
val_repr_index += 1;
|
||||
}
|
||||
}
|
||||
utarray_free(val_repr_ptrs);
|
||||
/* Generate and add the object IDs for the return values. */
|
||||
for (size_t i = 0; i < num_returns; ++i) {
|
||||
/* TODO(rkn): Later, this should be computed as a deterministic hash of (1)
|
||||
* the contents of the task, (2) the index i, and (3) a counter of the
|
||||
* number of tasks launched so far by the parent task. For now, we generate
|
||||
* it randomly. */
|
||||
*task_return(self->spec, i) = globally_unique_id();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void PyTask_dealloc(PyTask *self) {
|
||||
free_task_spec(self->spec);
|
||||
Py_TYPE(self)->tp_free((PyObject *) self);
|
||||
}
|
||||
|
||||
static PyObject *PyTask_function_id(PyObject *self) {
|
||||
function_id function_id = *task_function(((PyTask *) self)->spec);
|
||||
return PyObjectID_make(function_id);
|
||||
}
|
||||
|
||||
static PyObject *PyTask_arguments(PyObject *self) {
|
||||
int64_t num_args = task_num_args(((PyTask *) self)->spec);
|
||||
PyObject *arg_list = PyList_New((Py_ssize_t) num_args);
|
||||
task_spec *task = ((PyTask *) self)->spec;
|
||||
for (int i = 0; i < num_args; ++i) {
|
||||
if (task_arg_type(task, i) == ARG_BY_REF) {
|
||||
object_id object_id = *task_arg_id(task, i);
|
||||
PyList_SetItem(arg_list, i, PyObjectID_make(object_id));
|
||||
} else {
|
||||
PyObject *s =
|
||||
PyMarshal_ReadObjectFromString((char *) task_arg_val(task, i),
|
||||
(Py_ssize_t) task_arg_length(task, i));
|
||||
PyList_SetItem(arg_list, i, s);
|
||||
}
|
||||
}
|
||||
return arg_list;
|
||||
}
|
||||
|
||||
static PyObject *PyTask_returns(PyObject *self) {
|
||||
int64_t num_returns = task_num_returns(((PyTask *) self)->spec);
|
||||
PyObject *return_id_list = PyList_New((Py_ssize_t) num_returns);
|
||||
task_spec *task = ((PyTask *) self)->spec;
|
||||
for (int i = 0; i < num_returns; ++i) {
|
||||
object_id object_id = *task_return(task, i);
|
||||
PyList_SetItem(return_id_list, i, PyObjectID_make(object_id));
|
||||
}
|
||||
return return_id_list;
|
||||
}
|
||||
|
||||
static PyMethodDef PyTask_methods[] = {
|
||||
{"function_id", (PyCFunction) PyTask_function_id, METH_NOARGS,
|
||||
"Return the function ID for this task."},
|
||||
{"arguments", (PyCFunction) PyTask_arguments, METH_NOARGS,
|
||||
"Return the arguments for the task."},
|
||||
{"returns", (PyCFunction) PyTask_returns, METH_NOARGS,
|
||||
"Return the object IDs for the return values of the task."},
|
||||
{NULL} /* Sentinel */
|
||||
};
|
||||
|
||||
PyTypeObject PyTaskType = {
|
||||
PyObject_HEAD_INIT(NULL) 0, /* ob_size */
|
||||
"task.Task", /* tp_name */
|
||||
sizeof(PyTask), /* tp_basicsize */
|
||||
0, /* tp_itemsize */
|
||||
(destructor) PyTask_dealloc, /* tp_dealloc */
|
||||
0, /* tp_print */
|
||||
0, /* tp_getattr */
|
||||
0, /* tp_setattr */
|
||||
0, /* tp_compare */
|
||||
0, /* tp_repr */
|
||||
0, /* tp_as_number */
|
||||
0, /* tp_as_sequence */
|
||||
0, /* tp_as_mapping */
|
||||
0, /* tp_hash */
|
||||
0, /* tp_call */
|
||||
0, /* tp_str */
|
||||
0, /* tp_getattro */
|
||||
0, /* tp_setattro */
|
||||
0, /* tp_as_buffer */
|
||||
Py_TPFLAGS_DEFAULT, /* tp_flags */
|
||||
"Task object", /* tp_doc */
|
||||
0, /* tp_traverse */
|
||||
0, /* tp_clear */
|
||||
0, /* tp_richcompare */
|
||||
0, /* tp_weaklistoffset */
|
||||
0, /* tp_iter */
|
||||
0, /* tp_iternext */
|
||||
PyTask_methods, /* tp_methods */
|
||||
0, /* tp_members */
|
||||
0, /* tp_getset */
|
||||
0, /* tp_base */
|
||||
0, /* tp_dict */
|
||||
0, /* tp_descr_get */
|
||||
0, /* tp_descr_set */
|
||||
0, /* tp_dictoffset */
|
||||
(initproc) PyTask_init, /* tp_init */
|
||||
0, /* tp_alloc */
|
||||
PyType_GenericNew, /* tp_new */
|
||||
};
|
||||
|
||||
/* Create a PyTask from a C struct. The resulting PyTask takes ownership of the
|
||||
* task_spec and will deallocate the task_spec in the PyTask destructor. */
|
||||
PyObject *PyTask_make(task_spec *task_spec) {
|
||||
PyTask *result = PyObject_New(PyTask, &PyTaskType);
|
||||
result = (PyTask *) PyObject_Init((PyObject *) result, &PyTaskType);
|
||||
result->spec = task_spec;
|
||||
return (PyObject *) result;
|
||||
}
|
||||
|
||||
/* Define the methods for the module. */
|
||||
|
||||
#define SIZE_LIMIT 100
|
||||
#define NUM_ELEMENTS_LIMIT 1000
|
||||
|
||||
/**
|
||||
* This method checks if a Python object is sufficiently simple that it can be
|
||||
* serialized and passed by value as an argument to a task (without being put in
|
||||
* the object store). The details of which objects are sufficiently simple are
|
||||
* defined by this method and are not particularly important. But for
|
||||
* performance reasons, it is better to place "small" objects in the task itself
|
||||
* and "large" objects in the object store.
|
||||
*
|
||||
* @param value The Python object in question.
|
||||
* @param num_elements_contained If this method returns 1, then the number of
|
||||
* objects recursively contained within this object will be added to the
|
||||
* value at this address. This is used to make sure that we do not
|
||||
* serialize objects that are too large.
|
||||
* @return 0 if the object cannot be serialized in the task and 1 if it can.
|
||||
*/
|
||||
int is_simple_value(PyObject *value, int *num_elements_contained) {
|
||||
*num_elements_contained += 1;
|
||||
if (*num_elements_contained >= NUM_ELEMENTS_LIMIT) {
|
||||
return 0;
|
||||
}
|
||||
if (PyInt_Check(value) || PyLong_Check(value) || value == Py_False ||
|
||||
value == Py_True || PyFloat_Check(value) || value == Py_None) {
|
||||
return 1;
|
||||
}
|
||||
if (PyString_CheckExact(value)) {
|
||||
*num_elements_contained += PyString_Size(value);
|
||||
return (*num_elements_contained < NUM_ELEMENTS_LIMIT);
|
||||
}
|
||||
if (PyUnicode_CheckExact(value)) {
|
||||
*num_elements_contained += PyUnicode_GET_SIZE(value);
|
||||
return (*num_elements_contained < NUM_ELEMENTS_LIMIT);
|
||||
}
|
||||
if (PyList_CheckExact(value) && PyList_Size(value) < SIZE_LIMIT) {
|
||||
for (size_t i = 0; i < PyList_Size(value); ++i) {
|
||||
if (!is_simple_value(PyList_GetItem(value, i), num_elements_contained)) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return (*num_elements_contained < NUM_ELEMENTS_LIMIT);
|
||||
}
|
||||
if (PyDict_CheckExact(value) && PyDict_Size(value) < SIZE_LIMIT) {
|
||||
PyObject *key, *val;
|
||||
Py_ssize_t pos = 0;
|
||||
while (PyDict_Next(value, &pos, &key, &val)) {
|
||||
if (!is_simple_value(key, num_elements_contained) ||
|
||||
!is_simple_value(val, num_elements_contained)) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return (*num_elements_contained < NUM_ELEMENTS_LIMIT);
|
||||
}
|
||||
if (PyTuple_CheckExact(value) && PyTuple_Size(value) < SIZE_LIMIT) {
|
||||
for (size_t i = 0; i < PyTuple_Size(value); ++i) {
|
||||
if (!is_simple_value(PyTuple_GetItem(value, i), num_elements_contained)) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return (*num_elements_contained < NUM_ELEMENTS_LIMIT);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
PyObject *check_simple_value(PyObject *self, PyObject *args) {
|
||||
PyObject *value;
|
||||
if (!PyArg_ParseTuple(args, "O", &value)) {
|
||||
return NULL;
|
||||
}
|
||||
int num_elements_contained = 0;
|
||||
if (is_simple_value(value, &num_elements_contained)) {
|
||||
Py_RETURN_TRUE;
|
||||
}
|
||||
Py_RETURN_FALSE;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
#ifndef COMMON_EXTENSION_H
|
||||
#define COMMON_EXTENSION_H
|
||||
|
||||
#include <Python.h>
|
||||
#include "marshal.h"
|
||||
#include "structmember.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "task.h"
|
||||
|
||||
extern PyObject *CommonError;
|
||||
|
||||
// clang-format off
|
||||
typedef struct {
|
||||
PyObject_HEAD
|
||||
object_id object_id;
|
||||
} PyObjectID;
|
||||
|
||||
typedef struct {
|
||||
PyObject_HEAD
|
||||
task_spec *spec;
|
||||
} PyTask;
|
||||
// clang-format on
|
||||
|
||||
extern PyTypeObject PyObjectIDType;
|
||||
|
||||
extern PyTypeObject PyTaskType;
|
||||
|
||||
int PyObjectToUniqueID(PyObject *object, object_id *objectid);
|
||||
|
||||
PyObject *PyObjectID_make(object_id object_id);
|
||||
|
||||
PyObject *check_simple_value(PyObject *self, PyObject *args);
|
||||
|
||||
PyObject *PyTask_make(task_spec *task_spec);
|
||||
|
||||
#endif /* COMMON_EXTENSION_H */
|
||||
@@ -0,0 +1,38 @@
|
||||
#include <Python.h>
|
||||
#include "node.h"
|
||||
|
||||
#include "common_extension.h"
|
||||
|
||||
static PyMethodDef common_methods[] = {
|
||||
{"check_simple_value", check_simple_value, METH_VARARGS,
|
||||
"Should the object be passed by value?"},
|
||||
{NULL} /* Sentinel */
|
||||
};
|
||||
|
||||
#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
|
||||
#define PyMODINIT_FUNC void
|
||||
#endif
|
||||
|
||||
PyMODINIT_FUNC initcommon(void) {
|
||||
PyObject *m;
|
||||
|
||||
if (PyType_Ready(&PyTaskType) < 0)
|
||||
return;
|
||||
|
||||
if (PyType_Ready(&PyObjectIDType) < 0)
|
||||
return;
|
||||
|
||||
m = Py_InitModule3("common", common_methods,
|
||||
"A module for common types. This is used for testing.");
|
||||
|
||||
Py_INCREF(&PyTaskType);
|
||||
PyModule_AddObject(m, "Task", (PyObject *) &PyTaskType);
|
||||
|
||||
Py_INCREF(&PyObjectIDType);
|
||||
PyModule_AddObject(m, "ObjectID", (PyObject *) &PyObjectIDType);
|
||||
|
||||
char common_error[] = "common.error";
|
||||
CommonError = PyErr_NewException(common_error, NULL, NULL);
|
||||
Py_INCREF(CommonError);
|
||||
PyModule_AddObject(m, "common_error", CommonError);
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
from setuptools import setup, find_packages, Extension
|
||||
|
||||
common_module = Extension("common",
|
||||
sources=["common_module.c", "common_extension.c"],
|
||||
include_dirs=["../../", "../../thirdparty"],
|
||||
extra_objects=["../../build/libcommon.a"],
|
||||
extra_compile_args=["--std=c99", "-Werror"])
|
||||
|
||||
setup(name="Common",
|
||||
version="0.1",
|
||||
description="Common library for Ray",
|
||||
ext_modules=[common_module])
|
||||
@@ -0,0 +1,80 @@
|
||||
#include "logging.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include <inttypes.h>
|
||||
#include <hiredis/hiredis.h>
|
||||
#include <utstring.h>
|
||||
|
||||
#include "state/redis.h"
|
||||
#include "io.h"
|
||||
|
||||
static const char *log_levels[5] = {"DEBUG", "INFO", "WARN", "ERROR", "FATAL"};
|
||||
static const char *log_fmt =
|
||||
"HMSET log:%s:%s:%s log_level %s event_type %s message %s timestamp %s";
|
||||
|
||||
struct ray_logger_impl {
|
||||
/* String that identifies this client type. */
|
||||
const char *client_type;
|
||||
/* Suppress all log messages below this level. */
|
||||
int log_level;
|
||||
/* Whether or not we have a direct connection to Redis. */
|
||||
int is_direct;
|
||||
/* Either a db_handle or a socket to a process with a db_handle,
|
||||
* depending on the is_direct flag. */
|
||||
void *conn;
|
||||
};
|
||||
|
||||
ray_logger *init_ray_logger(const char *client_type,
|
||||
int log_level,
|
||||
int is_direct,
|
||||
void *conn) {
|
||||
ray_logger *logger = malloc(sizeof(ray_logger));
|
||||
logger->client_type = client_type;
|
||||
logger->log_level = log_level;
|
||||
logger->is_direct = is_direct;
|
||||
logger->conn = conn;
|
||||
return logger;
|
||||
}
|
||||
|
||||
void free_ray_logger(ray_logger *logger) {
|
||||
free(logger);
|
||||
}
|
||||
|
||||
void ray_log(ray_logger *logger,
|
||||
int log_level,
|
||||
const char *event_type,
|
||||
const char *message) {
|
||||
if (log_level < logger->log_level) {
|
||||
return;
|
||||
}
|
||||
if (log_level < RAY_DEBUG || log_level > RAY_FATAL) {
|
||||
return;
|
||||
}
|
||||
struct timeval tv;
|
||||
UT_string *timestamp;
|
||||
utstring_new(timestamp);
|
||||
gettimeofday(&tv, NULL);
|
||||
utstring_printf(timestamp, "%ld.%ld", tv.tv_sec, (long) tv.tv_usec);
|
||||
|
||||
UT_string *origin_id;
|
||||
utstring_new(origin_id);
|
||||
if (logger->is_direct) {
|
||||
db_handle *db = (db_handle *) logger->conn;
|
||||
utstring_printf(origin_id, "%" PRId64 ":%s", db->client_id, "");
|
||||
redisAsyncCommand(db->context, NULL, NULL, log_fmt,
|
||||
utstring_body(timestamp), logger->client_type,
|
||||
utstring_body(origin_id), log_levels[log_level],
|
||||
event_type, message, utstring_body(timestamp));
|
||||
} else {
|
||||
/* If we don't own a Redis connection, we leave our client
|
||||
* ID to be filled in by someone else. */
|
||||
utstring_printf(origin_id, "%s:%s", "%ld", "%ld");
|
||||
int *socket_fd = (int *) logger->conn;
|
||||
write_formatted_log_message(*socket_fd, log_fmt, utstring_body(timestamp),
|
||||
logger->client_type, utstring_body(origin_id),
|
||||
log_levels[log_level], event_type, message,
|
||||
utstring_body(timestamp));
|
||||
}
|
||||
utstring_free(origin_id);
|
||||
utstring_free(timestamp);
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
#ifndef LOGGING_H
|
||||
#define LOGGING_H
|
||||
|
||||
#define RAY_VERBOSE -1
|
||||
#define RAY_DEBUG 0
|
||||
#define RAY_INFO 1
|
||||
#define RAY_WARNING 2
|
||||
#define RAY_ERROR 3
|
||||
#define RAY_FATAL 4
|
||||
|
||||
/* Entity types. */
|
||||
#define RAY_FUNCTION "FUNCTION"
|
||||
#define RAY_OBJECT "OBJECT"
|
||||
#define RAY_TASK "TASK"
|
||||
|
||||
typedef struct ray_logger_impl ray_logger;
|
||||
|
||||
/* Initialize a Ray logger for the given client type and logging level. If the
|
||||
* is_direct flag is set, the logger will treat the given connection as a
|
||||
* direct connection to the log. Otherwise, it will treat it as a socket to
|
||||
* another process with a connection to the log.
|
||||
* NOTE: User is responsible for freeing the returned logger. */
|
||||
ray_logger *init_ray_logger(const char *client_type,
|
||||
int log_level,
|
||||
int is_direct,
|
||||
void *conn);
|
||||
|
||||
/* Free the logger. This does not free the connection to the log. */
|
||||
void free_ray_logger(ray_logger *logger);
|
||||
|
||||
/* Log an event at the given log level with the given event_type.
|
||||
* NOTE: message cannot contain spaces! JSON format is recommended.
|
||||
* TODO: Support spaces in messages. */
|
||||
void ray_log(ray_logger *logger,
|
||||
int log_level,
|
||||
const char *event_type,
|
||||
const char *message);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
#ifndef DB_H
|
||||
#define DB_H
|
||||
|
||||
#include "event_loop.h"
|
||||
|
||||
typedef struct db_handle_impl db_handle;
|
||||
|
||||
/* Connect to the global system store at address and port. Returns
|
||||
* a handle to the database, which must be freed with db_disconnect
|
||||
* after use. */
|
||||
db_handle *db_connect(const char *db_address,
|
||||
int db_port,
|
||||
const char *client_type,
|
||||
const char *client_addr,
|
||||
int client_port);
|
||||
|
||||
/* Attach global system store connection to event loop. */
|
||||
void db_attach(db_handle *db, event_loop *loop);
|
||||
|
||||
/* Disconnect from the global system store. */
|
||||
void db_disconnect(db_handle *db);
|
||||
|
||||
/**
|
||||
* Returns the client ID, according to the database.
|
||||
*
|
||||
* @param db The handle to the database.
|
||||
* @returns int The client ID for this connection to the database. If
|
||||
* this client has no connection to the database, returns -1.
|
||||
*/
|
||||
int get_client_id(db_handle *db);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,25 @@
|
||||
#include "common.h"
|
||||
#include "db.h"
|
||||
|
||||
/* The callback that is called when the result of a lookup
|
||||
* in the object table comes back. The callback should free
|
||||
* the manager_vector array, but NOT the strings they are pointing to. */
|
||||
typedef void (*lookup_callback)(object_id object_id,
|
||||
int manager_count,
|
||||
const char *manager_vector[],
|
||||
void *context);
|
||||
|
||||
/* Register a new object with the directory. */
|
||||
/* TODO(pcm): Retry, print for each attempt. */
|
||||
void object_table_add(db_handle *db, object_id object_id);
|
||||
|
||||
/* Remove object from the directory. */
|
||||
void object_table_remove(db_handle *db,
|
||||
object_id object_id,
|
||||
const char *manager);
|
||||
|
||||
/* Look up entry from the directory */
|
||||
void object_table_lookup(db_handle *db,
|
||||
object_id object_id,
|
||||
lookup_callback callback,
|
||||
void *context);
|
||||
@@ -0,0 +1,244 @@
|
||||
/* Redis implementation of the global state store */
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
#include <stdlib.h>
|
||||
#include "hiredis/adapters/ae.h"
|
||||
#include "utstring.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "db.h"
|
||||
#include "object_table.h"
|
||||
#include "task_log.h"
|
||||
#include "event_loop.h"
|
||||
#include "redis.h"
|
||||
#include "io.h"
|
||||
|
||||
#define LOG_REDIS_ERR(context, M, ...) \
|
||||
fprintf(stderr, "[ERROR] (%s:%d: message: %s) " M "\n", __FILE__, __LINE__, \
|
||||
context->errstr, ##__VA_ARGS__)
|
||||
|
||||
#define CHECK_REDIS_CONNECT(CONTEXT_TYPE, context, M, ...) \
|
||||
do { \
|
||||
CONTEXT_TYPE *_context = (context); \
|
||||
if (!_context) { \
|
||||
LOG_ERR("could not allocate redis context"); \
|
||||
exit(-1); \
|
||||
} \
|
||||
if (_context->err) { \
|
||||
LOG_REDIS_ERR(_context, M, ##__VA_ARGS__); \
|
||||
exit(-1); \
|
||||
} \
|
||||
} while (0);
|
||||
|
||||
db_handle *db_connect(const char *address,
|
||||
int port,
|
||||
const char *client_type,
|
||||
const char *client_addr,
|
||||
int client_port) {
|
||||
db_handle *db = malloc(sizeof(db_handle));
|
||||
/* Sync connection for initial handshake */
|
||||
redisReply *reply;
|
||||
long long num_clients;
|
||||
redisContext *context = redisConnect(address, port);
|
||||
CHECK_REDIS_CONNECT(redisContext, context, "could not connect to redis %s:%d",
|
||||
address, port);
|
||||
/* Add new client using optimistic locking. */
|
||||
while (1) {
|
||||
reply = redisCommand(context, "WATCH %s", client_type);
|
||||
freeReplyObject(reply);
|
||||
reply = redisCommand(context, "HLEN %s", client_type);
|
||||
num_clients = reply->integer;
|
||||
freeReplyObject(reply);
|
||||
reply = redisCommand(context, "MULTI");
|
||||
freeReplyObject(reply);
|
||||
reply = redisCommand(context, "HSET %s %lld %s:%d", client_type,
|
||||
num_clients, client_addr, client_port);
|
||||
freeReplyObject(reply);
|
||||
reply = redisCommand(context, "EXEC");
|
||||
CHECK(reply);
|
||||
if (reply->type != REDIS_REPLY_NIL) {
|
||||
freeReplyObject(reply);
|
||||
break;
|
||||
}
|
||||
freeReplyObject(reply);
|
||||
}
|
||||
|
||||
db->client_type = strdup(client_type);
|
||||
db->client_id = num_clients;
|
||||
db->service_cache = NULL;
|
||||
db->sync_context = context;
|
||||
utarray_new(db->callback_freelist, &ut_ptr_icd);
|
||||
|
||||
/* Establish async connection */
|
||||
db->context = redisAsyncConnect(address, port);
|
||||
CHECK_REDIS_CONNECT(redisAsyncContext, db->context,
|
||||
"could not connect to redis %s:%d", address, port);
|
||||
db->context->data = (void *) db;
|
||||
/* Establish async connection for subscription */
|
||||
db->sub_context = redisAsyncConnect(address, port);
|
||||
CHECK_REDIS_CONNECT(redisAsyncContext, db->sub_context,
|
||||
"could not connect to redis %s:%d", address, port);
|
||||
db->sub_context->data = (void *) db;
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
void db_disconnect(db_handle *db) {
|
||||
redisFree(db->sync_context);
|
||||
redisAsyncFree(db->context);
|
||||
redisAsyncFree(db->sub_context);
|
||||
service_cache_entry *e, *tmp;
|
||||
HASH_ITER(hh, db->service_cache, e, tmp) {
|
||||
free(e->addr);
|
||||
HASH_DEL(db->service_cache, e);
|
||||
free(e);
|
||||
}
|
||||
free(db->client_type);
|
||||
void **p = NULL;
|
||||
while ((p = (void **) utarray_next(db->callback_freelist, p))) {
|
||||
free(*p);
|
||||
}
|
||||
utarray_free(db->callback_freelist);
|
||||
free(db);
|
||||
}
|
||||
|
||||
void db_attach(db_handle *db, event_loop *loop) {
|
||||
redisAeAttach(loop, db->context);
|
||||
redisAeAttach(loop, db->sub_context);
|
||||
}
|
||||
|
||||
void object_table_add(db_handle *db, unique_id object_id) {
|
||||
redisAsyncCommand(db->context, NULL, NULL, "SADD obj:%b %d", &object_id.id[0],
|
||||
UNIQUE_ID_SIZE, db->client_id);
|
||||
if (db->context->err) {
|
||||
LOG_REDIS_ERR(db->context, "could not add object_table entry");
|
||||
}
|
||||
}
|
||||
|
||||
void object_table_get_entry(redisAsyncContext *c, void *r, void *privdata) {
|
||||
db_handle *db = c->data;
|
||||
lookup_callback_data *cb_data = privdata;
|
||||
redisReply *reply = r;
|
||||
if (reply == NULL)
|
||||
return;
|
||||
int *result = malloc(reply->elements * sizeof(int));
|
||||
int64_t manager_count = reply->elements;
|
||||
if (reply->type == REDIS_REPLY_ARRAY) {
|
||||
for (int j = 0; j < reply->elements; j++) {
|
||||
CHECK(reply->element[j]->type == REDIS_REPLY_STRING);
|
||||
result[j] = atoi(reply->element[j]->str);
|
||||
service_cache_entry *entry;
|
||||
HASH_FIND_INT(db->service_cache, &result[j], entry);
|
||||
if (!entry) {
|
||||
redisReply *reply = redisCommand(db->sync_context, "HGET %s %lld",
|
||||
db->client_type, result[j]);
|
||||
CHECK(reply->type == REDIS_REPLY_STRING);
|
||||
entry = malloc(sizeof(service_cache_entry));
|
||||
entry->service_id = result[j];
|
||||
entry->addr = strdup(reply->str);
|
||||
HASH_ADD_INT(db->service_cache, service_id, entry);
|
||||
freeReplyObject(reply);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
LOG_ERR("expected integer or string, received type %d", reply->type);
|
||||
exit(-1);
|
||||
}
|
||||
const char **manager_vector = malloc(manager_count * sizeof(char *));
|
||||
for (int j = 0; j < manager_count; ++j) {
|
||||
service_cache_entry *entry;
|
||||
HASH_FIND_INT(db->service_cache, &result[j], entry);
|
||||
manager_vector[j] = entry->addr;
|
||||
}
|
||||
cb_data->callback(cb_data->object_id, manager_count, manager_vector,
|
||||
cb_data->context);
|
||||
free(privdata);
|
||||
free(result);
|
||||
}
|
||||
|
||||
void object_table_lookup(db_handle *db,
|
||||
object_id object_id,
|
||||
lookup_callback callback,
|
||||
void *context) {
|
||||
lookup_callback_data *cb_data = malloc(sizeof(lookup_callback_data));
|
||||
cb_data->callback = callback;
|
||||
cb_data->object_id = object_id;
|
||||
cb_data->context = context;
|
||||
redisAsyncCommand(db->context, object_table_get_entry, cb_data,
|
||||
"SMEMBERS obj:%b", &object_id.id[0], UNIQUE_ID_SIZE);
|
||||
if (db->context->err) {
|
||||
LOG_REDIS_ERR(db->context, "error in object_table lookup");
|
||||
}
|
||||
}
|
||||
|
||||
void task_log_add_task(db_handle *db, task_instance *task_instance) {
|
||||
task_iid task_iid = *task_instance_id(task_instance);
|
||||
redisAsyncCommand(db->context, NULL, NULL, "HMSET tasklog:%b 0 %b",
|
||||
(char *) &task_iid.id[0], UNIQUE_ID_SIZE,
|
||||
(char *) task_instance, task_instance_size(task_instance));
|
||||
if (db->context->err) {
|
||||
LOG_REDIS_ERR(db->context, "error setting task in task_log_add_task");
|
||||
}
|
||||
node_id node = *task_instance_node(task_instance);
|
||||
int32_t state = *task_instance_state(task_instance);
|
||||
redisAsyncCommand(db->context, NULL, NULL, "PUBLISH task_log:%b:%d %b",
|
||||
(char *) &node.id[0], UNIQUE_ID_SIZE, state,
|
||||
(char *) task_instance, task_instance_size(task_instance));
|
||||
if (db->context->err) {
|
||||
LOG_REDIS_ERR(db->context, "error publishing task in task_log_add_task");
|
||||
}
|
||||
}
|
||||
|
||||
void task_log_redis_callback(redisAsyncContext *c,
|
||||
void *reply,
|
||||
void *privdata) {
|
||||
redisReply *r = reply;
|
||||
if (reply == NULL)
|
||||
return;
|
||||
CHECK(r->type == REDIS_REPLY_ARRAY);
|
||||
/* First entry is message type, second is topic, third is payload. */
|
||||
CHECK(r->elements > 2);
|
||||
/* If this condition is true, we got the initial message that acknowledged the
|
||||
* subscription. */
|
||||
if (r->element[2]->str == NULL) {
|
||||
return;
|
||||
}
|
||||
/* Otherwise, parse the task and call the callback. */
|
||||
CHECK(privdata);
|
||||
task_log_callback_data *callback_data = privdata;
|
||||
task_instance *instance = malloc(r->element[2]->len);
|
||||
memcpy(instance, r->element[2]->str, r->element[2]->len);
|
||||
callback_data->callback(instance, callback_data->userdata);
|
||||
task_instance_free(instance);
|
||||
}
|
||||
void task_log_register_callback(db_handle *db,
|
||||
task_log_callback callback,
|
||||
node_id node,
|
||||
int32_t state,
|
||||
void *userdata) {
|
||||
task_log_callback_data *callback_data =
|
||||
malloc(sizeof(task_log_callback_data));
|
||||
utarray_push_back(db->callback_freelist, &callback_data);
|
||||
callback_data->callback = callback;
|
||||
callback_data->userdata = userdata;
|
||||
if (memcmp(&node.id[0], &NIL_ID.id[0], UNIQUE_ID_SIZE) == 0) {
|
||||
redisAsyncCommand(db->sub_context, task_log_redis_callback, callback_data,
|
||||
"PSUBSCRIBE task_log:*:%d", state);
|
||||
} else {
|
||||
redisAsyncCommand(db->sub_context, task_log_redis_callback, callback_data,
|
||||
"SUBSCRIBE task_log:%b:%d", (char *) &node.id[0],
|
||||
UNIQUE_ID_SIZE, state);
|
||||
}
|
||||
if (db->sub_context->err) {
|
||||
LOG_REDIS_ERR(db->sub_context, "error in task_log_register_callback");
|
||||
}
|
||||
}
|
||||
|
||||
int get_client_id(db_handle *db) {
|
||||
if (db) {
|
||||
return db->client_id;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
#ifndef REDIS_H
|
||||
#define REDIS_H
|
||||
|
||||
#include "db.h"
|
||||
#include "object_table.h"
|
||||
#include "task_log.h"
|
||||
|
||||
#include "hiredis/hiredis.h"
|
||||
#include "hiredis/async.h"
|
||||
#include "uthash.h"
|
||||
#include "utarray.h"
|
||||
|
||||
typedef struct {
|
||||
/* Unique ID for this service. */
|
||||
int service_id;
|
||||
/* IP address and port of this service. */
|
||||
char *addr;
|
||||
/* Handle for the uthash table. */
|
||||
UT_hash_handle hh;
|
||||
} service_cache_entry;
|
||||
|
||||
typedef struct {
|
||||
/* The callback that will be called. */
|
||||
task_log_callback callback;
|
||||
/* Userdata associated with the callback. */
|
||||
void *userdata;
|
||||
} task_log_callback_data;
|
||||
|
||||
struct db_handle_impl {
|
||||
/* String that identifies this client type. */
|
||||
char *client_type;
|
||||
/* Unique ID for this client within the type. */
|
||||
int64_t client_id;
|
||||
/* Redis context for this global state store connection. */
|
||||
redisAsyncContext *context;
|
||||
/* Redis context for "subscribe" communication.
|
||||
* Yes, we need a separate one for that, see
|
||||
* https://github.com/redis/hiredis/issues/55 */
|
||||
redisAsyncContext *sub_context;
|
||||
/* The event loop this global state store connection is part of. */
|
||||
event_loop *loop;
|
||||
/* Index of the database connection in the event loop */
|
||||
int64_t db_index;
|
||||
/* Cache for the IP addresses of services. */
|
||||
service_cache_entry *service_cache;
|
||||
/* Redis context for synchronous connections.
|
||||
* Should only be used very rarely, it is not asynchronous. */
|
||||
redisContext *sync_context;
|
||||
/* Data structure for callbacks that needs to be freed. */
|
||||
UT_array *callback_freelist;
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
/* The callback that will be called. */
|
||||
lookup_callback callback;
|
||||
/* Object ID that is looked up. */
|
||||
object_id object_id;
|
||||
/* Data context for the callback. */
|
||||
void *context;
|
||||
} lookup_callback_data;
|
||||
|
||||
void object_table_get_entry(redisAsyncContext *c, void *r, void *privdata);
|
||||
|
||||
void object_table_lookup_callback(redisAsyncContext *c,
|
||||
void *r,
|
||||
void *privdata);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,41 @@
|
||||
#ifndef TASK_LOG_H
|
||||
#define TASK_LOG_H
|
||||
|
||||
#include "db.h"
|
||||
#include "task.h"
|
||||
|
||||
/* The task log is a message bus that is used for all communication between
|
||||
* local and global schedulers (and also persisted to the state database).
|
||||
* Here are examples of events that are recorded by the task log:
|
||||
*
|
||||
* 1) local scheduler writes it when submits a task to the global scheduler;
|
||||
* 2) global scheduler reads it to get the task submitted by local schedulers;
|
||||
* 3) global scheduler writes it when assigning the task to a local scheduler;
|
||||
* 4) local scheduler reads it to get its tasks assigned by global scheduler;
|
||||
* 5) local scheduler writes it when a task finishes execution;
|
||||
* 6) global scheduler reads it to get the tasks that have finished; */
|
||||
|
||||
/* Callback for subscribing to the task log. */
|
||||
typedef void (*task_log_callback)(task_instance *task_instance, void *userdata);
|
||||
|
||||
/* Initially add a task instance to the task log. */
|
||||
void task_log_add_task(db_handle *db, task_instance *task_instance);
|
||||
|
||||
/* Update task instance in the task log. */
|
||||
void task_log_update_task(db_handle *db,
|
||||
task_iid task_iid,
|
||||
int32_t state,
|
||||
node_id node);
|
||||
|
||||
/* Register callback for a certain event. The node specifies the node whose
|
||||
* events we want to listen to. If you want to listen to all events for this
|
||||
* node, use state_filter =
|
||||
* TASK_WAITING | TASK_SCHEDULED | TASK_RUNNING | TASK_DONE.
|
||||
* If you want to register to updates from all nodes, set node = NIL_ID. */
|
||||
void task_log_register_callback(db_handle *db,
|
||||
task_log_callback callback,
|
||||
node_id node,
|
||||
int32_t state_filter,
|
||||
void *userdata);
|
||||
|
||||
#endif /* TASK_LOG_H */
|
||||
@@ -0,0 +1,20 @@
|
||||
#ifndef TASK_TABLE_H
|
||||
#define TASK_TABLE_H
|
||||
|
||||
#include "db.h"
|
||||
#include "task.h"
|
||||
|
||||
/* Add task to the task table, handle errors here. */
|
||||
status task_table_add_task(db_handle *db, task_spec *task);
|
||||
|
||||
/* Callback for getting an entry from the task table. Task spec will be freed
|
||||
* by the system after the callback */
|
||||
typedef void (*task_table_callback)(task_spec *task, void *context);
|
||||
|
||||
/* Get specific task from the task table. */
|
||||
status task_table_get_task(db_handle *db,
|
||||
task_id task_id,
|
||||
task_table_callback callback,
|
||||
void *context);
|
||||
|
||||
#endif /* TASK_TABLE_H */
|
||||
@@ -0,0 +1,218 @@
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "utarray.h"
|
||||
|
||||
#include "task.h"
|
||||
#include "common.h"
|
||||
#include "io.h"
|
||||
|
||||
/* TASK SPECIFICATIONS */
|
||||
|
||||
/* Tasks are stored in a consecutive chunk of memory, the first
|
||||
* sizeof(task_spec) bytes are arranged according to the struct
|
||||
* task_spec. Then there is an array of task_args of length
|
||||
* (num_args + num_returns), and then follows the data of
|
||||
* pass-by-value arguments of size args_value_size. The offsets in the
|
||||
* task_arg.val are with respect to the end of the augmented structure,
|
||||
* i.e. with respect to the address &task_spec.args_and_returns[0] +
|
||||
* (task_spec->num_args + task_spec->num_returns) * sizeof(task_arg). */
|
||||
|
||||
typedef struct {
|
||||
/* Either ARG_BY_REF or ARG_BY_VAL. */
|
||||
int8_t type;
|
||||
union {
|
||||
object_id obj_id;
|
||||
struct {
|
||||
/* Offset where the data associated to this arg is located relative
|
||||
* to &task_spec.args_and_returns[0]. */
|
||||
ptrdiff_t offset;
|
||||
int64_t length;
|
||||
} value;
|
||||
};
|
||||
} task_arg;
|
||||
|
||||
struct task_spec_impl {
|
||||
/* Function ID of the task. */
|
||||
function_id function_id;
|
||||
/* Total number of arguments. */
|
||||
int64_t num_args;
|
||||
/* Index of the last argument that has been constructed. */
|
||||
int64_t arg_index;
|
||||
/* Number of return values. */
|
||||
int64_t num_returns;
|
||||
/* Number of bytes the pass-by-value arguments are occupying. */
|
||||
int64_t args_value_size;
|
||||
/* The offset of the number of bytes of pass-by-value data that
|
||||
* has been written so far, relative to &task_spec->args_and_returns[0] +
|
||||
* (task_spec->num_args + task_spec->num_returns) * sizeof(task_arg) */
|
||||
int64_t args_value_offset;
|
||||
/* Argument and return IDs as well as offsets for pass-by-value args. */
|
||||
task_arg args_and_returns[0];
|
||||
};
|
||||
|
||||
/* The size of a task specification is given by the following expression. */
|
||||
#define TASK_SPEC_SIZE(NUM_ARGS, NUM_RETURNS, ARGS_VALUE_SIZE) \
|
||||
(sizeof(task_spec) + ((NUM_ARGS) + (NUM_RETURNS)) * sizeof(task_arg) + \
|
||||
(ARGS_VALUE_SIZE))
|
||||
|
||||
task_spec *alloc_task_spec(function_id function_id,
|
||||
int64_t num_args,
|
||||
int64_t num_returns,
|
||||
int64_t args_value_size) {
|
||||
int64_t size = TASK_SPEC_SIZE(num_args, num_returns, args_value_size);
|
||||
task_spec *task = malloc(size);
|
||||
memset(task, 0, size);
|
||||
task->function_id = function_id;
|
||||
task->num_args = num_args;
|
||||
task->arg_index = 0;
|
||||
task->num_returns = num_returns;
|
||||
task->args_value_size = args_value_size;
|
||||
return task;
|
||||
}
|
||||
|
||||
int64_t task_size(task_spec *spec) {
|
||||
return TASK_SPEC_SIZE(spec->num_args, spec->num_returns,
|
||||
spec->args_value_size);
|
||||
}
|
||||
|
||||
unique_id *task_function(task_spec *spec) {
|
||||
return &spec->function_id;
|
||||
}
|
||||
|
||||
int64_t task_num_args(task_spec *spec) {
|
||||
return spec->num_args;
|
||||
}
|
||||
|
||||
int64_t task_num_returns(task_spec *spec) {
|
||||
return spec->num_returns;
|
||||
}
|
||||
|
||||
int8_t task_arg_type(task_spec *spec, int64_t arg_index) {
|
||||
CHECK(0 <= arg_index && arg_index < spec->num_args);
|
||||
return spec->args_and_returns[arg_index].type;
|
||||
}
|
||||
|
||||
object_id *task_arg_id(task_spec *spec, int64_t arg_index) {
|
||||
CHECK(0 <= arg_index && arg_index < spec->num_args);
|
||||
task_arg *arg = &spec->args_and_returns[arg_index];
|
||||
CHECK(arg->type == ARG_BY_REF)
|
||||
return &arg->obj_id;
|
||||
}
|
||||
|
||||
uint8_t *task_arg_val(task_spec *spec, int64_t arg_index) {
|
||||
CHECK(0 <= arg_index && arg_index < spec->num_args);
|
||||
task_arg *arg = &spec->args_and_returns[arg_index];
|
||||
CHECK(arg->type == ARG_BY_VAL);
|
||||
uint8_t *data = (uint8_t *) &spec->args_and_returns[0];
|
||||
data += (spec->num_args + spec->num_returns) * sizeof(task_arg);
|
||||
return data + arg->value.offset;
|
||||
}
|
||||
|
||||
int64_t task_arg_length(task_spec *spec, int64_t arg_index) {
|
||||
CHECK(0 <= arg_index && arg_index < spec->num_args);
|
||||
task_arg *arg = &spec->args_and_returns[arg_index];
|
||||
CHECK(arg->type == ARG_BY_VAL);
|
||||
return arg->value.length;
|
||||
}
|
||||
|
||||
int64_t task_args_add_ref(task_spec *spec, object_id obj_id) {
|
||||
task_arg *arg = &spec->args_and_returns[spec->arg_index];
|
||||
arg->type = ARG_BY_REF;
|
||||
arg->obj_id = obj_id;
|
||||
return spec->arg_index++;
|
||||
}
|
||||
|
||||
int64_t task_args_add_val(task_spec *spec, uint8_t *data, int64_t length) {
|
||||
task_arg *arg = &spec->args_and_returns[spec->arg_index];
|
||||
arg->type = ARG_BY_VAL;
|
||||
arg->value.offset = spec->args_value_offset;
|
||||
arg->value.length = length;
|
||||
uint8_t *addr = task_arg_val(spec, spec->arg_index);
|
||||
CHECK(spec->args_value_offset + length <= spec->args_value_size);
|
||||
CHECK(spec->arg_index != spec->num_args - 1 ||
|
||||
spec->args_value_offset + length == spec->args_value_size);
|
||||
memcpy(addr, data, length);
|
||||
spec->args_value_offset += length;
|
||||
return spec->arg_index++;
|
||||
}
|
||||
|
||||
object_id *task_return(task_spec *spec, int64_t ret_index) {
|
||||
CHECK(0 <= ret_index && ret_index < spec->num_returns);
|
||||
task_arg *ret = &spec->args_and_returns[spec->num_args + ret_index];
|
||||
CHECK(ret->type == ARG_BY_REF); /* No memory corruption. */
|
||||
return &ret->obj_id;
|
||||
}
|
||||
|
||||
void free_task_spec(task_spec *spec) {
|
||||
CHECK(spec->arg_index == spec->num_args); /* Task was fully constructed */
|
||||
free(spec);
|
||||
}
|
||||
|
||||
void print_task(task_spec *spec, UT_string *output) {
|
||||
/* For converting an id to hex, which has double the number
|
||||
* of bytes compared to the id (+ 1 byte for '\0'). */
|
||||
static char hex[2 * UNIQUE_ID_SIZE + 1];
|
||||
/* Print function id. */
|
||||
sha1_to_hex(&task_function(spec)->id[0], &hex[0]);
|
||||
utstring_printf(output, "fun %s ", &hex[0]);
|
||||
/* Print arguments. */
|
||||
for (int i = 0; i < task_num_args(spec); ++i) {
|
||||
sha1_to_hex(&task_arg_id(spec, i)->id[0], &hex[0]);
|
||||
utstring_printf(output, " id:%d %s", i, &hex[0]);
|
||||
}
|
||||
/* Print return ids. */
|
||||
for (int i = 0; i < task_num_returns(spec); ++i) {
|
||||
object_id *object_id = task_return(spec, i);
|
||||
sha1_to_hex(&object_id->id[0], &hex[0]);
|
||||
utstring_printf(output, " ret:%d %s", i, &hex[0]);
|
||||
}
|
||||
}
|
||||
|
||||
/* TASK INSTANCES */
|
||||
|
||||
struct task_instance_impl {
|
||||
task_iid iid;
|
||||
int32_t state;
|
||||
node_id node;
|
||||
task_spec spec;
|
||||
};
|
||||
|
||||
task_instance *make_task_instance(task_iid task_iid,
|
||||
task_spec *spec,
|
||||
int32_t state,
|
||||
node_id node) {
|
||||
int64_t size = sizeof(task_instance) - sizeof(task_spec) + task_size(spec);
|
||||
task_instance *result = malloc(size);
|
||||
memset(result, 0, size);
|
||||
result->iid = task_iid;
|
||||
result->state = state;
|
||||
result->node = node;
|
||||
memcpy(&result->spec, spec, task_size(spec));
|
||||
return result;
|
||||
}
|
||||
|
||||
int64_t task_instance_size(task_instance *instance) {
|
||||
return sizeof(task_instance) - sizeof(task_spec) + task_size(&instance->spec);
|
||||
}
|
||||
|
||||
task_iid *task_instance_id(task_instance *instance) {
|
||||
return &instance->iid;
|
||||
}
|
||||
|
||||
int32_t *task_instance_state(task_instance *instance) {
|
||||
return &instance->state;
|
||||
}
|
||||
|
||||
node_id *task_instance_node(task_instance *instance) {
|
||||
return &instance->node;
|
||||
}
|
||||
|
||||
task_spec *task_instance_task_spec(task_instance *instance) {
|
||||
return &instance->spec;
|
||||
}
|
||||
|
||||
void task_instance_free(task_instance *instance) {
|
||||
free(instance);
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
#ifndef TASK_H
|
||||
#define TASK_H
|
||||
|
||||
/* This API specifies the task data structures. It is in C so we can
|
||||
* easily construct tasks from other languages like Python. The datastructures
|
||||
* are also defined in such a way that memory is contiguous and all pointers
|
||||
* are relative, so that we can memcpy the datastructure and ship it over the
|
||||
* network without serialization and deserialization. */
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include "common.h"
|
||||
#include "utstring.h"
|
||||
|
||||
typedef unique_id function_id;
|
||||
|
||||
/* The task ID is a deterministic hash of the function ID that
|
||||
* the task executes and the argument IDs or argument values */
|
||||
typedef unique_id task_id;
|
||||
|
||||
/* The task instance ID is a globally unique ID generated which
|
||||
* identifies this particular execution of the task */
|
||||
typedef unique_id task_iid;
|
||||
|
||||
/* The node id is an identifier for the node the task is
|
||||
* scheduled on */
|
||||
typedef unique_id node_id;
|
||||
|
||||
/*
|
||||
* TASK SPECIFICATIONS: Contain all the information neccessary
|
||||
* to execute the task (function id, arguments, return object ids).
|
||||
*
|
||||
*/
|
||||
|
||||
typedef struct task_spec_impl task_spec;
|
||||
|
||||
/* If argument is passed by value or reference. */
|
||||
enum arg_type { ARG_BY_REF, ARG_BY_VAL };
|
||||
|
||||
/* Construct and modify task specifications. */
|
||||
|
||||
/* Allocating and initializing a task. */
|
||||
task_spec *alloc_task_spec(function_id function_id,
|
||||
int64_t num_args,
|
||||
int64_t num_returns,
|
||||
int64_t args_value_size);
|
||||
|
||||
/* Size of the task in bytes. */
|
||||
int64_t task_size(task_spec *spec);
|
||||
|
||||
/* Return the function ID of the task. */
|
||||
unique_id *task_function(task_spec *spec);
|
||||
|
||||
/* Getting the number of arguments and returns. */
|
||||
int64_t task_num_args(task_spec *spec);
|
||||
int64_t task_num_returns(task_spec *spec);
|
||||
|
||||
/* Getting task arguments. */
|
||||
int8_t task_arg_type(task_spec *spec, int64_t arg_index);
|
||||
unique_id *task_arg_id(task_spec *spec, int64_t arg_index);
|
||||
uint8_t *task_arg_val(task_spec *spec, int64_t arg_index);
|
||||
int64_t task_arg_length(task_spec *spec, int64_t arg_index);
|
||||
|
||||
/* Setting task arguments. Note that this API only allows you to set the
|
||||
* arguments in their order of appearance. */
|
||||
int64_t task_args_add_ref(task_spec *spec, object_id obj_id);
|
||||
int64_t task_args_add_val(task_spec *spec, uint8_t *data, int64_t length);
|
||||
|
||||
/* Getting and setting return arguments. Tasks return by reference for now. */
|
||||
unique_id *task_return(task_spec *spec, int64_t ret_index);
|
||||
|
||||
/* Freeing the task datastructure. */
|
||||
void free_task_spec(task_spec *spec);
|
||||
|
||||
/* Write the task specification to a file or socket. */
|
||||
void write_task(int fd, task_spec *spec);
|
||||
|
||||
/* Read the task specification from a file or socket. It is the user's
|
||||
* responsibility to free the task after it has been used. */
|
||||
task_spec *read_task(int fd);
|
||||
|
||||
/* Print task as a humanly readable string. */
|
||||
void print_task(task_spec *spec, UT_string *output);
|
||||
|
||||
/*
|
||||
* SCHEDULED TASK: Contains information about a scheduled task:
|
||||
* the task iid, the task specification and the task status
|
||||
* (WAITING, SCHEDULED, RUNNING, DONE) and which node the
|
||||
* task is scheduled on.
|
||||
*
|
||||
*/
|
||||
|
||||
/* The scheduling_state can be used as a flag when we are listening
|
||||
* for an event, for example TASK_WAITING | TASK_SCHEDULED. */
|
||||
enum scheduling_state {
|
||||
TASK_STATUS_WAITING = 1,
|
||||
TASK_STATUS_SCHEDULED = 2,
|
||||
TASK_STATUS_RUNNING = 4,
|
||||
TASK_STATUS_DONE = 8
|
||||
};
|
||||
|
||||
/* A task instance is one execution of a task specification.
|
||||
* It has a unique instance id, a state of execution (see scheduling_state)
|
||||
* and a node it is scheduled on or running on. */
|
||||
typedef struct task_instance_impl task_instance;
|
||||
|
||||
/* Allocate and initialize a new task instance. Must be freed with
|
||||
* scheduled_task_free after use. */
|
||||
task_instance *make_task_instance(task_iid task_iid,
|
||||
task_spec *task,
|
||||
int32_t state,
|
||||
node_id node);
|
||||
|
||||
/* Size of task instance structure in bytes. */
|
||||
int64_t task_instance_size(task_instance *instance);
|
||||
|
||||
/* Instance ID of the task instance. */
|
||||
task_iid *task_instance_id(task_instance *instance);
|
||||
|
||||
/* The scheduling state of the task instance. */
|
||||
int32_t *task_instance_state(task_instance *instance);
|
||||
|
||||
/* Node this task instance has been assigned to or is running on. */
|
||||
node_id *task_instance_node(task_instance *instance);
|
||||
|
||||
/* Task specification of this task instance. */
|
||||
task_spec *task_instance_task_spec(task_instance *instance);
|
||||
|
||||
/* Free this task instance datastructure. */
|
||||
void task_instance_free(task_instance *instance);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,24 @@
|
||||
#include "greatest.h"
|
||||
|
||||
#include "common.h"
|
||||
|
||||
SUITE(common_tests);
|
||||
|
||||
TEST sha1_test(void) {
|
||||
static char hex[2 * UNIQUE_ID_SIZE + 1];
|
||||
unique_id uid = globally_unique_id();
|
||||
sha1_to_hex(&uid.id[0], &hex[0]);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(common_tests) {
|
||||
RUN_TEST(sha1_test);
|
||||
}
|
||||
|
||||
GREATEST_MAIN_DEFS();
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
GREATEST_MAIN_BEGIN();
|
||||
RUN_SUITE(common_tests);
|
||||
GREATEST_MAIN_END();
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
#include "greatest.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/wait.h>
|
||||
|
||||
#include "event_loop.h"
|
||||
#include "test/example_task.h"
|
||||
#include "state/db.h"
|
||||
#include "state/object_table.h"
|
||||
#include "state/task_log.h"
|
||||
#include "state/redis.h"
|
||||
#include "task.h"
|
||||
|
||||
SUITE(db_tests);
|
||||
|
||||
const char *manager_addr = "127.0.0.1";
|
||||
int manager_port1 = 12345;
|
||||
int manager_port2 = 12346;
|
||||
char received_addr1[16] = {0};
|
||||
char received_port1[6] = {0};
|
||||
char received_addr2[16] = {0};
|
||||
char received_port2[6] = {0};
|
||||
|
||||
/* Test if entries have been written to the database. */
|
||||
void test_callback(object_id object_id,
|
||||
int manager_count,
|
||||
const char *manager_vector[],
|
||||
void *context) {
|
||||
CHECK(manager_count == 2);
|
||||
if (!manager_vector[0] ||
|
||||
sscanf(manager_vector[0], "%15[0-9.]:%5[0-9]", received_addr1,
|
||||
received_port1) != 2) {
|
||||
CHECK(0);
|
||||
}
|
||||
if (!manager_vector[1] ||
|
||||
sscanf(manager_vector[1], "%15[0-9.]:%5[0-9]", received_addr2,
|
||||
received_port2) != 2) {
|
||||
CHECK(0);
|
||||
}
|
||||
free(manager_vector);
|
||||
}
|
||||
|
||||
int timeout_handler(event_loop *loop, timer_id timer_id, void *context) {
|
||||
event_loop_stop(loop);
|
||||
return EVENT_LOOP_TIMER_DONE;
|
||||
}
|
||||
|
||||
TEST object_table_lookup_test(void) {
|
||||
event_loop *loop = event_loop_create();
|
||||
db_handle *db1 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
|
||||
manager_port1);
|
||||
db_handle *db2 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
|
||||
manager_port2);
|
||||
db_attach(db1, loop);
|
||||
db_attach(db2, loop);
|
||||
unique_id id = globally_unique_id();
|
||||
object_table_add(db1, id);
|
||||
object_table_add(db2, id);
|
||||
event_loop_add_timer(loop, 100, timeout_handler, NULL);
|
||||
event_loop_run(loop);
|
||||
object_table_lookup(db1, id, test_callback, NULL);
|
||||
event_loop_add_timer(loop, 100, timeout_handler, NULL);
|
||||
event_loop_run(loop);
|
||||
int port1 = atoi(received_port1);
|
||||
int port2 = atoi(received_port2);
|
||||
ASSERT_STR_EQ(&received_addr1[0], manager_addr);
|
||||
ASSERT((port1 == manager_port1 && port2 == manager_port2) ||
|
||||
(port2 == manager_port1 && port1 == manager_port2));
|
||||
|
||||
db_disconnect(db1);
|
||||
db_disconnect(db2);
|
||||
|
||||
event_loop_destroy(loop);
|
||||
PASS();
|
||||
}
|
||||
|
||||
void task_log_test_callback(task_instance *instance, void *userdata) {
|
||||
task_instance *other = userdata;
|
||||
CHECK(*task_instance_state(instance) == TASK_STATUS_SCHEDULED);
|
||||
CHECK(task_instance_size(instance) == task_instance_size(other));
|
||||
CHECK(memcmp(instance, other, task_instance_size(instance)) == 0);
|
||||
}
|
||||
|
||||
TEST task_log_test(void) {
|
||||
event_loop *loop = event_loop_create();
|
||||
db_handle *db = db_connect("127.0.0.1", 6379, "local_scheduler", "", -1);
|
||||
db_attach(db, loop);
|
||||
node_id node = globally_unique_id();
|
||||
task_spec *task = example_task();
|
||||
task_instance *instance = make_task_instance(globally_unique_id(), task,
|
||||
TASK_STATUS_SCHEDULED, node);
|
||||
task_log_register_callback(db, task_log_test_callback, node,
|
||||
TASK_STATUS_SCHEDULED, instance);
|
||||
task_log_add_task(db, instance);
|
||||
event_loop_add_timer(loop, 100, timeout_handler, NULL);
|
||||
event_loop_run(loop);
|
||||
task_instance_free(instance);
|
||||
free_task_spec(task);
|
||||
db_disconnect(db);
|
||||
event_loop_destroy(loop);
|
||||
PASS();
|
||||
}
|
||||
|
||||
int num_test_callback_called = 0;
|
||||
|
||||
void task_log_all_test_callback(task_instance *instance, void *userdata) {
|
||||
num_test_callback_called += 1;
|
||||
}
|
||||
|
||||
TEST task_log_all_test(void) {
|
||||
event_loop *loop = event_loop_create();
|
||||
db_handle *db = db_connect("127.0.0.1", 6379, "local_scheduler", "", -1);
|
||||
db_attach(db, loop);
|
||||
task_spec *task = example_task();
|
||||
/* Schedule two tasks on different nodes. */
|
||||
task_instance *instance1 = make_task_instance(
|
||||
globally_unique_id(), task, TASK_STATUS_SCHEDULED, globally_unique_id());
|
||||
task_instance *instance2 = make_task_instance(
|
||||
globally_unique_id(), task, TASK_STATUS_SCHEDULED, globally_unique_id());
|
||||
task_log_register_callback(db, task_log_all_test_callback, NIL_ID,
|
||||
TASK_STATUS_SCHEDULED, NULL);
|
||||
task_log_add_task(db, instance1);
|
||||
task_log_add_task(db, instance2);
|
||||
event_loop_add_timer(loop, 100, timeout_handler, NULL);
|
||||
event_loop_run(loop);
|
||||
task_instance_free(instance2);
|
||||
task_instance_free(instance1);
|
||||
free_task_spec(task);
|
||||
db_disconnect(db);
|
||||
event_loop_destroy(loop);
|
||||
ASSERT(num_test_callback_called == 2);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST unique_client_id_test(void) {
|
||||
const int num_conns = 50;
|
||||
|
||||
db_handle *db;
|
||||
pid_t pid = fork();
|
||||
for (int i = 0; i < num_conns; ++i) {
|
||||
db = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
|
||||
manager_port1);
|
||||
db_disconnect(db);
|
||||
}
|
||||
if (pid == 0) {
|
||||
exit(0);
|
||||
} else {
|
||||
wait(NULL);
|
||||
}
|
||||
|
||||
db = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
|
||||
manager_port1);
|
||||
ASSERT_EQ(get_client_id(db), num_conns * 2);
|
||||
db_disconnect(db);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(db_tests) {
|
||||
redisContext *context = redisConnect("127.0.0.1", 6379);
|
||||
freeReplyObject(redisCommand(context, "FLUSHALL"));
|
||||
RUN_REDIS_TEST(context, object_table_lookup_test);
|
||||
RUN_REDIS_TEST(context, task_log_test);
|
||||
RUN_REDIS_TEST(context, task_log_all_test);
|
||||
RUN_REDIS_TEST(context, unique_client_id_test);
|
||||
redisFree(context);
|
||||
}
|
||||
|
||||
GREATEST_MAIN_DEFS();
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
GREATEST_MAIN_BEGIN();
|
||||
RUN_SUITE(db_tests);
|
||||
GREATEST_MAIN_END();
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
#ifndef EXAMPLE_TASK_H
|
||||
#define EXAMPLE_TASK_H
|
||||
|
||||
#include "task.h"
|
||||
|
||||
task_spec *example_task(void) {
|
||||
function_id func_id = globally_unique_id();
|
||||
task_spec *task = alloc_task_spec(func_id, 2, 1, 0);
|
||||
task_args_add_ref(task, globally_unique_id());
|
||||
task_args_add_ref(task, globally_unique_id());
|
||||
return task;
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,106 @@
|
||||
#include "greatest.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <unistd.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
#include "io.h"
|
||||
#include "utstring.h"
|
||||
|
||||
SUITE(io_tests);
|
||||
|
||||
TEST ipc_socket_test(void) {
|
||||
const char *socket_pathname = "test-socket";
|
||||
int socket_fd = bind_ipc_sock(socket_pathname);
|
||||
ASSERT(socket_fd >= 0);
|
||||
|
||||
char *test_string = "hello world";
|
||||
char *test_bytes = "another string";
|
||||
pid_t pid = fork();
|
||||
if (pid == 0) {
|
||||
close(socket_fd);
|
||||
socket_fd = connect_ipc_sock(socket_pathname);
|
||||
ASSERT(socket_fd >= 0);
|
||||
write_log_message(socket_fd, test_string);
|
||||
write_message(socket_fd, LOG_MESSAGE, strlen(test_bytes),
|
||||
(uint8_t *) test_bytes);
|
||||
close(socket_fd);
|
||||
exit(0);
|
||||
} else {
|
||||
int client_fd = accept_client(socket_fd);
|
||||
ASSERT(client_fd >= 0);
|
||||
char *message = read_log_message(client_fd);
|
||||
ASSERT(message != NULL);
|
||||
ASSERT_STR_EQ(test_string, message);
|
||||
free(message);
|
||||
int64_t type;
|
||||
int64_t len;
|
||||
uint8_t *bytes;
|
||||
read_message(client_fd, &type, &len, &bytes);
|
||||
ASSERT(type == LOG_MESSAGE);
|
||||
ASSERT(memcmp(test_bytes, bytes, len) == 0);
|
||||
free(bytes);
|
||||
close(client_fd);
|
||||
close(socket_fd);
|
||||
unlink(socket_pathname);
|
||||
}
|
||||
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST long_ipc_socket_test(void) {
|
||||
const char *socket_pathname = "long-test-socket";
|
||||
int socket_fd = bind_ipc_sock(socket_pathname);
|
||||
ASSERT(socket_fd >= 0);
|
||||
|
||||
UT_string *test_string;
|
||||
utstring_new(test_string);
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
utstring_printf(test_string, "hello world ");
|
||||
}
|
||||
char *test_bytes = "another string";
|
||||
pid_t pid = fork();
|
||||
if (pid == 0) {
|
||||
close(socket_fd);
|
||||
socket_fd = connect_ipc_sock(socket_pathname);
|
||||
ASSERT(socket_fd >= 0);
|
||||
write_log_message(socket_fd, utstring_body(test_string));
|
||||
write_message(socket_fd, LOG_MESSAGE, strlen(test_bytes),
|
||||
(uint8_t *) test_bytes);
|
||||
close(socket_fd);
|
||||
exit(0);
|
||||
} else {
|
||||
int client_fd = accept_client(socket_fd);
|
||||
ASSERT(client_fd >= 0);
|
||||
char *message = read_log_message(client_fd);
|
||||
ASSERT(message != NULL);
|
||||
ASSERT_STR_EQ(utstring_body(test_string), message);
|
||||
free(message);
|
||||
int64_t type;
|
||||
int64_t len;
|
||||
uint8_t *bytes;
|
||||
read_message(client_fd, &type, &len, &bytes);
|
||||
ASSERT(type == LOG_MESSAGE);
|
||||
ASSERT(memcmp(test_bytes, bytes, len) == 0);
|
||||
free(bytes);
|
||||
close(client_fd);
|
||||
close(socket_fd);
|
||||
unlink(socket_pathname);
|
||||
}
|
||||
|
||||
utstring_free(test_string);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(io_tests) {
|
||||
RUN_TEST(ipc_socket_test);
|
||||
RUN_TEST(long_ipc_socket_test);
|
||||
}
|
||||
|
||||
GREATEST_MAIN_DEFS();
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
GREATEST_MAIN_BEGIN();
|
||||
RUN_SUITE(io_tests);
|
||||
GREATEST_MAIN_END();
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
#include "greatest.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "utarray.h"
|
||||
|
||||
#include "event_loop.h"
|
||||
#include "state/db.h"
|
||||
#include "state/redis.h"
|
||||
#include "io.h"
|
||||
#include "logging.h"
|
||||
|
||||
SUITE(redis_tests);
|
||||
|
||||
const char *test_set_format = "SET %s %s";
|
||||
const char *test_get_format = "GET %s";
|
||||
const char *test_key = "foo";
|
||||
const char *test_value = "bar";
|
||||
UT_array *connections = NULL;
|
||||
|
||||
int async_redis_socket_test_callback_called = 0;
|
||||
|
||||
void async_redis_socket_test_callback(redisAsyncContext *ac,
|
||||
void *r,
|
||||
void *privdata) {
|
||||
async_redis_socket_test_callback_called = 1;
|
||||
redisContext *context = redisConnect("127.0.0.1", 6379);
|
||||
redisReply *reply = redisCommand(context, test_get_format, test_key);
|
||||
redisFree(context);
|
||||
CHECK(reply != NULL);
|
||||
if (strcmp(reply->str, test_value)) {
|
||||
freeReplyObject(reply);
|
||||
CHECK(0);
|
||||
}
|
||||
freeReplyObject(reply);
|
||||
}
|
||||
|
||||
TEST redis_socket_test(void) {
|
||||
const char *socket_pathname = "redis-test-socket";
|
||||
redisContext *context = redisConnect("127.0.0.1", 6379);
|
||||
ASSERT(context != NULL);
|
||||
int socket_fd = bind_ipc_sock(socket_pathname);
|
||||
ASSERT(socket_fd >= 0);
|
||||
|
||||
int client_fd = connect_ipc_sock(socket_pathname);
|
||||
ASSERT(client_fd >= 0);
|
||||
write_formatted_log_message(client_fd, test_set_format, test_key, test_value);
|
||||
|
||||
int server_fd = accept_client(socket_fd);
|
||||
char *cmd = read_log_message(server_fd);
|
||||
close(client_fd);
|
||||
close(server_fd);
|
||||
close(socket_fd);
|
||||
unlink(socket_pathname);
|
||||
|
||||
redisReply *reply;
|
||||
reply = redisCommand(context, cmd, 0, 0);
|
||||
freeReplyObject(reply);
|
||||
reply = redisCommand(context, "GET %s", test_key);
|
||||
ASSERT(reply != NULL);
|
||||
ASSERT_STR_EQ(reply->str, test_value);
|
||||
freeReplyObject(reply);
|
||||
|
||||
free(cmd);
|
||||
redisFree(context);
|
||||
PASS();
|
||||
}
|
||||
|
||||
void redis_read_callback(event_loop *loop, int fd, void *context, int events) {
|
||||
db_handle *db = context;
|
||||
char *cmd = read_log_message(fd);
|
||||
redisAsyncCommand(db->context, async_redis_socket_test_callback, NULL, cmd,
|
||||
db->client_id, 0);
|
||||
free(cmd);
|
||||
}
|
||||
|
||||
void redis_accept_callback(event_loop *loop,
|
||||
int socket_fd,
|
||||
void *context,
|
||||
int events) {
|
||||
int accept_fd = accept_client(socket_fd);
|
||||
CHECK(accept_fd >= 0);
|
||||
utarray_push_back(connections, &accept_fd);
|
||||
event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, redis_read_callback,
|
||||
context);
|
||||
}
|
||||
|
||||
int timeout_handler(event_loop *loop, timer_id timer_id, void *context) {
|
||||
event_loop_stop(loop);
|
||||
return EVENT_LOOP_TIMER_DONE;
|
||||
}
|
||||
|
||||
TEST async_redis_socket_test(void) {
|
||||
utarray_new(connections, &ut_int_icd);
|
||||
event_loop *loop = event_loop_create();
|
||||
|
||||
/* Start IPC channel. */
|
||||
const char *socket_pathname = "async-redis-test-socket";
|
||||
int socket_fd = bind_ipc_sock(socket_pathname);
|
||||
ASSERT(socket_fd >= 0);
|
||||
utarray_push_back(connections, &socket_fd);
|
||||
|
||||
/* Start connection to Redis. */
|
||||
db_handle *db = db_connect("127.0.0.1", 6379, "", "", 0);
|
||||
db_attach(db, loop);
|
||||
|
||||
/* Send a command to the Redis process. */
|
||||
int client_fd = connect_ipc_sock(socket_pathname);
|
||||
ASSERT(client_fd >= 0);
|
||||
utarray_push_back(connections, &client_fd);
|
||||
write_formatted_log_message(client_fd, test_set_format, test_key, test_value);
|
||||
|
||||
event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, redis_read_callback,
|
||||
db);
|
||||
event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, redis_accept_callback,
|
||||
db);
|
||||
event_loop_add_timer(loop, 100, timeout_handler, NULL);
|
||||
event_loop_run(loop);
|
||||
|
||||
CHECK(async_redis_socket_test_callback_called);
|
||||
|
||||
db_disconnect(db);
|
||||
event_loop_destroy(loop);
|
||||
for (int *p = (int *) utarray_front(connections); p != NULL;
|
||||
p = (int *) utarray_next(connections, p)) {
|
||||
close(*p);
|
||||
}
|
||||
unlink(socket_pathname);
|
||||
utarray_free(connections);
|
||||
PASS();
|
||||
}
|
||||
|
||||
int logging_test_callback_called = 0;
|
||||
|
||||
void logging_test_callback(redisAsyncContext *ac, void *r, void *privdata) {
|
||||
logging_test_callback_called = 1;
|
||||
redisContext *context = redisConnect("127.0.0.1", 6379);
|
||||
redisReply *reply = redisCommand(context, "KEYS %s", "log:*");
|
||||
redisFree(context);
|
||||
CHECK(reply != NULL);
|
||||
CHECK(reply->elements > 0);
|
||||
freeReplyObject(reply);
|
||||
}
|
||||
|
||||
void logging_read_callback(event_loop *loop,
|
||||
int fd,
|
||||
void *context,
|
||||
int events) {
|
||||
db_handle *conn = context;
|
||||
char *cmd = read_log_message(fd);
|
||||
redisAsyncCommand(conn->context, logging_test_callback, NULL, cmd,
|
||||
conn->client_id, 0);
|
||||
free(cmd);
|
||||
}
|
||||
|
||||
void logging_accept_callback(event_loop *loop,
|
||||
int socket_fd,
|
||||
void *context,
|
||||
int events) {
|
||||
int accept_fd = accept_client(socket_fd);
|
||||
CHECK(accept_fd >= 0);
|
||||
utarray_push_back(connections, &accept_fd);
|
||||
event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, logging_read_callback,
|
||||
context);
|
||||
}
|
||||
|
||||
TEST logging_test(void) {
|
||||
utarray_new(connections, &ut_int_icd);
|
||||
event_loop *loop = event_loop_create();
|
||||
|
||||
/* Start IPC channel. */
|
||||
const char *socket_pathname = "logging-test-socket";
|
||||
int socket_fd = bind_ipc_sock(socket_pathname);
|
||||
ASSERT(socket_fd >= 0);
|
||||
utarray_push_back(connections, &socket_fd);
|
||||
|
||||
/* Start connection to Redis. */
|
||||
db_handle *conn = db_connect("127.0.0.1", 6379, "", "", 0);
|
||||
db_attach(conn, loop);
|
||||
|
||||
/* Send a command to the Redis process. */
|
||||
int client_fd = connect_ipc_sock(socket_pathname);
|
||||
ASSERT(client_fd >= 0);
|
||||
utarray_push_back(connections, &client_fd);
|
||||
ray_logger *logger = init_ray_logger("worker", RAY_INFO, 0, &client_fd);
|
||||
ray_log(logger, RAY_INFO, "TEST", "Message");
|
||||
|
||||
event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, logging_accept_callback,
|
||||
conn);
|
||||
event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, logging_read_callback,
|
||||
conn);
|
||||
event_loop_add_timer(loop, 100, timeout_handler, NULL);
|
||||
event_loop_run(loop);
|
||||
|
||||
CHECK(logging_test_callback_called);
|
||||
|
||||
free_ray_logger(logger);
|
||||
db_disconnect(conn);
|
||||
event_loop_destroy(loop);
|
||||
for (int *p = (int *) utarray_front(connections); p != NULL;
|
||||
p = (int *) utarray_next(connections, p)) {
|
||||
close(*p);
|
||||
}
|
||||
unlink(socket_pathname);
|
||||
utarray_free(connections);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(redis_tests) {
|
||||
redisContext *context = redisConnect("127.0.0.1", 6379);
|
||||
freeReplyObject(redisCommand(context, "FLUSHALL"));
|
||||
RUN_REDIS_TEST(context, redis_socket_test);
|
||||
RUN_REDIS_TEST(context, async_redis_socket_test);
|
||||
RUN_REDIS_TEST(context, logging_test);
|
||||
redisFree(context);
|
||||
}
|
||||
|
||||
GREATEST_MAIN_DEFS();
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
GREATEST_MAIN_BEGIN();
|
||||
RUN_SUITE(redis_tests);
|
||||
GREATEST_MAIN_END();
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
#include "greatest.h"
|
||||
|
||||
#include <unistd.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
|
||||
#include "common.h"
|
||||
#include "test/example_task.h"
|
||||
#include "task.h"
|
||||
#include "io.h"
|
||||
|
||||
SUITE(task_tests);
|
||||
|
||||
TEST task_test(void) {
|
||||
function_id func_id = globally_unique_id();
|
||||
task_spec *task = alloc_task_spec(func_id, 4, 2, 10);
|
||||
ASSERT(task_num_args(task) == 4);
|
||||
ASSERT(task_num_returns(task) == 2);
|
||||
|
||||
unique_id arg1 = globally_unique_id();
|
||||
ASSERT(task_args_add_ref(task, arg1) == 0);
|
||||
ASSERT(task_args_add_val(task, (uint8_t *) "hello", 5) == 1);
|
||||
unique_id arg2 = globally_unique_id();
|
||||
ASSERT(task_args_add_ref(task, arg2) == 2);
|
||||
ASSERT(task_args_add_val(task, (uint8_t *) "world", 5) == 3);
|
||||
|
||||
unique_id ret0 = globally_unique_id();
|
||||
unique_id ret1 = globally_unique_id();
|
||||
memcpy(task_return(task, 0), &ret0, sizeof(ret0));
|
||||
memcpy(task_return(task, 1), &ret1, sizeof(ret1));
|
||||
|
||||
ASSERT(memcmp(task_arg_id(task, 0), &arg1, sizeof(arg1)) == 0);
|
||||
ASSERT(memcmp(task_arg_val(task, 1), (uint8_t *) "hello",
|
||||
task_arg_length(task, 1)) == 0);
|
||||
ASSERT(memcmp(task_arg_id(task, 2), &arg2, sizeof(arg2)) == 0);
|
||||
ASSERT(memcmp(task_arg_val(task, 3), (uint8_t *) "world",
|
||||
task_arg_length(task, 3)) == 0);
|
||||
|
||||
ASSERT(memcmp(task_return(task, 0), &ret0, sizeof(unique_id)) == 0);
|
||||
ASSERT(memcmp(task_return(task, 1), &ret1, sizeof(unique_id)) == 0);
|
||||
|
||||
free_task_spec(task);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST send_task(void) {
|
||||
function_id func_id = globally_unique_id();
|
||||
task_spec *task = alloc_task_spec(func_id, 4, 2, 10);
|
||||
*task_return(task, 1) = globally_unique_id();
|
||||
int fd[2];
|
||||
socketpair(AF_UNIX, SOCK_STREAM, 0, fd);
|
||||
write_message(fd[0], SUBMIT_TASK, task_size(task), (uint8_t *) task);
|
||||
int64_t type;
|
||||
int64_t length;
|
||||
uint8_t *message;
|
||||
read_message(fd[1], &type, &length, &message);
|
||||
task_spec *result = (task_spec *) message;
|
||||
ASSERT(type == SUBMIT_TASK);
|
||||
ASSERT(memcmp(task, result, task_size(task)) == 0);
|
||||
ASSERT(memcmp(task, result, task_size(result)) == 0);
|
||||
free(task);
|
||||
free(result);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(task_tests) {
|
||||
RUN_TEST(task_test);
|
||||
RUN_TEST(send_task);
|
||||
}
|
||||
|
||||
GREATEST_MAIN_DEFS();
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
GREATEST_MAIN_BEGIN();
|
||||
RUN_SUITE(task_tests);
|
||||
GREATEST_MAIN_END();
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import pickle
|
||||
import unittest
|
||||
|
||||
import common
|
||||
|
||||
BASE_SIMPLE_OBJECTS = [
|
||||
0, 1, 100000, 0L, 1L, 100000L, 1L << 100, 0.0, 0.5, 0.9, 100000.1, (), [], {},
|
||||
"", 990 * "h", u"", 990 * u"h"
|
||||
]
|
||||
|
||||
LIST_SIMPLE_OBJECTS = [[obj] for obj in BASE_SIMPLE_OBJECTS]
|
||||
TUPLE_SIMPLE_OBJECTS = [(obj,) for obj in BASE_SIMPLE_OBJECTS]
|
||||
DICT_SIMPLE_OBJECTS = [{(): obj} for obj in BASE_SIMPLE_OBJECTS]
|
||||
|
||||
SIMPLE_OBJECTS = (BASE_SIMPLE_OBJECTS +
|
||||
LIST_SIMPLE_OBJECTS +
|
||||
TUPLE_SIMPLE_OBJECTS +
|
||||
DICT_SIMPLE_OBJECTS)
|
||||
|
||||
# Create some complex objects that cannot be serialized by value in tasks.
|
||||
|
||||
l = []
|
||||
l.append(l)
|
||||
|
||||
class Foo(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
BASE_COMPLEX_OBJECTS = [999 * "h", 999 * u"h", l, Foo(), 10 * [10 * [10 * [1]]]]
|
||||
|
||||
LIST_COMPLEX_OBJECTS = [[obj] for obj in BASE_COMPLEX_OBJECTS]
|
||||
TUPLE_COMPLEX_OBJECTS = [(obj,) for obj in BASE_COMPLEX_OBJECTS]
|
||||
DICT_COMPLEX_OBJECTS = [{(): obj} for obj in BASE_COMPLEX_OBJECTS]
|
||||
|
||||
COMPLEX_OBJECTS = (BASE_COMPLEX_OBJECTS +
|
||||
LIST_COMPLEX_OBJECTS +
|
||||
TUPLE_COMPLEX_OBJECTS +
|
||||
DICT_COMPLEX_OBJECTS)
|
||||
|
||||
class TestSerialization(unittest.TestCase):
|
||||
|
||||
def test_serialize_by_value(self):
|
||||
|
||||
for val in SIMPLE_OBJECTS:
|
||||
self.assertTrue(common.check_simple_value(val))
|
||||
for val in COMPLEX_OBJECTS:
|
||||
self.assertFalse(common.check_simple_value(val))
|
||||
|
||||
class TestObjectID(unittest.TestCase):
|
||||
|
||||
def test_create_object_id(self):
|
||||
object_id = common.ObjectID(20 * "a")
|
||||
|
||||
def test_cannot_pickle_object_ids(self):
|
||||
object_ids = [common.ObjectID(20 * chr(i)) for i in range(256)]
|
||||
def f():
|
||||
return object_ids
|
||||
def g(val=object_ids):
|
||||
return 1
|
||||
def h():
|
||||
x = object_ids[0]
|
||||
return 1
|
||||
# Make sure that object IDs cannot be pickled (including functions that
|
||||
# close over object IDs).
|
||||
self.assertRaises(Exception, lambda : pickling.dumps(object_ids[0]))
|
||||
self.assertRaises(Exception, lambda : pickling.dumps(object_ids))
|
||||
self.assertRaises(Exception, lambda : pickling.dumps(f))
|
||||
self.assertRaises(Exception, lambda : pickling.dumps(g))
|
||||
self.assertRaises(Exception, lambda : pickling.dumps(h))
|
||||
|
||||
class TestTask(unittest.TestCase):
|
||||
|
||||
def test_create_task(self):
|
||||
# TODO(rkn): The function ID should be a FunctionID object, not an ObjectID.
|
||||
function_id = common.ObjectID(20 * "a")
|
||||
object_ids = [common.ObjectID(20 * chr(i)) for i in range(256)]
|
||||
args_list = [
|
||||
[],
|
||||
1 * [1],
|
||||
10 * [1],
|
||||
100 * [1],
|
||||
1000 * [1],
|
||||
1 * ["a"],
|
||||
10 * ["a"],
|
||||
100 * ["a"],
|
||||
1000 * ["a"],
|
||||
[1, 1.3, 2L, 1L << 100, "hi", u"hi", [1, 2]],
|
||||
object_ids[:1],
|
||||
object_ids[:2],
|
||||
object_ids[:3],
|
||||
object_ids[:4],
|
||||
object_ids[:5],
|
||||
object_ids[:10],
|
||||
object_ids[:100],
|
||||
object_ids[:256],
|
||||
[1, object_ids[0]],
|
||||
[object_ids[0], "a"],
|
||||
[1, object_ids[0], "a"],
|
||||
[object_ids[0], 1, object_ids[1], "a"],
|
||||
object_ids[:3] + [1, "hi", 2.3] + object_ids[:5],
|
||||
object_ids + 100 * ["a"] + object_ids
|
||||
]
|
||||
for args in args_list:
|
||||
for num_return_vals in [0, 1, 2, 3, 5, 10, 100]:
|
||||
task = common.Task(function_id, args, num_return_vals)
|
||||
self.assertEqual(function_id.id(), task.function_id().id())
|
||||
retrieved_args = task.arguments()
|
||||
self.assertEqual(num_return_vals, len(task.returns()))
|
||||
self.assertEqual(len(args), len(retrieved_args))
|
||||
for i in range(len(retrieved_args)):
|
||||
if isinstance(retrieved_args[i], common.ObjectID):
|
||||
self.assertEqual(retrieved_args[i].id(), args[i].id())
|
||||
else:
|
||||
self.assertEqual(retrieved_args[i], args[i])
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Vendored
+465
@@ -0,0 +1,465 @@
|
||||
/* A simple event-driven programming library. Originally I wrote this code
|
||||
* for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
|
||||
* it in form of a library for easy reuse.
|
||||
*
|
||||
* Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <stdlib.h>
|
||||
#include <poll.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include "ae.h"
|
||||
#include "zmalloc.h"
|
||||
#include "config.h"
|
||||
|
||||
/* Include the best multiplexing layer supported by this system.
|
||||
* The following should be ordered by performances, descending. */
|
||||
#ifdef HAVE_EVPORT
|
||||
#include "ae_evport.c"
|
||||
#else
|
||||
#ifdef HAVE_EPOLL
|
||||
#include "ae_epoll.c"
|
||||
#else
|
||||
#ifdef HAVE_KQUEUE
|
||||
#include "ae_kqueue.c"
|
||||
#else
|
||||
#include "ae_select.c"
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
aeEventLoop *aeCreateEventLoop(int setsize) {
|
||||
aeEventLoop *eventLoop;
|
||||
int i;
|
||||
|
||||
if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err;
|
||||
eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
|
||||
eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
|
||||
if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
|
||||
eventLoop->setsize = setsize;
|
||||
eventLoop->lastTime = time(NULL);
|
||||
eventLoop->timeEventHead = NULL;
|
||||
eventLoop->timeEventNextId = 0;
|
||||
eventLoop->stop = 0;
|
||||
eventLoop->maxfd = -1;
|
||||
eventLoop->beforesleep = NULL;
|
||||
if (aeApiCreate(eventLoop) == -1) goto err;
|
||||
/* Events with mask == AE_NONE are not set. So let's initialize the
|
||||
* vector with it. */
|
||||
for (i = 0; i < setsize; i++)
|
||||
eventLoop->events[i].mask = AE_NONE;
|
||||
return eventLoop;
|
||||
|
||||
err:
|
||||
if (eventLoop) {
|
||||
zfree(eventLoop->events);
|
||||
zfree(eventLoop->fired);
|
||||
zfree(eventLoop);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Return the current set size. */
|
||||
int aeGetSetSize(aeEventLoop *eventLoop) {
|
||||
return eventLoop->setsize;
|
||||
}
|
||||
|
||||
/* Resize the maximum set size of the event loop.
|
||||
* If the requested set size is smaller than the current set size, but
|
||||
* there is already a file descriptor in use that is >= the requested
|
||||
* set size minus one, AE_ERR is returned and the operation is not
|
||||
* performed at all.
|
||||
*
|
||||
* Otherwise AE_OK is returned and the operation is successful. */
|
||||
int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) {
|
||||
int i;
|
||||
|
||||
if (setsize == eventLoop->setsize) return AE_OK;
|
||||
if (eventLoop->maxfd >= setsize) return AE_ERR;
|
||||
if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR;
|
||||
|
||||
eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
|
||||
eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
|
||||
eventLoop->setsize = setsize;
|
||||
|
||||
/* Make sure that if we created new slots, they are initialized with
|
||||
* an AE_NONE mask. */
|
||||
for (i = eventLoop->maxfd+1; i < setsize; i++)
|
||||
eventLoop->events[i].mask = AE_NONE;
|
||||
return AE_OK;
|
||||
}
|
||||
|
||||
void aeDeleteEventLoop(aeEventLoop *eventLoop) {
|
||||
aeApiFree(eventLoop);
|
||||
zfree(eventLoop->events);
|
||||
zfree(eventLoop->fired);
|
||||
zfree(eventLoop);
|
||||
}
|
||||
|
||||
void aeStop(aeEventLoop *eventLoop) {
|
||||
eventLoop->stop = 1;
|
||||
}
|
||||
|
||||
int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
|
||||
aeFileProc *proc, void *clientData)
|
||||
{
|
||||
if (fd >= eventLoop->setsize) {
|
||||
errno = ERANGE;
|
||||
return AE_ERR;
|
||||
}
|
||||
aeFileEvent *fe = &eventLoop->events[fd];
|
||||
|
||||
if (aeApiAddEvent(eventLoop, fd, mask) == -1)
|
||||
return AE_ERR;
|
||||
fe->mask |= mask;
|
||||
if (mask & AE_READABLE) fe->rfileProc = proc;
|
||||
if (mask & AE_WRITABLE) fe->wfileProc = proc;
|
||||
fe->clientData = clientData;
|
||||
if (fd > eventLoop->maxfd)
|
||||
eventLoop->maxfd = fd;
|
||||
return AE_OK;
|
||||
}
|
||||
|
||||
void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask)
|
||||
{
|
||||
if (fd >= eventLoop->setsize) return;
|
||||
aeFileEvent *fe = &eventLoop->events[fd];
|
||||
if (fe->mask == AE_NONE) return;
|
||||
|
||||
aeApiDelEvent(eventLoop, fd, mask);
|
||||
fe->mask = fe->mask & (~mask);
|
||||
if (fd == eventLoop->maxfd && fe->mask == AE_NONE) {
|
||||
/* Update the max fd */
|
||||
int j;
|
||||
|
||||
for (j = eventLoop->maxfd-1; j >= 0; j--)
|
||||
if (eventLoop->events[j].mask != AE_NONE) break;
|
||||
eventLoop->maxfd = j;
|
||||
}
|
||||
}
|
||||
|
||||
int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
|
||||
if (fd >= eventLoop->setsize) return 0;
|
||||
aeFileEvent *fe = &eventLoop->events[fd];
|
||||
|
||||
return fe->mask;
|
||||
}
|
||||
|
||||
static void aeGetTime(long *seconds, long *milliseconds)
|
||||
{
|
||||
struct timeval tv;
|
||||
|
||||
gettimeofday(&tv, NULL);
|
||||
*seconds = tv.tv_sec;
|
||||
*milliseconds = tv.tv_usec/1000;
|
||||
}
|
||||
|
||||
static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) {
|
||||
long cur_sec, cur_ms, when_sec, when_ms;
|
||||
|
||||
aeGetTime(&cur_sec, &cur_ms);
|
||||
when_sec = cur_sec + milliseconds/1000;
|
||||
when_ms = cur_ms + milliseconds%1000;
|
||||
if (when_ms >= 1000) {
|
||||
when_sec ++;
|
||||
when_ms -= 1000;
|
||||
}
|
||||
*sec = when_sec;
|
||||
*ms = when_ms;
|
||||
}
|
||||
|
||||
long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
|
||||
aeTimeProc *proc, void *clientData,
|
||||
aeEventFinalizerProc *finalizerProc)
|
||||
{
|
||||
long long id = eventLoop->timeEventNextId++;
|
||||
aeTimeEvent *te;
|
||||
|
||||
te = zmalloc(sizeof(*te));
|
||||
if (te == NULL) return AE_ERR;
|
||||
te->id = id;
|
||||
aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms);
|
||||
te->timeProc = proc;
|
||||
te->finalizerProc = finalizerProc;
|
||||
te->clientData = clientData;
|
||||
te->next = eventLoop->timeEventHead;
|
||||
eventLoop->timeEventHead = te;
|
||||
return id;
|
||||
}
|
||||
|
||||
int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
|
||||
{
|
||||
aeTimeEvent *te = eventLoop->timeEventHead;
|
||||
while(te) {
|
||||
if (te->id == id) {
|
||||
te->id = AE_DELETED_EVENT_ID;
|
||||
return AE_OK;
|
||||
}
|
||||
te = te->next;
|
||||
}
|
||||
return AE_ERR; /* NO event with the specified ID found */
|
||||
}
|
||||
|
||||
/* Search the first timer to fire.
|
||||
* This operation is useful to know how many time the select can be
|
||||
* put in sleep without to delay any event.
|
||||
* If there are no timers NULL is returned.
|
||||
*
|
||||
* Note that's O(N) since time events are unsorted.
|
||||
* Possible optimizations (not needed by Redis so far, but...):
|
||||
* 1) Insert the event in order, so that the nearest is just the head.
|
||||
* Much better but still insertion or deletion of timers is O(N).
|
||||
* 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)).
|
||||
*/
|
||||
static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop)
|
||||
{
|
||||
aeTimeEvent *te = eventLoop->timeEventHead;
|
||||
aeTimeEvent *nearest = NULL;
|
||||
|
||||
while(te) {
|
||||
if (!nearest || te->when_sec < nearest->when_sec ||
|
||||
(te->when_sec == nearest->when_sec &&
|
||||
te->when_ms < nearest->when_ms))
|
||||
nearest = te;
|
||||
te = te->next;
|
||||
}
|
||||
return nearest;
|
||||
}
|
||||
|
||||
/* Process time events */
|
||||
static int processTimeEvents(aeEventLoop *eventLoop) {
|
||||
int processed = 0;
|
||||
aeTimeEvent *te, *prev;
|
||||
long long maxId;
|
||||
time_t now = time(NULL);
|
||||
|
||||
/* If the system clock is moved to the future, and then set back to the
|
||||
* right value, time events may be delayed in a random way. Often this
|
||||
* means that scheduled operations will not be performed soon enough.
|
||||
*
|
||||
* Here we try to detect system clock skews, and force all the time
|
||||
* events to be processed ASAP when this happens: the idea is that
|
||||
* processing events earlier is less dangerous than delaying them
|
||||
* indefinitely, and practice suggests it is. */
|
||||
if (now < eventLoop->lastTime) {
|
||||
te = eventLoop->timeEventHead;
|
||||
while(te) {
|
||||
te->when_sec = 0;
|
||||
te = te->next;
|
||||
}
|
||||
}
|
||||
eventLoop->lastTime = now;
|
||||
|
||||
prev = NULL;
|
||||
te = eventLoop->timeEventHead;
|
||||
maxId = eventLoop->timeEventNextId-1;
|
||||
while(te) {
|
||||
long now_sec, now_ms;
|
||||
long long id;
|
||||
|
||||
/* Remove events scheduled for deletion. */
|
||||
if (te->id == AE_DELETED_EVENT_ID) {
|
||||
aeTimeEvent *next = te->next;
|
||||
if (prev == NULL)
|
||||
eventLoop->timeEventHead = te->next;
|
||||
else
|
||||
prev->next = te->next;
|
||||
if (te->finalizerProc)
|
||||
te->finalizerProc(eventLoop, te->clientData);
|
||||
zfree(te);
|
||||
te = next;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Make sure we don't process time events created by time events in
|
||||
* this iteration. Note that this check is currently useless: we always
|
||||
* add new timers on the head, however if we change the implementation
|
||||
* detail, this check may be useful again: we keep it here for future
|
||||
* defense. */
|
||||
if (te->id > maxId) {
|
||||
te = te->next;
|
||||
continue;
|
||||
}
|
||||
aeGetTime(&now_sec, &now_ms);
|
||||
if (now_sec > te->when_sec ||
|
||||
(now_sec == te->when_sec && now_ms >= te->when_ms))
|
||||
{
|
||||
int retval;
|
||||
|
||||
id = te->id;
|
||||
retval = te->timeProc(eventLoop, id, te->clientData);
|
||||
processed++;
|
||||
if (retval != AE_NOMORE) {
|
||||
aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
|
||||
} else {
|
||||
te->id = AE_DELETED_EVENT_ID;
|
||||
}
|
||||
}
|
||||
prev = te;
|
||||
te = te->next;
|
||||
}
|
||||
return processed;
|
||||
}
|
||||
|
||||
/* Process every pending time event, then every pending file event
|
||||
* (that may be registered by time event callbacks just processed).
|
||||
* Without special flags the function sleeps until some file event
|
||||
* fires, or when the next time event occurs (if any).
|
||||
*
|
||||
* If flags is 0, the function does nothing and returns.
|
||||
* if flags has AE_ALL_EVENTS set, all the kind of events are processed.
|
||||
* if flags has AE_FILE_EVENTS set, file events are processed.
|
||||
* if flags has AE_TIME_EVENTS set, time events are processed.
|
||||
* if flags has AE_DONT_WAIT set the function returns ASAP until all
|
||||
* the events that's possible to process without to wait are processed.
|
||||
*
|
||||
* The function returns the number of events processed. */
|
||||
int aeProcessEvents(aeEventLoop *eventLoop, int flags)
|
||||
{
|
||||
int processed = 0, numevents;
|
||||
|
||||
/* Nothing to do? return ASAP */
|
||||
if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0;
|
||||
|
||||
/* Note that we want call select() even if there are no
|
||||
* file events to process as long as we want to process time
|
||||
* events, in order to sleep until the next time event is ready
|
||||
* to fire. */
|
||||
if (eventLoop->maxfd != -1 ||
|
||||
((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) {
|
||||
int j;
|
||||
aeTimeEvent *shortest = NULL;
|
||||
struct timeval tv, *tvp;
|
||||
|
||||
if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT))
|
||||
shortest = aeSearchNearestTimer(eventLoop);
|
||||
if (shortest) {
|
||||
long now_sec, now_ms;
|
||||
|
||||
aeGetTime(&now_sec, &now_ms);
|
||||
tvp = &tv;
|
||||
|
||||
/* How many milliseconds we need to wait for the next
|
||||
* time event to fire? */
|
||||
long long ms =
|
||||
(shortest->when_sec - now_sec)*1000 +
|
||||
shortest->when_ms - now_ms;
|
||||
|
||||
if (ms > 0) {
|
||||
tvp->tv_sec = ms/1000;
|
||||
tvp->tv_usec = (ms % 1000)*1000;
|
||||
} else {
|
||||
tvp->tv_sec = 0;
|
||||
tvp->tv_usec = 0;
|
||||
}
|
||||
} else {
|
||||
/* If we have to check for events but need to return
|
||||
* ASAP because of AE_DONT_WAIT we need to set the timeout
|
||||
* to zero */
|
||||
if (flags & AE_DONT_WAIT) {
|
||||
tv.tv_sec = tv.tv_usec = 0;
|
||||
tvp = &tv;
|
||||
} else {
|
||||
/* Otherwise we can block */
|
||||
tvp = NULL; /* wait forever */
|
||||
}
|
||||
}
|
||||
|
||||
numevents = aeApiPoll(eventLoop, tvp);
|
||||
for (j = 0; j < numevents; j++) {
|
||||
aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd];
|
||||
int mask = eventLoop->fired[j].mask;
|
||||
int fd = eventLoop->fired[j].fd;
|
||||
int rfired = 0;
|
||||
|
||||
/* note the fe->mask & mask & ... code: maybe an already processed
|
||||
* event removed an element that fired and we still didn't
|
||||
* processed, so we check if the event is still valid. */
|
||||
if (fe->mask & mask & AE_READABLE) {
|
||||
rfired = 1;
|
||||
fe->rfileProc(eventLoop,fd,fe->clientData,mask);
|
||||
}
|
||||
if (fe->mask & mask & AE_WRITABLE) {
|
||||
if (!rfired || fe->wfileProc != fe->rfileProc)
|
||||
fe->wfileProc(eventLoop,fd,fe->clientData,mask);
|
||||
}
|
||||
processed++;
|
||||
}
|
||||
}
|
||||
/* Check time events */
|
||||
if (flags & AE_TIME_EVENTS)
|
||||
processed += processTimeEvents(eventLoop);
|
||||
|
||||
return processed; /* return the number of processed file/time events */
|
||||
}
|
||||
|
||||
/* Wait for milliseconds until the given file descriptor becomes
|
||||
* writable/readable/exception */
|
||||
int aeWait(int fd, int mask, long long milliseconds) {
|
||||
struct pollfd pfd;
|
||||
int retmask = 0, retval;
|
||||
|
||||
memset(&pfd, 0, sizeof(pfd));
|
||||
pfd.fd = fd;
|
||||
if (mask & AE_READABLE) pfd.events |= POLLIN;
|
||||
if (mask & AE_WRITABLE) pfd.events |= POLLOUT;
|
||||
|
||||
if ((retval = poll(&pfd, 1, milliseconds))== 1) {
|
||||
if (pfd.revents & POLLIN) retmask |= AE_READABLE;
|
||||
if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE;
|
||||
if (pfd.revents & POLLERR) retmask |= AE_WRITABLE;
|
||||
if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE;
|
||||
return retmask;
|
||||
} else {
|
||||
return retval;
|
||||
}
|
||||
}
|
||||
|
||||
void aeMain(aeEventLoop *eventLoop) {
|
||||
eventLoop->stop = 0;
|
||||
while (!eventLoop->stop) {
|
||||
if (eventLoop->beforesleep != NULL)
|
||||
eventLoop->beforesleep(eventLoop);
|
||||
aeProcessEvents(eventLoop, AE_ALL_EVENTS);
|
||||
}
|
||||
}
|
||||
|
||||
char *aeGetApiName(void) {
|
||||
return aeApiName();
|
||||
}
|
||||
|
||||
void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) {
|
||||
eventLoop->beforesleep = beforesleep;
|
||||
}
|
||||
Vendored
+123
@@ -0,0 +1,123 @@
|
||||
/* A simple event-driven programming library. Originally I wrote this code
|
||||
* for the Jim's event-loop (Jim is a Tcl interpreter) but later translated
|
||||
* it in form of a library for easy reuse.
|
||||
*
|
||||
* Copyright (c) 2006-2012, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __AE_H__
|
||||
#define __AE_H__
|
||||
|
||||
#include <time.h>
|
||||
|
||||
#define AE_OK 0
|
||||
#define AE_ERR -1
|
||||
|
||||
#define AE_NONE 0
|
||||
#define AE_READABLE 1
|
||||
#define AE_WRITABLE 2
|
||||
|
||||
#define AE_FILE_EVENTS 1
|
||||
#define AE_TIME_EVENTS 2
|
||||
#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
|
||||
#define AE_DONT_WAIT 4
|
||||
|
||||
#define AE_NOMORE -1
|
||||
#define AE_DELETED_EVENT_ID -1
|
||||
|
||||
/* Macros */
|
||||
#define AE_NOTUSED(V) ((void) V)
|
||||
|
||||
struct aeEventLoop;
|
||||
|
||||
/* Types and data structures */
|
||||
typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
|
||||
typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData);
|
||||
typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData);
|
||||
typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop);
|
||||
|
||||
/* File event structure */
|
||||
typedef struct aeFileEvent {
|
||||
int mask; /* one of AE_(READABLE|WRITABLE) */
|
||||
aeFileProc *rfileProc;
|
||||
aeFileProc *wfileProc;
|
||||
void *clientData;
|
||||
} aeFileEvent;
|
||||
|
||||
/* Time event structure */
|
||||
typedef struct aeTimeEvent {
|
||||
long long id; /* time event identifier. */
|
||||
long when_sec; /* seconds */
|
||||
long when_ms; /* milliseconds */
|
||||
aeTimeProc *timeProc;
|
||||
aeEventFinalizerProc *finalizerProc;
|
||||
void *clientData;
|
||||
struct aeTimeEvent *next;
|
||||
} aeTimeEvent;
|
||||
|
||||
/* A fired event */
|
||||
typedef struct aeFiredEvent {
|
||||
int fd;
|
||||
int mask;
|
||||
} aeFiredEvent;
|
||||
|
||||
/* State of an event based program */
|
||||
typedef struct aeEventLoop {
|
||||
int maxfd; /* highest file descriptor currently registered */
|
||||
int setsize; /* max number of file descriptors tracked */
|
||||
long long timeEventNextId;
|
||||
time_t lastTime; /* Used to detect system clock skew */
|
||||
aeFileEvent *events; /* Registered events */
|
||||
aeFiredEvent *fired; /* Fired events */
|
||||
aeTimeEvent *timeEventHead;
|
||||
int stop;
|
||||
void *apidata; /* This is used for polling API specific data */
|
||||
aeBeforeSleepProc *beforesleep;
|
||||
} aeEventLoop;
|
||||
|
||||
/* Prototypes */
|
||||
aeEventLoop *aeCreateEventLoop(int setsize);
|
||||
void aeDeleteEventLoop(aeEventLoop *eventLoop);
|
||||
void aeStop(aeEventLoop *eventLoop);
|
||||
int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
|
||||
aeFileProc *proc, void *clientData);
|
||||
void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask);
|
||||
int aeGetFileEvents(aeEventLoop *eventLoop, int fd);
|
||||
long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
|
||||
aeTimeProc *proc, void *clientData,
|
||||
aeEventFinalizerProc *finalizerProc);
|
||||
int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id);
|
||||
int aeProcessEvents(aeEventLoop *eventLoop, int flags);
|
||||
int aeWait(int fd, int mask, long long milliseconds);
|
||||
void aeMain(aeEventLoop *eventLoop);
|
||||
char *aeGetApiName(void);
|
||||
void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep);
|
||||
int aeGetSetSize(aeEventLoop *eventLoop);
|
||||
int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
|
||||
|
||||
#endif
|
||||
Vendored
+135
@@ -0,0 +1,135 @@
|
||||
/* Linux epoll(2) based ae.c module
|
||||
*
|
||||
* Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
|
||||
#include <sys/epoll.h>
|
||||
|
||||
typedef struct aeApiState {
|
||||
int epfd;
|
||||
struct epoll_event *events;
|
||||
} aeApiState;
|
||||
|
||||
static int aeApiCreate(aeEventLoop *eventLoop) {
|
||||
aeApiState *state = zmalloc(sizeof(aeApiState));
|
||||
|
||||
if (!state) return -1;
|
||||
state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize);
|
||||
if (!state->events) {
|
||||
zfree(state);
|
||||
return -1;
|
||||
}
|
||||
state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */
|
||||
if (state->epfd == -1) {
|
||||
zfree(state->events);
|
||||
zfree(state);
|
||||
return -1;
|
||||
}
|
||||
eventLoop->apidata = state;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
|
||||
state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void aeApiFree(aeEventLoop *eventLoop) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
|
||||
close(state->epfd);
|
||||
zfree(state->events);
|
||||
zfree(state);
|
||||
}
|
||||
|
||||
static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
struct epoll_event ee = {0}; /* avoid valgrind warning */
|
||||
/* If the fd was already monitored for some event, we need a MOD
|
||||
* operation. Otherwise we need an ADD operation. */
|
||||
int op = eventLoop->events[fd].mask == AE_NONE ?
|
||||
EPOLL_CTL_ADD : EPOLL_CTL_MOD;
|
||||
|
||||
ee.events = 0;
|
||||
mask |= eventLoop->events[fd].mask; /* Merge old events */
|
||||
if (mask & AE_READABLE) ee.events |= EPOLLIN;
|
||||
if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
|
||||
ee.data.fd = fd;
|
||||
if (epoll_ctl(state->epfd,op,fd,&ee) == -1) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
struct epoll_event ee = {0}; /* avoid valgrind warning */
|
||||
int mask = eventLoop->events[fd].mask & (~delmask);
|
||||
|
||||
ee.events = 0;
|
||||
if (mask & AE_READABLE) ee.events |= EPOLLIN;
|
||||
if (mask & AE_WRITABLE) ee.events |= EPOLLOUT;
|
||||
ee.data.fd = fd;
|
||||
if (mask != AE_NONE) {
|
||||
epoll_ctl(state->epfd,EPOLL_CTL_MOD,fd,&ee);
|
||||
} else {
|
||||
/* Note, Kernel < 2.6.9 requires a non null event pointer even for
|
||||
* EPOLL_CTL_DEL. */
|
||||
epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee);
|
||||
}
|
||||
}
|
||||
|
||||
static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
int retval, numevents = 0;
|
||||
|
||||
retval = epoll_wait(state->epfd,state->events,eventLoop->setsize,
|
||||
tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1);
|
||||
if (retval > 0) {
|
||||
int j;
|
||||
|
||||
numevents = retval;
|
||||
for (j = 0; j < numevents; j++) {
|
||||
int mask = 0;
|
||||
struct epoll_event *e = state->events+j;
|
||||
|
||||
if (e->events & EPOLLIN) mask |= AE_READABLE;
|
||||
if (e->events & EPOLLOUT) mask |= AE_WRITABLE;
|
||||
if (e->events & EPOLLERR) mask |= AE_WRITABLE;
|
||||
if (e->events & EPOLLHUP) mask |= AE_WRITABLE;
|
||||
eventLoop->fired[j].fd = e->data.fd;
|
||||
eventLoop->fired[j].mask = mask;
|
||||
}
|
||||
}
|
||||
return numevents;
|
||||
}
|
||||
|
||||
static char *aeApiName(void) {
|
||||
return "epoll";
|
||||
}
|
||||
Vendored
+320
@@ -0,0 +1,320 @@
|
||||
/* ae.c module for illumos event ports.
|
||||
*
|
||||
* Copyright (c) 2012, Joyent, Inc. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <port.h>
|
||||
#include <poll.h>
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/time.h>
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
static int evport_debug = 0;
|
||||
|
||||
/*
|
||||
* This file implements the ae API using event ports, present on Solaris-based
|
||||
* systems since Solaris 10. Using the event port interface, we associate file
|
||||
* descriptors with the port. Each association also includes the set of poll(2)
|
||||
* events that the consumer is interested in (e.g., POLLIN and POLLOUT).
|
||||
*
|
||||
* There's one tricky piece to this implementation: when we return events via
|
||||
* aeApiPoll, the corresponding file descriptors become dissociated from the
|
||||
* port. This is necessary because poll events are level-triggered, so if the
|
||||
* fd didn't become dissociated, it would immediately fire another event since
|
||||
* the underlying state hasn't changed yet. We must re-associate the file
|
||||
* descriptor, but only after we know that our caller has actually read from it.
|
||||
* The ae API does not tell us exactly when that happens, but we do know that
|
||||
* it must happen by the time aeApiPoll is called again. Our solution is to
|
||||
* keep track of the last fds returned by aeApiPoll and re-associate them next
|
||||
* time aeApiPoll is invoked.
|
||||
*
|
||||
* To summarize, in this module, each fd association is EITHER (a) represented
|
||||
* only via the in-kernel association OR (b) represented by pending_fds and
|
||||
* pending_masks. (b) is only true for the last fds we returned from aeApiPoll,
|
||||
* and only until we enter aeApiPoll again (at which point we restore the
|
||||
* in-kernel association).
|
||||
*/
|
||||
#define MAX_EVENT_BATCHSZ 512
|
||||
|
||||
typedef struct aeApiState {
|
||||
int portfd; /* event port */
|
||||
int npending; /* # of pending fds */
|
||||
int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */
|
||||
int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */
|
||||
} aeApiState;
|
||||
|
||||
static int aeApiCreate(aeEventLoop *eventLoop) {
|
||||
int i;
|
||||
aeApiState *state = zmalloc(sizeof(aeApiState));
|
||||
if (!state) return -1;
|
||||
|
||||
state->portfd = port_create();
|
||||
if (state->portfd == -1) {
|
||||
zfree(state);
|
||||
return -1;
|
||||
}
|
||||
|
||||
state->npending = 0;
|
||||
|
||||
for (i = 0; i < MAX_EVENT_BATCHSZ; i++) {
|
||||
state->pending_fds[i] = -1;
|
||||
state->pending_masks[i] = AE_NONE;
|
||||
}
|
||||
|
||||
eventLoop->apidata = state;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
|
||||
/* Nothing to resize here. */
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void aeApiFree(aeEventLoop *eventLoop) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
|
||||
close(state->portfd);
|
||||
zfree(state);
|
||||
}
|
||||
|
||||
static int aeApiLookupPending(aeApiState *state, int fd) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < state->npending; i++) {
|
||||
if (state->pending_fds[i] == fd)
|
||||
return (i);
|
||||
}
|
||||
|
||||
return (-1);
|
||||
}
|
||||
|
||||
/*
|
||||
* Helper function to invoke port_associate for the given fd and mask.
|
||||
*/
|
||||
static int aeApiAssociate(const char *where, int portfd, int fd, int mask) {
|
||||
int events = 0;
|
||||
int rv, err;
|
||||
|
||||
if (mask & AE_READABLE)
|
||||
events |= POLLIN;
|
||||
if (mask & AE_WRITABLE)
|
||||
events |= POLLOUT;
|
||||
|
||||
if (evport_debug)
|
||||
fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events);
|
||||
|
||||
rv = port_associate(portfd, PORT_SOURCE_FD, fd, events,
|
||||
(void *)(uintptr_t)mask);
|
||||
err = errno;
|
||||
|
||||
if (evport_debug)
|
||||
fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err));
|
||||
|
||||
if (rv == -1) {
|
||||
fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err));
|
||||
|
||||
if (err == EAGAIN)
|
||||
fprintf(stderr, "aeApiAssociate: event port limit exceeded.");
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
int fullmask, pfd;
|
||||
|
||||
if (evport_debug)
|
||||
fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask);
|
||||
|
||||
/*
|
||||
* Since port_associate's "events" argument replaces any existing events, we
|
||||
* must be sure to include whatever events are already associated when
|
||||
* we call port_associate() again.
|
||||
*/
|
||||
fullmask = mask | eventLoop->events[fd].mask;
|
||||
pfd = aeApiLookupPending(state, fd);
|
||||
|
||||
if (pfd != -1) {
|
||||
/*
|
||||
* This fd was recently returned from aeApiPoll. It should be safe to
|
||||
* assume that the consumer has processed that poll event, but we play
|
||||
* it safer by simply updating pending_mask. The fd will be
|
||||
* re-associated as usual when aeApiPoll is called again.
|
||||
*/
|
||||
if (evport_debug)
|
||||
fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd);
|
||||
state->pending_masks[pfd] |= fullmask;
|
||||
return 0;
|
||||
}
|
||||
|
||||
return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask));
|
||||
}
|
||||
|
||||
static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
int fullmask, pfd;
|
||||
|
||||
if (evport_debug)
|
||||
fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask);
|
||||
|
||||
pfd = aeApiLookupPending(state, fd);
|
||||
|
||||
if (pfd != -1) {
|
||||
if (evport_debug)
|
||||
fprintf(stderr, "deleting event from pending fd %d\n", fd);
|
||||
|
||||
/*
|
||||
* This fd was just returned from aeApiPoll, so it's not currently
|
||||
* associated with the port. All we need to do is update
|
||||
* pending_mask appropriately.
|
||||
*/
|
||||
state->pending_masks[pfd] &= ~mask;
|
||||
|
||||
if (state->pending_masks[pfd] == AE_NONE)
|
||||
state->pending_fds[pfd] = -1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* The fd is currently associated with the port. Like with the add case
|
||||
* above, we must look at the full mask for the file descriptor before
|
||||
* updating that association. We don't have a good way of knowing what the
|
||||
* events are without looking into the eventLoop state directly. We rely on
|
||||
* the fact that our caller has already updated the mask in the eventLoop.
|
||||
*/
|
||||
|
||||
fullmask = eventLoop->events[fd].mask;
|
||||
if (fullmask == AE_NONE) {
|
||||
/*
|
||||
* We're removing *all* events, so use port_dissociate to remove the
|
||||
* association completely. Failure here indicates a bug.
|
||||
*/
|
||||
if (evport_debug)
|
||||
fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd);
|
||||
|
||||
if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) {
|
||||
perror("aeApiDelEvent: port_dissociate");
|
||||
abort(); /* will not return */
|
||||
}
|
||||
} else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd,
|
||||
fullmask) != 0) {
|
||||
/*
|
||||
* ENOMEM is a potentially transient condition, but the kernel won't
|
||||
* generally return it unless things are really bad. EAGAIN indicates
|
||||
* we've reached an resource limit, for which it doesn't make sense to
|
||||
* retry (counter-intuitively). All other errors indicate a bug. In any
|
||||
* of these cases, the best we can do is to abort.
|
||||
*/
|
||||
abort(); /* will not return */
|
||||
}
|
||||
}
|
||||
|
||||
static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
struct timespec timeout, *tsp;
|
||||
int mask, i;
|
||||
uint_t nevents;
|
||||
port_event_t event[MAX_EVENT_BATCHSZ];
|
||||
|
||||
/*
|
||||
* If we've returned fd events before, we must re-associate them with the
|
||||
* port now, before calling port_get(). See the block comment at the top of
|
||||
* this file for an explanation of why.
|
||||
*/
|
||||
for (i = 0; i < state->npending; i++) {
|
||||
if (state->pending_fds[i] == -1)
|
||||
/* This fd has since been deleted. */
|
||||
continue;
|
||||
|
||||
if (aeApiAssociate("aeApiPoll", state->portfd,
|
||||
state->pending_fds[i], state->pending_masks[i]) != 0) {
|
||||
/* See aeApiDelEvent for why this case is fatal. */
|
||||
abort();
|
||||
}
|
||||
|
||||
state->pending_masks[i] = AE_NONE;
|
||||
state->pending_fds[i] = -1;
|
||||
}
|
||||
|
||||
state->npending = 0;
|
||||
|
||||
if (tvp != NULL) {
|
||||
timeout.tv_sec = tvp->tv_sec;
|
||||
timeout.tv_nsec = tvp->tv_usec * 1000;
|
||||
tsp = &timeout;
|
||||
} else {
|
||||
tsp = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* port_getn can return with errno == ETIME having returned some events (!).
|
||||
* So if we get ETIME, we check nevents, too.
|
||||
*/
|
||||
nevents = 1;
|
||||
if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents,
|
||||
tsp) == -1 && (errno != ETIME || nevents == 0)) {
|
||||
if (errno == ETIME || errno == EINTR)
|
||||
return 0;
|
||||
|
||||
/* Any other error indicates a bug. */
|
||||
perror("aeApiPoll: port_get");
|
||||
abort();
|
||||
}
|
||||
|
||||
state->npending = nevents;
|
||||
|
||||
for (i = 0; i < nevents; i++) {
|
||||
mask = 0;
|
||||
if (event[i].portev_events & POLLIN)
|
||||
mask |= AE_READABLE;
|
||||
if (event[i].portev_events & POLLOUT)
|
||||
mask |= AE_WRITABLE;
|
||||
|
||||
eventLoop->fired[i].fd = event[i].portev_object;
|
||||
eventLoop->fired[i].mask = mask;
|
||||
|
||||
if (evport_debug)
|
||||
fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n",
|
||||
(int)event[i].portev_object, mask);
|
||||
|
||||
state->pending_fds[i] = event[i].portev_object;
|
||||
state->pending_masks[i] = (uintptr_t)event[i].portev_user;
|
||||
}
|
||||
|
||||
return nevents;
|
||||
}
|
||||
|
||||
static char *aeApiName(void) {
|
||||
return "evport";
|
||||
}
|
||||
Vendored
+138
@@ -0,0 +1,138 @@
|
||||
/* Kqueue(2)-based ae.c module
|
||||
*
|
||||
* Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/event.h>
|
||||
#include <sys/time.h>
|
||||
|
||||
typedef struct aeApiState {
|
||||
int kqfd;
|
||||
struct kevent *events;
|
||||
} aeApiState;
|
||||
|
||||
static int aeApiCreate(aeEventLoop *eventLoop) {
|
||||
aeApiState *state = zmalloc(sizeof(aeApiState));
|
||||
|
||||
if (!state) return -1;
|
||||
state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize);
|
||||
if (!state->events) {
|
||||
zfree(state);
|
||||
return -1;
|
||||
}
|
||||
state->kqfd = kqueue();
|
||||
if (state->kqfd == -1) {
|
||||
zfree(state->events);
|
||||
zfree(state);
|
||||
return -1;
|
||||
}
|
||||
eventLoop->apidata = state;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
|
||||
state->events = zrealloc(state->events, sizeof(struct kevent)*setsize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void aeApiFree(aeEventLoop *eventLoop) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
|
||||
close(state->kqfd);
|
||||
zfree(state->events);
|
||||
zfree(state);
|
||||
}
|
||||
|
||||
static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
struct kevent ke;
|
||||
|
||||
if (mask & AE_READABLE) {
|
||||
EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
|
||||
if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
|
||||
}
|
||||
if (mask & AE_WRITABLE) {
|
||||
EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL);
|
||||
if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
struct kevent ke;
|
||||
|
||||
if (mask & AE_READABLE) {
|
||||
EV_SET(&ke, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
|
||||
kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
|
||||
}
|
||||
if (mask & AE_WRITABLE) {
|
||||
EV_SET(&ke, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
|
||||
kevent(state->kqfd, &ke, 1, NULL, 0, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
int retval, numevents = 0;
|
||||
|
||||
if (tvp != NULL) {
|
||||
struct timespec timeout;
|
||||
timeout.tv_sec = tvp->tv_sec;
|
||||
timeout.tv_nsec = tvp->tv_usec * 1000;
|
||||
retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
|
||||
&timeout);
|
||||
} else {
|
||||
retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize,
|
||||
NULL);
|
||||
}
|
||||
|
||||
if (retval > 0) {
|
||||
int j;
|
||||
|
||||
numevents = retval;
|
||||
for(j = 0; j < numevents; j++) {
|
||||
int mask = 0;
|
||||
struct kevent *e = state->events+j;
|
||||
|
||||
if (e->filter == EVFILT_READ) mask |= AE_READABLE;
|
||||
if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
|
||||
eventLoop->fired[j].fd = e->ident;
|
||||
eventLoop->fired[j].mask = mask;
|
||||
}
|
||||
}
|
||||
return numevents;
|
||||
}
|
||||
|
||||
static char *aeApiName(void) {
|
||||
return "kqueue";
|
||||
}
|
||||
Vendored
+106
@@ -0,0 +1,106 @@
|
||||
/* Select()-based ae.c module.
|
||||
*
|
||||
* Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
|
||||
#include <sys/select.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef struct aeApiState {
|
||||
fd_set rfds, wfds;
|
||||
/* We need to have a copy of the fd sets as it's not safe to reuse
|
||||
* FD sets after select(). */
|
||||
fd_set _rfds, _wfds;
|
||||
} aeApiState;
|
||||
|
||||
static int aeApiCreate(aeEventLoop *eventLoop) {
|
||||
aeApiState *state = zmalloc(sizeof(aeApiState));
|
||||
|
||||
if (!state) return -1;
|
||||
FD_ZERO(&state->rfds);
|
||||
FD_ZERO(&state->wfds);
|
||||
eventLoop->apidata = state;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
|
||||
/* Just ensure we have enough room in the fd_set type. */
|
||||
if (setsize >= FD_SETSIZE) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void aeApiFree(aeEventLoop *eventLoop) {
|
||||
zfree(eventLoop->apidata);
|
||||
}
|
||||
|
||||
static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
|
||||
if (mask & AE_READABLE) FD_SET(fd,&state->rfds);
|
||||
if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
|
||||
if (mask & AE_READABLE) FD_CLR(fd,&state->rfds);
|
||||
if (mask & AE_WRITABLE) FD_CLR(fd,&state->wfds);
|
||||
}
|
||||
|
||||
static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
|
||||
aeApiState *state = eventLoop->apidata;
|
||||
int retval, j, numevents = 0;
|
||||
|
||||
memcpy(&state->_rfds,&state->rfds,sizeof(fd_set));
|
||||
memcpy(&state->_wfds,&state->wfds,sizeof(fd_set));
|
||||
|
||||
retval = select(eventLoop->maxfd+1,
|
||||
&state->_rfds,&state->_wfds,NULL,tvp);
|
||||
if (retval > 0) {
|
||||
for (j = 0; j <= eventLoop->maxfd; j++) {
|
||||
int mask = 0;
|
||||
aeFileEvent *fe = &eventLoop->events[j];
|
||||
|
||||
if (fe->mask == AE_NONE) continue;
|
||||
if (fe->mask & AE_READABLE && FD_ISSET(j,&state->_rfds))
|
||||
mask |= AE_READABLE;
|
||||
if (fe->mask & AE_WRITABLE && FD_ISSET(j,&state->_wfds))
|
||||
mask |= AE_WRITABLE;
|
||||
eventLoop->fired[numevents].fd = j;
|
||||
eventLoop->fired[numevents].mask = mask;
|
||||
numevents++;
|
||||
}
|
||||
}
|
||||
return numevents;
|
||||
}
|
||||
|
||||
static char *aeApiName(void) {
|
||||
return "select";
|
||||
}
|
||||
Vendored
+54
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __CONFIG_H
|
||||
#define __CONFIG_H
|
||||
|
||||
#ifdef __APPLE__
|
||||
#include <AvailabilityMacros.h>
|
||||
#endif
|
||||
|
||||
/* Test for polling API */
|
||||
#ifdef __linux__
|
||||
#define HAVE_EPOLL 1
|
||||
#endif
|
||||
|
||||
#if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__)
|
||||
#define HAVE_KQUEUE 1
|
||||
#endif
|
||||
|
||||
#ifdef __sun
|
||||
#include <sys/feature_tests.h>
|
||||
#ifdef _DTRACE_VERSION
|
||||
#define HAVE_EVPORT 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
#endif
|
||||
Vendored
+16
@@ -0,0 +1,16 @@
|
||||
#ifndef _ZMALLOC_H
|
||||
#define _ZMALLOC_H
|
||||
|
||||
#ifndef zmalloc
|
||||
#define zmalloc malloc
|
||||
#endif
|
||||
|
||||
#ifndef zfree
|
||||
#define zfree free
|
||||
#endif
|
||||
|
||||
#ifndef zrealloc
|
||||
#define zrealloc realloc
|
||||
#endif
|
||||
|
||||
#endif /* _ZMALLOC_H */
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
if [ ! -f redis-3.2.3/src/redis-server ]; then
|
||||
wget http://download.redis.io/releases/redis-3.2.3.tar.gz
|
||||
tar xvfz redis-3.2.3.tar.gz
|
||||
cd redis-3.2.3
|
||||
make
|
||||
fi
|
||||
Vendored
+1023
File diff suppressed because it is too large
Load Diff
+1
Submodule src/common/thirdparty/hiredis added at 5f98e1d35d
Vendored
+238
@@ -0,0 +1,238 @@
|
||||
/*
|
||||
Copyright (c) 2008-2016, Troy D. Hanson http://troydhanson.github.com/uthash/
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
|
||||
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
||||
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
|
||||
OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/* a dynamic array implementation using macros
|
||||
*/
|
||||
#ifndef UTARRAY_H
|
||||
#define UTARRAY_H
|
||||
|
||||
#define UTARRAY_VERSION 2.0.1
|
||||
|
||||
#ifdef __GNUC__
|
||||
#define _UNUSED_ __attribute__ ((__unused__))
|
||||
#else
|
||||
#define _UNUSED_
|
||||
#endif
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <string.h> /* memset, etc */
|
||||
#include <stdlib.h> /* exit */
|
||||
|
||||
#ifndef oom
|
||||
#define oom() exit(-1)
|
||||
#endif
|
||||
|
||||
typedef void (ctor_f)(void *dst, const void *src);
|
||||
typedef void (dtor_f)(void *elt);
|
||||
typedef void (init_f)(void *elt);
|
||||
typedef struct {
|
||||
size_t sz;
|
||||
init_f *init;
|
||||
ctor_f *copy;
|
||||
dtor_f *dtor;
|
||||
} UT_icd;
|
||||
|
||||
typedef struct {
|
||||
unsigned i,n;/* i: index of next available slot, n: num slots */
|
||||
UT_icd icd; /* initializer, copy and destructor functions */
|
||||
char *d; /* n slots of size icd->sz*/
|
||||
} UT_array;
|
||||
|
||||
#define utarray_init(a,_icd) do { \
|
||||
memset(a,0,sizeof(UT_array)); \
|
||||
(a)->icd = *(_icd); \
|
||||
} while(0)
|
||||
|
||||
#define utarray_done(a) do { \
|
||||
if ((a)->n) { \
|
||||
if ((a)->icd.dtor) { \
|
||||
unsigned _ut_i; \
|
||||
for(_ut_i=0; _ut_i < (a)->i; _ut_i++) { \
|
||||
(a)->icd.dtor(utarray_eltptr(a,_ut_i)); \
|
||||
} \
|
||||
} \
|
||||
free((a)->d); \
|
||||
} \
|
||||
(a)->n=0; \
|
||||
} while(0)
|
||||
|
||||
#define utarray_new(a,_icd) do { \
|
||||
(a) = (UT_array*)malloc(sizeof(UT_array)); \
|
||||
if ((a) == NULL) oom(); \
|
||||
utarray_init(a,_icd); \
|
||||
} while(0)
|
||||
|
||||
#define utarray_free(a) do { \
|
||||
utarray_done(a); \
|
||||
free(a); \
|
||||
} while(0)
|
||||
|
||||
#define utarray_reserve(a,by) do { \
|
||||
if (((a)->i+(by)) > (a)->n) { \
|
||||
char *utarray_tmp; \
|
||||
while (((a)->i+(by)) > (a)->n) { (a)->n = ((a)->n ? (2*(a)->n) : 8); } \
|
||||
utarray_tmp=(char*)realloc((a)->d, (a)->n*(a)->icd.sz); \
|
||||
if (utarray_tmp == NULL) oom(); \
|
||||
(a)->d=utarray_tmp; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define utarray_push_back(a,p) do { \
|
||||
utarray_reserve(a,1); \
|
||||
if ((a)->icd.copy) { (a)->icd.copy( _utarray_eltptr(a,(a)->i++), p); } \
|
||||
else { memcpy(_utarray_eltptr(a,(a)->i++), p, (a)->icd.sz); }; \
|
||||
} while(0)
|
||||
|
||||
#define utarray_pop_back(a) do { \
|
||||
if ((a)->icd.dtor) { (a)->icd.dtor( _utarray_eltptr(a,--((a)->i))); } \
|
||||
else { (a)->i--; } \
|
||||
} while(0)
|
||||
|
||||
#define utarray_extend_back(a) do { \
|
||||
utarray_reserve(a,1); \
|
||||
if ((a)->icd.init) { (a)->icd.init(_utarray_eltptr(a,(a)->i)); } \
|
||||
else { memset(_utarray_eltptr(a,(a)->i),0,(a)->icd.sz); } \
|
||||
(a)->i++; \
|
||||
} while(0)
|
||||
|
||||
#define utarray_len(a) ((a)->i)
|
||||
|
||||
#define utarray_eltptr(a,j) (((j) < (a)->i) ? _utarray_eltptr(a,j) : NULL)
|
||||
#define _utarray_eltptr(a,j) ((a)->d + ((a)->icd.sz * (j)))
|
||||
|
||||
#define utarray_insert(a,p,j) do { \
|
||||
if ((j) > (a)->i) utarray_resize(a,j); \
|
||||
utarray_reserve(a,1); \
|
||||
if ((j) < (a)->i) { \
|
||||
memmove( _utarray_eltptr(a,(j)+1), _utarray_eltptr(a,j), \
|
||||
((a)->i - (j))*((a)->icd.sz)); \
|
||||
} \
|
||||
if ((a)->icd.copy) { (a)->icd.copy( _utarray_eltptr(a,j), p); } \
|
||||
else { memcpy(_utarray_eltptr(a,j), p, (a)->icd.sz); }; \
|
||||
(a)->i++; \
|
||||
} while(0)
|
||||
|
||||
#define utarray_inserta(a,w,j) do { \
|
||||
if (utarray_len(w) == 0) break; \
|
||||
if ((j) > (a)->i) utarray_resize(a,j); \
|
||||
utarray_reserve(a,utarray_len(w)); \
|
||||
if ((j) < (a)->i) { \
|
||||
memmove(_utarray_eltptr(a,(j)+utarray_len(w)), \
|
||||
_utarray_eltptr(a,j), \
|
||||
((a)->i - (j))*((a)->icd.sz)); \
|
||||
} \
|
||||
if ((a)->icd.copy) { \
|
||||
unsigned _ut_i; \
|
||||
for(_ut_i=0;_ut_i<(w)->i;_ut_i++) { \
|
||||
(a)->icd.copy(_utarray_eltptr(a, (j) + _ut_i), _utarray_eltptr(w, _ut_i)); \
|
||||
} \
|
||||
} else { \
|
||||
memcpy(_utarray_eltptr(a,j), _utarray_eltptr(w,0), \
|
||||
utarray_len(w)*((a)->icd.sz)); \
|
||||
} \
|
||||
(a)->i += utarray_len(w); \
|
||||
} while(0)
|
||||
|
||||
#define utarray_resize(dst,num) do { \
|
||||
unsigned _ut_i; \
|
||||
if ((dst)->i > (unsigned)(num)) { \
|
||||
if ((dst)->icd.dtor) { \
|
||||
for (_ut_i = (num); _ut_i < (dst)->i; ++_ut_i) { \
|
||||
(dst)->icd.dtor(_utarray_eltptr(dst, _ut_i)); \
|
||||
} \
|
||||
} \
|
||||
} else if ((dst)->i < (unsigned)(num)) { \
|
||||
utarray_reserve(dst, (num) - (dst)->i); \
|
||||
if ((dst)->icd.init) { \
|
||||
for (_ut_i = (dst)->i; _ut_i < (unsigned)(num); ++_ut_i) { \
|
||||
(dst)->icd.init(_utarray_eltptr(dst, _ut_i)); \
|
||||
} \
|
||||
} else { \
|
||||
memset(_utarray_eltptr(dst, (dst)->i), 0, (dst)->icd.sz*((num) - (dst)->i)); \
|
||||
} \
|
||||
} \
|
||||
(dst)->i = (num); \
|
||||
} while(0)
|
||||
|
||||
#define utarray_concat(dst,src) do { \
|
||||
utarray_inserta(dst, src, utarray_len(dst)); \
|
||||
} while(0)
|
||||
|
||||
#define utarray_erase(a,pos,len) do { \
|
||||
if ((a)->icd.dtor) { \
|
||||
unsigned _ut_i; \
|
||||
for (_ut_i = 0; _ut_i < (len); _ut_i++) { \
|
||||
(a)->icd.dtor(utarray_eltptr(a, (pos) + _ut_i)); \
|
||||
} \
|
||||
} \
|
||||
if ((a)->i > ((pos) + (len))) { \
|
||||
memmove(_utarray_eltptr(a, pos), _utarray_eltptr(a, (pos) + (len)), \
|
||||
((a)->i - ((pos) + (len))) * (a)->icd.sz); \
|
||||
} \
|
||||
(a)->i -= (len); \
|
||||
} while(0)
|
||||
|
||||
#define utarray_renew(a,u) do { \
|
||||
if (a) utarray_clear(a); \
|
||||
else utarray_new(a, u); \
|
||||
} while(0)
|
||||
|
||||
#define utarray_clear(a) do { \
|
||||
if ((a)->i > 0) { \
|
||||
if ((a)->icd.dtor) { \
|
||||
unsigned _ut_i; \
|
||||
for(_ut_i=0; _ut_i < (a)->i; _ut_i++) { \
|
||||
(a)->icd.dtor(_utarray_eltptr(a, _ut_i)); \
|
||||
} \
|
||||
} \
|
||||
(a)->i = 0; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define utarray_sort(a,cmp) do { \
|
||||
qsort((a)->d, (a)->i, (a)->icd.sz, cmp); \
|
||||
} while(0)
|
||||
|
||||
#define utarray_find(a,v,cmp) bsearch((v),(a)->d,(a)->i,(a)->icd.sz,cmp)
|
||||
|
||||
#define utarray_front(a) (((a)->i) ? (_utarray_eltptr(a,0)) : NULL)
|
||||
#define utarray_next(a,e) (((e)==NULL) ? utarray_front(a) : ((((a)->i) > (utarray_eltidx(a,e)+1)) ? _utarray_eltptr(a,utarray_eltidx(a,e)+1) : NULL))
|
||||
#define utarray_prev(a,e) (((e)==NULL) ? utarray_back(a) : ((utarray_eltidx(a,e) > 0) ? _utarray_eltptr(a,utarray_eltidx(a,e)-1) : NULL))
|
||||
#define utarray_back(a) (((a)->i) ? (_utarray_eltptr(a,(a)->i-1)) : NULL)
|
||||
#define utarray_eltidx(a,e) (((char*)(e) >= (a)->d) ? (((char*)(e) - (a)->d)/(a)->icd.sz) : -1)
|
||||
|
||||
/* last we pre-define a few icd for common utarrays of ints and strings */
|
||||
static void utarray_str_cpy(void *dst, const void *src) {
|
||||
char **_src = (char**)src, **_dst = (char**)dst;
|
||||
*_dst = (*_src == NULL) ? NULL : strdup(*_src);
|
||||
}
|
||||
static void utarray_str_dtor(void *elt) {
|
||||
char **eltc = (char**)elt;
|
||||
if (*eltc != NULL) free(*eltc);
|
||||
}
|
||||
static const UT_icd ut_str_icd _UNUSED_ = {sizeof(char*),NULL,utarray_str_cpy,utarray_str_dtor};
|
||||
static const UT_icd ut_int_icd _UNUSED_ = {sizeof(int),NULL,NULL,NULL};
|
||||
static const UT_icd ut_ptr_icd _UNUSED_ = {sizeof(void*),NULL,NULL,NULL};
|
||||
|
||||
|
||||
#endif /* UTARRAY_H */
|
||||
Vendored
+1074
File diff suppressed because it is too large
Load Diff
Vendored
+895
@@ -0,0 +1,895 @@
|
||||
/*
|
||||
Copyright (c) 2007-2016, Troy D. Hanson http://troydhanson.github.com/uthash/
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
|
||||
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
||||
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
|
||||
OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef UTLIST_H
|
||||
#define UTLIST_H
|
||||
|
||||
#define UTLIST_VERSION 2.0.1
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
/*
|
||||
* This file contains macros to manipulate singly and doubly-linked lists.
|
||||
*
|
||||
* 1. LL_ macros: singly-linked lists.
|
||||
* 2. DL_ macros: doubly-linked lists.
|
||||
* 3. CDL_ macros: circular doubly-linked lists.
|
||||
*
|
||||
* To use singly-linked lists, your structure must have a "next" pointer.
|
||||
* To use doubly-linked lists, your structure must "prev" and "next" pointers.
|
||||
* Either way, the pointer to the head of the list must be initialized to NULL.
|
||||
*
|
||||
* ----------------.EXAMPLE -------------------------
|
||||
* struct item {
|
||||
* int id;
|
||||
* struct item *prev, *next;
|
||||
* }
|
||||
*
|
||||
* struct item *list = NULL:
|
||||
*
|
||||
* int main() {
|
||||
* struct item *item;
|
||||
* ... allocate and populate item ...
|
||||
* DL_APPEND(list, item);
|
||||
* }
|
||||
* --------------------------------------------------
|
||||
*
|
||||
* For doubly-linked lists, the append and delete macros are O(1)
|
||||
* For singly-linked lists, append and delete are O(n) but prepend is O(1)
|
||||
* The sort macro is O(n log(n)) for all types of single/double/circular lists.
|
||||
*/
|
||||
|
||||
/* These macros use decltype or the earlier __typeof GNU extension.
|
||||
As decltype is only available in newer compilers (VS2010 or gcc 4.3+
|
||||
when compiling c++ code), this code uses whatever method is needed
|
||||
or, for VS2008 where neither is available, uses casting workarounds. */
|
||||
#ifdef _MSC_VER /* MS compiler */
|
||||
#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */
|
||||
#define LDECLTYPE(x) decltype(x)
|
||||
#else /* VS2008 or older (or VS2010 in C mode) */
|
||||
#define NO_DECLTYPE
|
||||
#endif
|
||||
#elif defined(__ICCARM__)
|
||||
#define NO_DECLTYPE
|
||||
#else /* GNU, Sun and other compilers */
|
||||
#define LDECLTYPE(x) __typeof(x)
|
||||
#endif
|
||||
|
||||
/* for VS2008 we use some workarounds to get around the lack of decltype,
|
||||
* namely, we always reassign our tmp variable to the list head if we need
|
||||
* to dereference its prev/next pointers, and save/restore the real head.*/
|
||||
#ifdef NO_DECLTYPE
|
||||
#define IF_NO_DECLTYPE(x) x
|
||||
#define LDECLTYPE(x) char*
|
||||
#define _SV(elt,list) _tmp = (char*)(list); {char **_alias = (char**)&(list); *_alias = (elt); }
|
||||
#define _NEXT(elt,list,next) ((char*)((list)->next))
|
||||
#define _NEXTASGN(elt,list,to,next) { char **_alias = (char**)&((list)->next); *_alias=(char*)(to); }
|
||||
/* #define _PREV(elt,list,prev) ((char*)((list)->prev)) */
|
||||
#define _PREVASGN(elt,list,to,prev) { char **_alias = (char**)&((list)->prev); *_alias=(char*)(to); }
|
||||
#define _RS(list) { char **_alias = (char**)&(list); *_alias=_tmp; }
|
||||
#define _CASTASGN(a,b) { char **_alias = (char**)&(a); *_alias=(char*)(b); }
|
||||
#else
|
||||
#define IF_NO_DECLTYPE(x)
|
||||
#define _SV(elt,list)
|
||||
#define _NEXT(elt,list,next) ((elt)->next)
|
||||
#define _NEXTASGN(elt,list,to,next) ((elt)->next)=(to)
|
||||
/* #define _PREV(elt,list,prev) ((elt)->prev) */
|
||||
#define _PREVASGN(elt,list,to,prev) ((elt)->prev)=(to)
|
||||
#define _RS(list)
|
||||
#define _CASTASGN(a,b) (a)=(b)
|
||||
#endif
|
||||
|
||||
/******************************************************************************
|
||||
* The sort macro is an adaptation of Simon Tatham's O(n log(n)) mergesort *
|
||||
* Unwieldy variable names used here to avoid shadowing passed-in variables. *
|
||||
*****************************************************************************/
|
||||
#define LL_SORT(list, cmp) \
|
||||
LL_SORT2(list, cmp, next)
|
||||
|
||||
#define LL_SORT2(list, cmp, next) \
|
||||
do { \
|
||||
LDECLTYPE(list) _ls_p; \
|
||||
LDECLTYPE(list) _ls_q; \
|
||||
LDECLTYPE(list) _ls_e; \
|
||||
LDECLTYPE(list) _ls_tail; \
|
||||
IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \
|
||||
int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \
|
||||
if (list) { \
|
||||
_ls_insize = 1; \
|
||||
_ls_looping = 1; \
|
||||
while (_ls_looping) { \
|
||||
_CASTASGN(_ls_p,list); \
|
||||
(list) = NULL; \
|
||||
_ls_tail = NULL; \
|
||||
_ls_nmerges = 0; \
|
||||
while (_ls_p) { \
|
||||
_ls_nmerges++; \
|
||||
_ls_q = _ls_p; \
|
||||
_ls_psize = 0; \
|
||||
for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \
|
||||
_ls_psize++; \
|
||||
_SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \
|
||||
if (!_ls_q) break; \
|
||||
} \
|
||||
_ls_qsize = _ls_insize; \
|
||||
while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \
|
||||
if (_ls_psize == 0) { \
|
||||
_ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
|
||||
_NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
|
||||
} else if (_ls_qsize == 0 || !_ls_q) { \
|
||||
_ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
|
||||
_NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
|
||||
} else if (cmp(_ls_p,_ls_q) <= 0) { \
|
||||
_ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
|
||||
_NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
|
||||
} else { \
|
||||
_ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
|
||||
_NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
|
||||
} \
|
||||
if (_ls_tail) { \
|
||||
_SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \
|
||||
} else { \
|
||||
_CASTASGN(list,_ls_e); \
|
||||
} \
|
||||
_ls_tail = _ls_e; \
|
||||
} \
|
||||
_ls_p = _ls_q; \
|
||||
} \
|
||||
if (_ls_tail) { \
|
||||
_SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \
|
||||
} \
|
||||
if (_ls_nmerges <= 1) { \
|
||||
_ls_looping=0; \
|
||||
} \
|
||||
_ls_insize *= 2; \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
|
||||
#define DL_SORT(list, cmp) \
|
||||
DL_SORT2(list, cmp, prev, next)
|
||||
|
||||
#define DL_SORT2(list, cmp, prev, next) \
|
||||
do { \
|
||||
LDECLTYPE(list) _ls_p; \
|
||||
LDECLTYPE(list) _ls_q; \
|
||||
LDECLTYPE(list) _ls_e; \
|
||||
LDECLTYPE(list) _ls_tail; \
|
||||
IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \
|
||||
int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \
|
||||
if (list) { \
|
||||
_ls_insize = 1; \
|
||||
_ls_looping = 1; \
|
||||
while (_ls_looping) { \
|
||||
_CASTASGN(_ls_p,list); \
|
||||
(list) = NULL; \
|
||||
_ls_tail = NULL; \
|
||||
_ls_nmerges = 0; \
|
||||
while (_ls_p) { \
|
||||
_ls_nmerges++; \
|
||||
_ls_q = _ls_p; \
|
||||
_ls_psize = 0; \
|
||||
for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \
|
||||
_ls_psize++; \
|
||||
_SV(_ls_q,list); _ls_q = _NEXT(_ls_q,list,next); _RS(list); \
|
||||
if (!_ls_q) break; \
|
||||
} \
|
||||
_ls_qsize = _ls_insize; \
|
||||
while ((_ls_psize > 0) || ((_ls_qsize > 0) && _ls_q)) { \
|
||||
if (_ls_psize == 0) { \
|
||||
_ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
|
||||
_NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
|
||||
} else if ((_ls_qsize == 0) || (!_ls_q)) { \
|
||||
_ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
|
||||
_NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
|
||||
} else if (cmp(_ls_p,_ls_q) <= 0) { \
|
||||
_ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
|
||||
_NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
|
||||
} else { \
|
||||
_ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
|
||||
_NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
|
||||
} \
|
||||
if (_ls_tail) { \
|
||||
_SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \
|
||||
} else { \
|
||||
_CASTASGN(list,_ls_e); \
|
||||
} \
|
||||
_SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \
|
||||
_ls_tail = _ls_e; \
|
||||
} \
|
||||
_ls_p = _ls_q; \
|
||||
} \
|
||||
_CASTASGN((list)->prev, _ls_tail); \
|
||||
_SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,NULL,next); _RS(list); \
|
||||
if (_ls_nmerges <= 1) { \
|
||||
_ls_looping=0; \
|
||||
} \
|
||||
_ls_insize *= 2; \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CDL_SORT(list, cmp) \
|
||||
CDL_SORT2(list, cmp, prev, next)
|
||||
|
||||
#define CDL_SORT2(list, cmp, prev, next) \
|
||||
do { \
|
||||
LDECLTYPE(list) _ls_p; \
|
||||
LDECLTYPE(list) _ls_q; \
|
||||
LDECLTYPE(list) _ls_e; \
|
||||
LDECLTYPE(list) _ls_tail; \
|
||||
LDECLTYPE(list) _ls_oldhead; \
|
||||
LDECLTYPE(list) _tmp; \
|
||||
int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \
|
||||
if (list) { \
|
||||
_ls_insize = 1; \
|
||||
_ls_looping = 1; \
|
||||
while (_ls_looping) { \
|
||||
_CASTASGN(_ls_p,list); \
|
||||
_CASTASGN(_ls_oldhead,list); \
|
||||
(list) = NULL; \
|
||||
_ls_tail = NULL; \
|
||||
_ls_nmerges = 0; \
|
||||
while (_ls_p) { \
|
||||
_ls_nmerges++; \
|
||||
_ls_q = _ls_p; \
|
||||
_ls_psize = 0; \
|
||||
for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \
|
||||
_ls_psize++; \
|
||||
_SV(_ls_q,list); \
|
||||
if (_NEXT(_ls_q,list,next) == _ls_oldhead) { \
|
||||
_ls_q = NULL; \
|
||||
} else { \
|
||||
_ls_q = _NEXT(_ls_q,list,next); \
|
||||
} \
|
||||
_RS(list); \
|
||||
if (!_ls_q) break; \
|
||||
} \
|
||||
_ls_qsize = _ls_insize; \
|
||||
while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \
|
||||
if (_ls_psize == 0) { \
|
||||
_ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
|
||||
_NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
|
||||
if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \
|
||||
} else if (_ls_qsize == 0 || !_ls_q) { \
|
||||
_ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
|
||||
_NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
|
||||
if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \
|
||||
} else if (cmp(_ls_p,_ls_q) <= 0) { \
|
||||
_ls_e = _ls_p; _SV(_ls_p,list); _ls_p = \
|
||||
_NEXT(_ls_p,list,next); _RS(list); _ls_psize--; \
|
||||
if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \
|
||||
} else { \
|
||||
_ls_e = _ls_q; _SV(_ls_q,list); _ls_q = \
|
||||
_NEXT(_ls_q,list,next); _RS(list); _ls_qsize--; \
|
||||
if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \
|
||||
} \
|
||||
if (_ls_tail) { \
|
||||
_SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_ls_e,next); _RS(list); \
|
||||
} else { \
|
||||
_CASTASGN(list,_ls_e); \
|
||||
} \
|
||||
_SV(_ls_e,list); _PREVASGN(_ls_e,list,_ls_tail,prev); _RS(list); \
|
||||
_ls_tail = _ls_e; \
|
||||
} \
|
||||
_ls_p = _ls_q; \
|
||||
} \
|
||||
_CASTASGN((list)->prev,_ls_tail); \
|
||||
_CASTASGN(_tmp,list); \
|
||||
_SV(_ls_tail,list); _NEXTASGN(_ls_tail,list,_tmp,next); _RS(list); \
|
||||
if (_ls_nmerges <= 1) { \
|
||||
_ls_looping=0; \
|
||||
} \
|
||||
_ls_insize *= 2; \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/******************************************************************************
|
||||
* singly linked list macros (non-circular) *
|
||||
*****************************************************************************/
|
||||
#define LL_PREPEND(head,add) \
|
||||
LL_PREPEND2(head,add,next)
|
||||
|
||||
#define LL_PREPEND2(head,add,next) \
|
||||
do { \
|
||||
(add)->next = (head); \
|
||||
(head) = (add); \
|
||||
} while (0)
|
||||
|
||||
#define LL_CONCAT(head1,head2) \
|
||||
LL_CONCAT2(head1,head2,next)
|
||||
|
||||
#define LL_CONCAT2(head1,head2,next) \
|
||||
do { \
|
||||
LDECLTYPE(head1) _tmp; \
|
||||
if (head1) { \
|
||||
_tmp = (head1); \
|
||||
while (_tmp->next) { _tmp = _tmp->next; } \
|
||||
_tmp->next=(head2); \
|
||||
} else { \
|
||||
(head1)=(head2); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define LL_APPEND(head,add) \
|
||||
LL_APPEND2(head,add,next)
|
||||
|
||||
#define LL_APPEND2(head,add,next) \
|
||||
do { \
|
||||
LDECLTYPE(head) _tmp; \
|
||||
(add)->next=NULL; \
|
||||
if (head) { \
|
||||
_tmp = (head); \
|
||||
while (_tmp->next) { _tmp = _tmp->next; } \
|
||||
_tmp->next=(add); \
|
||||
} else { \
|
||||
(head)=(add); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define LL_DELETE(head,del) \
|
||||
LL_DELETE2(head,del,next)
|
||||
|
||||
#define LL_DELETE2(head,del,next) \
|
||||
do { \
|
||||
LDECLTYPE(head) _tmp; \
|
||||
if ((head) == (del)) { \
|
||||
(head)=(head)->next; \
|
||||
} else { \
|
||||
_tmp = (head); \
|
||||
while (_tmp->next && (_tmp->next != (del))) { \
|
||||
_tmp = _tmp->next; \
|
||||
} \
|
||||
if (_tmp->next) { \
|
||||
_tmp->next = (del)->next; \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define LL_COUNT(head,el,counter) \
|
||||
LL_COUNT2(head,el,counter,next) \
|
||||
|
||||
#define LL_COUNT2(head,el,counter,next) \
|
||||
do { \
|
||||
(counter) = 0; \
|
||||
LL_FOREACH2(head,el,next) { ++(counter); } \
|
||||
} while (0)
|
||||
|
||||
#define LL_FOREACH(head,el) \
|
||||
LL_FOREACH2(head,el,next)
|
||||
|
||||
#define LL_FOREACH2(head,el,next) \
|
||||
for ((el) = (head); el; (el) = (el)->next)
|
||||
|
||||
#define LL_FOREACH_SAFE(head,el,tmp) \
|
||||
LL_FOREACH_SAFE2(head,el,tmp,next)
|
||||
|
||||
#define LL_FOREACH_SAFE2(head,el,tmp,next) \
|
||||
for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp))
|
||||
|
||||
#define LL_SEARCH_SCALAR(head,out,field,val) \
|
||||
LL_SEARCH_SCALAR2(head,out,field,val,next)
|
||||
|
||||
#define LL_SEARCH_SCALAR2(head,out,field,val,next) \
|
||||
do { \
|
||||
LL_FOREACH2(head,out,next) { \
|
||||
if ((out)->field == (val)) break; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define LL_SEARCH(head,out,elt,cmp) \
|
||||
LL_SEARCH2(head,out,elt,cmp,next)
|
||||
|
||||
#define LL_SEARCH2(head,out,elt,cmp,next) \
|
||||
do { \
|
||||
LL_FOREACH2(head,out,next) { \
|
||||
if ((cmp(out,elt))==0) break; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define LL_REPLACE_ELEM2(head, el, add, next) \
|
||||
do { \
|
||||
LDECLTYPE(head) _tmp; \
|
||||
assert((head) != NULL); \
|
||||
assert((el) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
(add)->next = (el)->next; \
|
||||
if ((head) == (el)) { \
|
||||
(head) = (add); \
|
||||
} else { \
|
||||
_tmp = (head); \
|
||||
while (_tmp->next && (_tmp->next != (el))) { \
|
||||
_tmp = _tmp->next; \
|
||||
} \
|
||||
if (_tmp->next) { \
|
||||
_tmp->next = (add); \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define LL_REPLACE_ELEM(head, el, add) \
|
||||
LL_REPLACE_ELEM2(head, el, add, next)
|
||||
|
||||
#define LL_PREPEND_ELEM2(head, el, add, next) \
|
||||
do { \
|
||||
if (el) { \
|
||||
LDECLTYPE(head) _tmp; \
|
||||
assert((head) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
(add)->next = (el); \
|
||||
if ((head) == (el)) { \
|
||||
(head) = (add); \
|
||||
} else { \
|
||||
_tmp = (head); \
|
||||
while (_tmp->next && (_tmp->next != (el))) { \
|
||||
_tmp = _tmp->next; \
|
||||
} \
|
||||
if (_tmp->next) { \
|
||||
_tmp->next = (add); \
|
||||
} \
|
||||
} \
|
||||
} else { \
|
||||
LL_APPEND2(head, add, next); \
|
||||
} \
|
||||
} while (0) \
|
||||
|
||||
#define LL_PREPEND_ELEM(head, el, add) \
|
||||
LL_PREPEND_ELEM2(head, el, add, next)
|
||||
|
||||
#define LL_APPEND_ELEM2(head, el, add, next) \
|
||||
do { \
|
||||
if (el) { \
|
||||
assert((head) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
(add)->next = (el)->next; \
|
||||
(el)->next = (add); \
|
||||
} else { \
|
||||
LL_PREPEND2(head, add, next); \
|
||||
} \
|
||||
} while (0) \
|
||||
|
||||
#define LL_APPEND_ELEM(head, el, add) \
|
||||
LL_APPEND_ELEM2(head, el, add, next)
|
||||
|
||||
#ifdef NO_DECLTYPE
|
||||
/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */
|
||||
|
||||
#undef LL_CONCAT2
|
||||
#define LL_CONCAT2(head1,head2,next) \
|
||||
do { \
|
||||
char *_tmp; \
|
||||
if (head1) { \
|
||||
_tmp = (char*)(head1); \
|
||||
while ((head1)->next) { (head1) = (head1)->next; } \
|
||||
(head1)->next = (head2); \
|
||||
_RS(head1); \
|
||||
} else { \
|
||||
(head1)=(head2); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#undef LL_APPEND2
|
||||
#define LL_APPEND2(head,add,next) \
|
||||
do { \
|
||||
if (head) { \
|
||||
(add)->next = head; /* use add->next as a temp variable */ \
|
||||
while ((add)->next->next) { (add)->next = (add)->next->next; } \
|
||||
(add)->next->next=(add); \
|
||||
} else { \
|
||||
(head)=(add); \
|
||||
} \
|
||||
(add)->next=NULL; \
|
||||
} while (0)
|
||||
|
||||
#undef LL_DELETE2
|
||||
#define LL_DELETE2(head,del,next) \
|
||||
do { \
|
||||
if ((head) == (del)) { \
|
||||
(head)=(head)->next; \
|
||||
} else { \
|
||||
char *_tmp = (char*)(head); \
|
||||
while ((head)->next && ((head)->next != (del))) { \
|
||||
(head) = (head)->next; \
|
||||
} \
|
||||
if ((head)->next) { \
|
||||
(head)->next = ((del)->next); \
|
||||
} \
|
||||
_RS(head); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#undef LL_REPLACE_ELEM2
|
||||
#define LL_REPLACE_ELEM2(head, el, add, next) \
|
||||
do { \
|
||||
assert((head) != NULL); \
|
||||
assert((el) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
if ((head) == (el)) { \
|
||||
(head) = (add); \
|
||||
} else { \
|
||||
(add)->next = head; \
|
||||
while ((add)->next->next && ((add)->next->next != (el))) { \
|
||||
(add)->next = (add)->next->next; \
|
||||
} \
|
||||
if ((add)->next->next) { \
|
||||
(add)->next->next = (add); \
|
||||
} \
|
||||
} \
|
||||
(add)->next = (el)->next; \
|
||||
} while (0)
|
||||
|
||||
#undef LL_PREPEND_ELEM2
|
||||
#define LL_PREPEND_ELEM2(head, el, add, next) \
|
||||
do { \
|
||||
if (el) { \
|
||||
assert((head) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
if ((head) == (el)) { \
|
||||
(head) = (add); \
|
||||
} else { \
|
||||
(add)->next = (head); \
|
||||
while ((add)->next->next && ((add)->next->next != (el))) { \
|
||||
(add)->next = (add)->next->next; \
|
||||
} \
|
||||
if ((add)->next->next) { \
|
||||
(add)->next->next = (add); \
|
||||
} \
|
||||
} \
|
||||
(add)->next = (el); \
|
||||
} else { \
|
||||
LL_APPEND2(head, add, next); \
|
||||
} \
|
||||
} while (0) \
|
||||
|
||||
#endif /* NO_DECLTYPE */
|
||||
|
||||
/******************************************************************************
|
||||
* doubly linked list macros (non-circular) *
|
||||
*****************************************************************************/
|
||||
#define DL_PREPEND(head,add) \
|
||||
DL_PREPEND2(head,add,prev,next)
|
||||
|
||||
#define DL_PREPEND2(head,add,prev,next) \
|
||||
do { \
|
||||
(add)->next = (head); \
|
||||
if (head) { \
|
||||
(add)->prev = (head)->prev; \
|
||||
(head)->prev = (add); \
|
||||
} else { \
|
||||
(add)->prev = (add); \
|
||||
} \
|
||||
(head) = (add); \
|
||||
} while (0)
|
||||
|
||||
#define DL_APPEND(head,add) \
|
||||
DL_APPEND2(head,add,prev,next)
|
||||
|
||||
#define DL_APPEND2(head,add,prev,next) \
|
||||
do { \
|
||||
if (head) { \
|
||||
(add)->prev = (head)->prev; \
|
||||
(head)->prev->next = (add); \
|
||||
(head)->prev = (add); \
|
||||
(add)->next = NULL; \
|
||||
} else { \
|
||||
(head)=(add); \
|
||||
(head)->prev = (head); \
|
||||
(head)->next = NULL; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define DL_CONCAT(head1,head2) \
|
||||
DL_CONCAT2(head1,head2,prev,next)
|
||||
|
||||
#define DL_CONCAT2(head1,head2,prev,next) \
|
||||
do { \
|
||||
LDECLTYPE(head1) _tmp; \
|
||||
if (head2) { \
|
||||
if (head1) { \
|
||||
_CASTASGN(_tmp, (head2)->prev); \
|
||||
(head2)->prev = (head1)->prev; \
|
||||
(head1)->prev->next = (head2); \
|
||||
_CASTASGN((head1)->prev, _tmp); \
|
||||
} else { \
|
||||
(head1)=(head2); \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define DL_DELETE(head,del) \
|
||||
DL_DELETE2(head,del,prev,next)
|
||||
|
||||
#define DL_DELETE2(head,del,prev,next) \
|
||||
do { \
|
||||
assert((del)->prev != NULL); \
|
||||
if ((del)->prev == (del)) { \
|
||||
(head)=NULL; \
|
||||
} else if ((del)==(head)) { \
|
||||
(del)->next->prev = (del)->prev; \
|
||||
(head) = (del)->next; \
|
||||
} else { \
|
||||
(del)->prev->next = (del)->next; \
|
||||
if ((del)->next) { \
|
||||
(del)->next->prev = (del)->prev; \
|
||||
} else { \
|
||||
(head)->prev = (del)->prev; \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define DL_COUNT(head,el,counter) \
|
||||
DL_COUNT2(head,el,counter,next) \
|
||||
|
||||
#define DL_COUNT2(head,el,counter,next) \
|
||||
do { \
|
||||
(counter) = 0; \
|
||||
DL_FOREACH2(head,el,next) { ++(counter); } \
|
||||
} while (0)
|
||||
|
||||
#define DL_FOREACH(head,el) \
|
||||
DL_FOREACH2(head,el,next)
|
||||
|
||||
#define DL_FOREACH2(head,el,next) \
|
||||
for ((el) = (head); el; (el) = (el)->next)
|
||||
|
||||
/* this version is safe for deleting the elements during iteration */
|
||||
#define DL_FOREACH_SAFE(head,el,tmp) \
|
||||
DL_FOREACH_SAFE2(head,el,tmp,next)
|
||||
|
||||
#define DL_FOREACH_SAFE2(head,el,tmp,next) \
|
||||
for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp))
|
||||
|
||||
/* these are identical to their singly-linked list counterparts */
|
||||
#define DL_SEARCH_SCALAR LL_SEARCH_SCALAR
|
||||
#define DL_SEARCH LL_SEARCH
|
||||
#define DL_SEARCH_SCALAR2 LL_SEARCH_SCALAR2
|
||||
#define DL_SEARCH2 LL_SEARCH2
|
||||
|
||||
#define DL_REPLACE_ELEM2(head, el, add, prev, next) \
|
||||
do { \
|
||||
assert((head) != NULL); \
|
||||
assert((el) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
if ((head) == (el)) { \
|
||||
(head) = (add); \
|
||||
(add)->next = (el)->next; \
|
||||
if ((el)->next == NULL) { \
|
||||
(add)->prev = (add); \
|
||||
} else { \
|
||||
(add)->prev = (el)->prev; \
|
||||
(add)->next->prev = (add); \
|
||||
} \
|
||||
} else { \
|
||||
(add)->next = (el)->next; \
|
||||
(add)->prev = (el)->prev; \
|
||||
(add)->prev->next = (add); \
|
||||
if ((el)->next == NULL) { \
|
||||
(head)->prev = (add); \
|
||||
} else { \
|
||||
(add)->next->prev = (add); \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define DL_REPLACE_ELEM(head, el, add) \
|
||||
DL_REPLACE_ELEM2(head, el, add, prev, next)
|
||||
|
||||
#define DL_PREPEND_ELEM2(head, el, add, prev, next) \
|
||||
do { \
|
||||
if (el) { \
|
||||
assert((head) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
(add)->next = (el); \
|
||||
(add)->prev = (el)->prev; \
|
||||
(el)->prev = (add); \
|
||||
if ((head) == (el)) { \
|
||||
(head) = (add); \
|
||||
} else { \
|
||||
(add)->prev->next = (add); \
|
||||
} \
|
||||
} else { \
|
||||
DL_APPEND2(head, add, prev, next); \
|
||||
} \
|
||||
} while (0) \
|
||||
|
||||
#define DL_PREPEND_ELEM(head, el, add) \
|
||||
DL_PREPEND_ELEM2(head, el, add, prev, next)
|
||||
|
||||
#define DL_APPEND_ELEM2(head, el, add, prev, next) \
|
||||
do { \
|
||||
if (el) { \
|
||||
assert((head) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
(add)->next = (el)->next; \
|
||||
(add)->prev = (el); \
|
||||
(el)->next = (add); \
|
||||
if ((add)->next) { \
|
||||
(add)->next->prev = (add); \
|
||||
} else { \
|
||||
(head)->prev = (add); \
|
||||
} \
|
||||
} else { \
|
||||
DL_PREPEND2(head, add, prev, next); \
|
||||
} \
|
||||
} while (0) \
|
||||
|
||||
#define DL_APPEND_ELEM(head, el, add) \
|
||||
DL_APPEND_ELEM2(head, el, add, prev, next)
|
||||
|
||||
/******************************************************************************
|
||||
* circular doubly linked list macros *
|
||||
*****************************************************************************/
|
||||
#define CDL_APPEND(head,add) \
|
||||
CDL_APPEND2(head,add,prev,next)
|
||||
|
||||
#define CDL_APPEND2(head,add,prev,next) \
|
||||
do { \
|
||||
if (head) { \
|
||||
(add)->prev = (head)->prev; \
|
||||
(add)->next = (head); \
|
||||
(head)->prev = (add); \
|
||||
(add)->prev->next = (add); \
|
||||
} else { \
|
||||
(add)->prev = (add); \
|
||||
(add)->next = (add); \
|
||||
(head) = (add); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CDL_PREPEND(head,add) \
|
||||
CDL_PREPEND2(head,add,prev,next)
|
||||
|
||||
#define CDL_PREPEND2(head,add,prev,next) \
|
||||
do { \
|
||||
if (head) { \
|
||||
(add)->prev = (head)->prev; \
|
||||
(add)->next = (head); \
|
||||
(head)->prev = (add); \
|
||||
(add)->prev->next = (add); \
|
||||
} else { \
|
||||
(add)->prev = (add); \
|
||||
(add)->next = (add); \
|
||||
} \
|
||||
(head) = (add); \
|
||||
} while (0)
|
||||
|
||||
#define CDL_DELETE(head,del) \
|
||||
CDL_DELETE2(head,del,prev,next)
|
||||
|
||||
#define CDL_DELETE2(head,del,prev,next) \
|
||||
do { \
|
||||
if (((head)==(del)) && ((head)->next == (head))) { \
|
||||
(head) = NULL; \
|
||||
} else { \
|
||||
(del)->next->prev = (del)->prev; \
|
||||
(del)->prev->next = (del)->next; \
|
||||
if ((del) == (head)) (head)=(del)->next; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CDL_COUNT(head,el,counter) \
|
||||
CDL_COUNT2(head,el,counter,next) \
|
||||
|
||||
#define CDL_COUNT2(head, el, counter,next) \
|
||||
do { \
|
||||
(counter) = 0; \
|
||||
CDL_FOREACH2(head,el,next) { ++(counter); } \
|
||||
} while (0)
|
||||
|
||||
#define CDL_FOREACH(head,el) \
|
||||
CDL_FOREACH2(head,el,next)
|
||||
|
||||
#define CDL_FOREACH2(head,el,next) \
|
||||
for ((el)=(head);el;(el)=(((el)->next==(head)) ? NULL : (el)->next))
|
||||
|
||||
#define CDL_FOREACH_SAFE(head,el,tmp1,tmp2) \
|
||||
CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next)
|
||||
|
||||
#define CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next) \
|
||||
for ((el) = (head), (tmp1) = (head) ? (head)->prev : NULL; \
|
||||
(el) && ((tmp2) = (el)->next, 1); \
|
||||
(el) = ((el) == (tmp1) ? NULL : (tmp2)))
|
||||
|
||||
#define CDL_SEARCH_SCALAR(head,out,field,val) \
|
||||
CDL_SEARCH_SCALAR2(head,out,field,val,next)
|
||||
|
||||
#define CDL_SEARCH_SCALAR2(head,out,field,val,next) \
|
||||
do { \
|
||||
CDL_FOREACH2(head,out,next) { \
|
||||
if ((out)->field == (val)) break; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CDL_SEARCH(head,out,elt,cmp) \
|
||||
CDL_SEARCH2(head,out,elt,cmp,next)
|
||||
|
||||
#define CDL_SEARCH2(head,out,elt,cmp,next) \
|
||||
do { \
|
||||
CDL_FOREACH2(head,out,next) { \
|
||||
if ((cmp(out,elt))==0) break; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CDL_REPLACE_ELEM2(head, el, add, prev, next) \
|
||||
do { \
|
||||
assert((head) != NULL); \
|
||||
assert((el) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
if ((el)->next == (el)) { \
|
||||
(add)->next = (add); \
|
||||
(add)->prev = (add); \
|
||||
(head) = (add); \
|
||||
} else { \
|
||||
(add)->next = (el)->next; \
|
||||
(add)->prev = (el)->prev; \
|
||||
(add)->next->prev = (add); \
|
||||
(add)->prev->next = (add); \
|
||||
if ((head) == (el)) { \
|
||||
(head) = (add); \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CDL_REPLACE_ELEM(head, el, add) \
|
||||
CDL_REPLACE_ELEM2(head, el, add, prev, next)
|
||||
|
||||
#define CDL_PREPEND_ELEM2(head, el, add, prev, next) \
|
||||
do { \
|
||||
if (el) { \
|
||||
assert((head) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
(add)->next = (el); \
|
||||
(add)->prev = (el)->prev; \
|
||||
(el)->prev = (add); \
|
||||
(add)->prev->next = (add); \
|
||||
if ((head) == (el)) { \
|
||||
(head) = (add); \
|
||||
} \
|
||||
} else { \
|
||||
CDL_APPEND2(head, add, prev, next); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CDL_PREPEND_ELEM(head, el, add) \
|
||||
CDL_PREPEND_ELEM2(head, el, add, prev, next)
|
||||
|
||||
#define CDL_APPEND_ELEM2(head, el, add, prev, next) \
|
||||
do { \
|
||||
if (el) { \
|
||||
assert((head) != NULL); \
|
||||
assert((add) != NULL); \
|
||||
(add)->next = (el)->next; \
|
||||
(add)->prev = (el); \
|
||||
(el)->next = (add); \
|
||||
(add)->next->prev = (add); \
|
||||
} else { \
|
||||
CDL_PREPEND2(head, add, prev, next); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define CDL_APPEND_ELEM(head, el, add) \
|
||||
CDL_APPEND_ELEM2(head, el, add, prev, next)
|
||||
|
||||
#endif /* UTLIST_H */
|
||||
Vendored
+398
@@ -0,0 +1,398 @@
|
||||
/*
|
||||
Copyright (c) 2008-2016, Troy D. Hanson http://troydhanson.github.com/uthash/
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
|
||||
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
||||
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
|
||||
OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/* a dynamic string implementation using macros
|
||||
*/
|
||||
#ifndef UTSTRING_H
|
||||
#define UTSTRING_H
|
||||
|
||||
#define UTSTRING_VERSION 2.0.1
|
||||
|
||||
#ifdef __GNUC__
|
||||
#define _UNUSED_ __attribute__ ((__unused__))
|
||||
#else
|
||||
#define _UNUSED_
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdarg.h>
|
||||
|
||||
#ifndef oom
|
||||
#define oom() exit(-1)
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
char *d;
|
||||
size_t n; /* allocd size */
|
||||
size_t i; /* index of first unused byte */
|
||||
} UT_string;
|
||||
|
||||
#define utstring_reserve(s,amt) \
|
||||
do { \
|
||||
if (((s)->n - (s)->i) < (size_t)(amt)) { \
|
||||
char *utstring_tmp = (char*)realloc( \
|
||||
(s)->d, (s)->n + (amt)); \
|
||||
if (utstring_tmp == NULL) oom(); \
|
||||
(s)->d = utstring_tmp; \
|
||||
(s)->n += (amt); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define utstring_init(s) \
|
||||
do { \
|
||||
(s)->n = 0; (s)->i = 0; (s)->d = NULL; \
|
||||
utstring_reserve(s,100); \
|
||||
(s)->d[0] = '\0'; \
|
||||
} while(0)
|
||||
|
||||
#define utstring_done(s) \
|
||||
do { \
|
||||
if ((s)->d != NULL) free((s)->d); \
|
||||
(s)->n = 0; \
|
||||
} while(0)
|
||||
|
||||
#define utstring_free(s) \
|
||||
do { \
|
||||
utstring_done(s); \
|
||||
free(s); \
|
||||
} while(0)
|
||||
|
||||
#define utstring_new(s) \
|
||||
do { \
|
||||
s = (UT_string*)calloc(sizeof(UT_string),1); \
|
||||
if (!s) oom(); \
|
||||
utstring_init(s); \
|
||||
} while(0)
|
||||
|
||||
#define utstring_renew(s) \
|
||||
do { \
|
||||
if (s) { \
|
||||
utstring_clear(s); \
|
||||
} else { \
|
||||
utstring_new(s); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define utstring_clear(s) \
|
||||
do { \
|
||||
(s)->i = 0; \
|
||||
(s)->d[0] = '\0'; \
|
||||
} while(0)
|
||||
|
||||
#define utstring_bincpy(s,b,l) \
|
||||
do { \
|
||||
utstring_reserve((s),(l)+1); \
|
||||
if (l) memcpy(&(s)->d[(s)->i], b, l); \
|
||||
(s)->i += (l); \
|
||||
(s)->d[(s)->i]='\0'; \
|
||||
} while(0)
|
||||
|
||||
#define utstring_concat(dst,src) \
|
||||
do { \
|
||||
utstring_reserve((dst),((src)->i)+1); \
|
||||
if ((src)->i) memcpy(&(dst)->d[(dst)->i], (src)->d, (src)->i); \
|
||||
(dst)->i += (src)->i; \
|
||||
(dst)->d[(dst)->i]='\0'; \
|
||||
} while(0)
|
||||
|
||||
#define utstring_len(s) ((unsigned)((s)->i))
|
||||
|
||||
#define utstring_body(s) ((s)->d)
|
||||
|
||||
_UNUSED_ static void utstring_printf_va(UT_string *s, const char *fmt, va_list ap) {
|
||||
int n;
|
||||
va_list cp;
|
||||
for (;;) {
|
||||
#ifdef _WIN32
|
||||
cp = ap;
|
||||
#else
|
||||
va_copy(cp, ap);
|
||||
#endif
|
||||
n = vsnprintf (&s->d[s->i], s->n-s->i, fmt, cp);
|
||||
va_end(cp);
|
||||
|
||||
if ((n > -1) && ((size_t) n < (s->n-s->i))) {
|
||||
s->i += n;
|
||||
return;
|
||||
}
|
||||
|
||||
/* Else try again with more space. */
|
||||
if (n > -1) utstring_reserve(s,n+1); /* exact */
|
||||
else utstring_reserve(s,(s->n)*2); /* 2x */
|
||||
}
|
||||
}
|
||||
#ifdef __GNUC__
|
||||
/* support printf format checking (2=the format string, 3=start of varargs) */
|
||||
static void utstring_printf(UT_string *s, const char *fmt, ...)
|
||||
__attribute__ (( format( printf, 2, 3) ));
|
||||
#endif
|
||||
_UNUSED_ static void utstring_printf(UT_string *s, const char *fmt, ...) {
|
||||
va_list ap;
|
||||
va_start(ap,fmt);
|
||||
utstring_printf_va(s,fmt,ap);
|
||||
va_end(ap);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
* begin substring search functions *
|
||||
******************************************************************************/
|
||||
/* Build KMP table from left to right. */
|
||||
_UNUSED_ static void _utstring_BuildTable(
|
||||
const char *P_Needle,
|
||||
size_t P_NeedleLen,
|
||||
long *P_KMP_Table)
|
||||
{
|
||||
long i, j;
|
||||
|
||||
i = 0;
|
||||
j = i - 1;
|
||||
P_KMP_Table[i] = j;
|
||||
while (i < (long) P_NeedleLen)
|
||||
{
|
||||
while ( (j > -1) && (P_Needle[i] != P_Needle[j]) )
|
||||
{
|
||||
j = P_KMP_Table[j];
|
||||
}
|
||||
i++;
|
||||
j++;
|
||||
if (i < (long) P_NeedleLen)
|
||||
{
|
||||
if (P_Needle[i] == P_Needle[j])
|
||||
{
|
||||
P_KMP_Table[i] = P_KMP_Table[j];
|
||||
}
|
||||
else
|
||||
{
|
||||
P_KMP_Table[i] = j;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
P_KMP_Table[i] = j;
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
/* Build KMP table from right to left. */
|
||||
_UNUSED_ static void _utstring_BuildTableR(
|
||||
const char *P_Needle,
|
||||
size_t P_NeedleLen,
|
||||
long *P_KMP_Table)
|
||||
{
|
||||
long i, j;
|
||||
|
||||
i = P_NeedleLen - 1;
|
||||
j = i + 1;
|
||||
P_KMP_Table[i + 1] = j;
|
||||
while (i >= 0)
|
||||
{
|
||||
while ( (j < (long) P_NeedleLen) && (P_Needle[i] != P_Needle[j]) )
|
||||
{
|
||||
j = P_KMP_Table[j + 1];
|
||||
}
|
||||
i--;
|
||||
j--;
|
||||
if (i >= 0)
|
||||
{
|
||||
if (P_Needle[i] == P_Needle[j])
|
||||
{
|
||||
P_KMP_Table[i + 1] = P_KMP_Table[j + 1];
|
||||
}
|
||||
else
|
||||
{
|
||||
P_KMP_Table[i + 1] = j;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
P_KMP_Table[i + 1] = j;
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
/* Search data from left to right. ( Multiple search mode. ) */
|
||||
_UNUSED_ static long _utstring_find(
|
||||
const char *P_Haystack,
|
||||
size_t P_HaystackLen,
|
||||
const char *P_Needle,
|
||||
size_t P_NeedleLen,
|
||||
long *P_KMP_Table)
|
||||
{
|
||||
long i, j;
|
||||
long V_FindPosition = -1;
|
||||
|
||||
/* Search from left to right. */
|
||||
i = j = 0;
|
||||
while ( (j < (int)P_HaystackLen) && (((P_HaystackLen - j) + i) >= P_NeedleLen) )
|
||||
{
|
||||
while ( (i > -1) && (P_Needle[i] != P_Haystack[j]) )
|
||||
{
|
||||
i = P_KMP_Table[i];
|
||||
}
|
||||
i++;
|
||||
j++;
|
||||
if (i >= (int)P_NeedleLen)
|
||||
{
|
||||
/* Found. */
|
||||
V_FindPosition = j - i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return V_FindPosition;
|
||||
}
|
||||
|
||||
|
||||
/* Search data from right to left. ( Multiple search mode. ) */
|
||||
_UNUSED_ static long _utstring_findR(
|
||||
const char *P_Haystack,
|
||||
size_t P_HaystackLen,
|
||||
const char *P_Needle,
|
||||
size_t P_NeedleLen,
|
||||
long *P_KMP_Table)
|
||||
{
|
||||
long i, j;
|
||||
long V_FindPosition = -1;
|
||||
|
||||
/* Search from right to left. */
|
||||
j = (P_HaystackLen - 1);
|
||||
i = (P_NeedleLen - 1);
|
||||
while ( (j >= 0) && (j >= i) )
|
||||
{
|
||||
while ( (i < (int)P_NeedleLen) && (P_Needle[i] != P_Haystack[j]) )
|
||||
{
|
||||
i = P_KMP_Table[i + 1];
|
||||
}
|
||||
i--;
|
||||
j--;
|
||||
if (i < 0)
|
||||
{
|
||||
/* Found. */
|
||||
V_FindPosition = j + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return V_FindPosition;
|
||||
}
|
||||
|
||||
|
||||
/* Search data from left to right. ( One time search mode. ) */
|
||||
_UNUSED_ static long utstring_find(
|
||||
UT_string *s,
|
||||
long P_StartPosition, /* Start from 0. -1 means last position. */
|
||||
const char *P_Needle,
|
||||
size_t P_NeedleLen)
|
||||
{
|
||||
long V_StartPosition;
|
||||
long V_HaystackLen;
|
||||
long *V_KMP_Table;
|
||||
long V_FindPosition = -1;
|
||||
|
||||
if (P_StartPosition < 0)
|
||||
{
|
||||
V_StartPosition = s->i + P_StartPosition;
|
||||
}
|
||||
else
|
||||
{
|
||||
V_StartPosition = P_StartPosition;
|
||||
}
|
||||
V_HaystackLen = s->i - V_StartPosition;
|
||||
if ( (V_HaystackLen >= (long) P_NeedleLen) && (P_NeedleLen > 0) )
|
||||
{
|
||||
V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1));
|
||||
if (V_KMP_Table != NULL)
|
||||
{
|
||||
_utstring_BuildTable(P_Needle, P_NeedleLen, V_KMP_Table);
|
||||
|
||||
V_FindPosition = _utstring_find(s->d + V_StartPosition,
|
||||
V_HaystackLen,
|
||||
P_Needle,
|
||||
P_NeedleLen,
|
||||
V_KMP_Table);
|
||||
if (V_FindPosition >= 0)
|
||||
{
|
||||
V_FindPosition += V_StartPosition;
|
||||
}
|
||||
|
||||
free(V_KMP_Table);
|
||||
}
|
||||
}
|
||||
|
||||
return V_FindPosition;
|
||||
}
|
||||
|
||||
|
||||
/* Search data from right to left. ( One time search mode. ) */
|
||||
_UNUSED_ static long utstring_findR(
|
||||
UT_string *s,
|
||||
long P_StartPosition, /* Start from 0. -1 means last position. */
|
||||
const char *P_Needle,
|
||||
size_t P_NeedleLen)
|
||||
{
|
||||
long V_StartPosition;
|
||||
long V_HaystackLen;
|
||||
long *V_KMP_Table;
|
||||
long V_FindPosition = -1;
|
||||
|
||||
if (P_StartPosition < 0)
|
||||
{
|
||||
V_StartPosition = s->i + P_StartPosition;
|
||||
}
|
||||
else
|
||||
{
|
||||
V_StartPosition = P_StartPosition;
|
||||
}
|
||||
V_HaystackLen = V_StartPosition + 1;
|
||||
if ( (V_HaystackLen >= (long) P_NeedleLen) && (P_NeedleLen > 0) )
|
||||
{
|
||||
V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1));
|
||||
if (V_KMP_Table != NULL)
|
||||
{
|
||||
_utstring_BuildTableR(P_Needle, P_NeedleLen, V_KMP_Table);
|
||||
|
||||
V_FindPosition = _utstring_findR(s->d,
|
||||
V_HaystackLen,
|
||||
P_Needle,
|
||||
P_NeedleLen,
|
||||
V_KMP_Table);
|
||||
|
||||
free(V_KMP_Table);
|
||||
}
|
||||
}
|
||||
|
||||
return V_FindPosition;
|
||||
}
|
||||
/*******************************************************************************
|
||||
* end substring search functions *
|
||||
******************************************************************************/
|
||||
|
||||
#endif /* UTSTRING_H */
|
||||
Reference in New Issue
Block a user