Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use flatbuffers for some messages from Redis. #341

Merged
merged 14 commits into from
Mar 11, 2017
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Compile the Ray redis module with C++.
  • Loading branch information
robertnishihara committed Mar 8, 2017
commit c5703d2c78720d7aabfec57e60dceea3321376f2
2 changes: 1 addition & 1 deletion python/ray/global_scheduler/test/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
TASK_STATUS_RUNNING = 8
TASK_STATUS_DONE = 16

# These constants are an implementation detail of ray_redis_module.c, so this
# These constants are an implementation detail of ray_redis_module.cc, so this
# must be kept in sync with that file.
DB_CLIENT_PREFIX = "CL:"
TASK_PREFIX = "TT:"
Expand Down
2 changes: 1 addition & 1 deletion python/ray/monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
NIL_ID = b"\xff" * DB_CLIENT_ID_SIZE
# common/task.h
TASK_STATUS_LOST = 32
# common/redis_module/ray_redis_module.c
# common/redis_module/ray_redis_module.cc
TASK_PREFIX = "TT:"
DB_CLIENT_PREFIX = "CL:"
DB_CLIENT_TABLE_NAME = b"db_clients"
Expand Down
2 changes: 1 addition & 1 deletion python/ray/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -723,7 +723,7 @@ def get_address_info_from_redis_helper(redis_address, node_ip_address):
# must have run "CONFIG SET protected-mode no".
redis_client = redis.StrictRedis(host=redis_ip_address, port=int(redis_port))
# The client table prefix must be kept in sync with the file
# "src/common/redis_module/ray_redis_module.c" where it is defined.
# "src/common/redis_module/ray_redis_module.cc" where it is defined.
REDIS_CLIENT_TABLE_PREFIX = "CL:"
client_keys = redis_client.keys("{}*".format(REDIS_CLIENT_TABLE_PREFIX))
# Filter to clients on the same node and do some basic checking.
Expand Down
2 changes: 1 addition & 1 deletion scripts/start_ray.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def check_no_existing_redis_clients(node_ip_address, redis_address):
redis_ip_address, redis_port = redis_address.split(":")
redis_client = redis.StrictRedis(host=redis_ip_address, port=int(redis_port))
# The client table prefix must be kept in sync with the file
# "src/common/redis_module/ray_redis_module.c" where it is defined.
# "src/common/redis_module/ray_redis_module.cc" where it is defined.
REDIS_CLIENT_TABLE_PREFIX = "CL:"
client_keys = redis_client.keys("{}*".format(REDIS_CLIENT_TABLE_PREFIX))
# Filter to clients on the same node and do some basic checking.
Expand Down
6 changes: 3 additions & 3 deletions src/common/redis_module/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@ cmake_minimum_required(VERSION 2.8)
project(ray_redis_module)

if(APPLE)
set(REDIS_MODULE_CFLAGS -W -Wall -dynamic -fno-common -g -ggdb -std=c99 -O2)
set(REDIS_MODULE_CFLAGS -W -Wall -dynamic -fno-common -g -ggdb -std=c++11 -O2)
set(REDIS_MODULE_LDFLAGS "-undefined dynamic_lookup")
set(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
else()
set(REDIS_MODULE_CFLAGS -W -Wall -fno-common -g -ggdb -std=c99 -O2)
set(REDIS_MODULE_CFLAGS -W -Wall -fno-common -g -ggdb -std=c++11 -O2)
set(REDIS_MODULE_LDFLAGS -shared)
endif()

add_library(ray_redis_module SHARED ray_redis_module.c)
add_library(ray_redis_module SHARED ray_redis_module.cc)

target_compile_options(ray_redis_module PUBLIC ${REDIS_MODULE_CFLAGS} -fPIC)
target_link_libraries(ray_redis_module ${REDIS_MODULE_LDFLAGS})
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ RedisModuleKey *OpenPrefixedKey(RedisModuleCtx *ctx,
int mode) {
RedisModuleString *prefixed_keyname =
RedisString_Format(ctx, "%s%S", prefix, keyname);
RedisModuleKey *key = RedisModule_OpenKey(ctx, prefixed_keyname, mode);
RedisModuleKey *key =
(RedisModuleKey *) RedisModule_OpenKey(ctx, prefixed_keyname, mode);
RedisModule_FreeString(ctx, prefixed_keyname);
return key;
}
Expand Down Expand Up @@ -1038,6 +1039,8 @@ int TaskTableGet_RedisCommand(RedisModuleCtx *ctx,
return ReplyWithTask(ctx, argv[1]);
}

extern "C" {

/* This function must be present on each Redis module. It is used in order to
* register the commands into the Redis server. */
int RedisModule_OnLoad(RedisModuleCtx *ctx,
Expand Down Expand Up @@ -1135,3 +1138,5 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx,

return REDISMODULE_OK;
}

} /* extern "C" */
6 changes: 3 additions & 3 deletions src/common/state/redis.cc
Original file line number Diff line number Diff line change
Expand Up @@ -641,7 +641,7 @@ void redis_object_table_subscribe_to_notifications(
TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
/* The object channel prefix must match the value defined in
* src/common/redismodule/ray_redis_module.c. */
* src/common/redismodule/ray_redis_module.cc. */
const char *object_channel_prefix = "OC:";
const char *object_channel_bcast = "BCAST";
int status = REDIS_OK;
Expand Down Expand Up @@ -869,7 +869,7 @@ void redis_task_table_test_and_update(TableCallbackData *callback_data) {
}
}

/* The format of the payload is described in ray_redis_module.c and is
/* The format of the payload is described in ray_redis_module.cc and is
* "<task ID> <state> <local scheduler ID> <task specification>". TODO(rkn):
* Make this code nicer. */
void parse_task_table_subscribe_callback(char *payload,
Expand Down Expand Up @@ -969,7 +969,7 @@ void redis_task_table_subscribe_callback(redisAsyncContext *c,
void redis_task_table_subscribe(TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
TaskTableSubscribeData *data = (TaskTableSubscribeData *) callback_data->data;
/* TASK_CHANNEL_PREFIX is defined in ray_redis_module.c and must be kept in
/* TASK_CHANNEL_PREFIX is defined in ray_redis_module.cc and must be kept in
* sync with that file. */
const char *TASK_CHANNEL_PREFIX = "TT:";
int status;
Expand Down
2 changes: 1 addition & 1 deletion webui/backend/ray_ui.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

IDENTIFIER_LENGTH = 20

# This prefix must match the value defined in ray_redis_module.c.
# This prefix must match the value defined in ray_redis_module.cc.
DB_CLIENT_PREFIX = b"CL:"

def hex_identifier(identifier):
Expand Down